You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2011/10/10 21:58:32 UTC
svn commit: r1181188 [2/8] - in /lucene/dev/branches/solrcloud: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/
dev-tools/idea/modules/grouping/ dev-tools/idea/solr/
dev-tools/idea/solr/contrib/analysis-extras/ dev-tools/idea...
Modified: lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/search/SearcherManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/search/SearcherManager.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/search/SearcherManager.java (original)
+++ lucene/dev/branches/solrcloud/lucene/contrib/misc/src/java/org/apache/lucene/search/SearcherManager.java Mon Oct 10 19:58:24 2011
@@ -17,157 +17,163 @@ package org.apache.lucene.search;
* limitations under the License.
*/
-import java.io.Closeable;
import java.io.IOException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Semaphore;
+import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.NRTManager; // javadocs
+import org.apache.lucene.search.IndexSearcher; // javadocs
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
-/** Utility class to safely share {@link IndexSearcher} instances
- * across multiple threads, while periodically reopening.
- * This class ensures each IndexSearcher instance is not
- * closed until it is no longer needed.
- *
- * <p>Use {@link #acquire} to obtain the current searcher, and
- * {@link #release} to release it, like this:
- *
- * <pre>
- * IndexSearcher s = manager.acquire();
- * try {
- * // Do searching, doc retrieval, etc. with s
- * } finally {
- * manager.release(s);
- * }
- * // Do not use s after this!
- * s = null;
- * </pre>
- *
- * <p>In addition you should periodically call {@link
- * #maybeReopen}. While it's possible to call this just
- * before running each query, this is discouraged since it
- * penalizes the unlucky queries that do the reopen. It's
- * better to use a separate background thread, that
- * periodically calls maybeReopen. Finally, be sure to
- * call {@link #close} once you are done.
- *
- * <p><b>NOTE</b>: if you have an {@link IndexWriter}, it's
- * better to use {@link NRTManager} since that class pulls
- * near-real-time readers from the IndexWriter.
- *
- * @lucene.experimental
+/**
+ * Utility class to safely share {@link IndexSearcher} instances across multiple
+ * threads, while periodically reopening. This class ensures each searcher is
+ * closed only once all threads have finished using it.
+ *
+ * <p>
+ * Use {@link #acquire} to obtain the current searcher, and {@link #release} to
+ * release it, like this:
+ *
+ * <pre>
+ * IndexSearcher s = manager.acquire();
+ * try {
+ * // Do searching, doc retrieval, etc. with s
+ * } finally {
+ * manager.release(s);
+ * }
+ * // Do not use s after this!
+ * s = null;
+ * </pre>
+ *
+ * <p>
+ * In addition you should periodically call {@link #maybeReopen}. While it's
+ * possible to call this just before running each query, this is discouraged
+ * since it penalizes the unlucky queries that do the reopen. It's better to use
+ * a separate background thread, that periodically calls maybeReopen. Finally,
+ * be sure to call {@link #close} once you are done.
+ *
+ * <p>
+ * <b>NOTE</b>: if you have an {@link IndexWriter}, it's better to use
+ * {@link NRTManager} since that class pulls near-real-time readers from the
+ * IndexWriter.
+ *
+ * @lucene.experimental
*/
-public class SearcherManager implements Closeable {
+public abstract class SearcherManager {
- // Current searcher
- private volatile IndexSearcher currentSearcher;
- private final SearcherWarmer warmer;
- private final Semaphore reopening = new Semaphore(1);
- private final ExecutorService es;
-
- /** Opens an initial searcher from the Directory.
- *
- * @param dir Directory to open the searcher from
- *
- * @param warmer optional {@link SearcherWarmer}. Pass
- * null if you don't require the searcher to warmed
- * before going live.
- *
- * <p><b>NOTE</b>: the provided {@link SearcherWarmer} is
- * not invoked for the initial searcher; you should
- * warm it yourself if necessary.
- */
- public SearcherManager(Directory dir, SearcherWarmer warmer) throws IOException {
- this(dir, warmer, null);
- }
-
- /** Opens an initial searcher from the Directory.
- *
- * @param dir Directory to open the searcher from
- *
- * @param warmer optional {@link SearcherWarmer}. Pass
- * null if you don't require the searcher to warmed
- * before going live.
- *
- * @param es optional ExecutorService so different segments can
- * be searched concurrently (see {@link
- * IndexSearcher#IndexSearcher(IndexReader,ExecutorService)}. Pass null
- * to search segments sequentially.
- *
- * <p><b>NOTE</b>: the provided {@link SearcherWarmer} is
- * not invoked for the initial searcher; you should
- * warm it yourself if necessary.
- */
- public SearcherManager(Directory dir, SearcherWarmer warmer, ExecutorService es) throws IOException {
+ protected volatile IndexSearcher currentSearcher;
+ protected final ExecutorService es;
+ protected final SearcherWarmer warmer;
+ protected final Semaphore reopenLock = new Semaphore(1);
+
+ protected SearcherManager(IndexReader openedReader, SearcherWarmer warmer,
+ ExecutorService es) throws IOException {
this.es = es;
- currentSearcher = new IndexSearcher(IndexReader.open(dir), this.es);
this.warmer = warmer;
+ currentSearcher = new IndexSearcher(openedReader, es);
}
- /** You must call this, periodically, to perform a
- * reopen. This calls {@link IndexReader#openIfChanged} on the
- * underlying reader, and if that returns a new reader,
- * it's warmed (if you provided a {@link SearcherWarmer}
- * and then swapped into production.
- *
- * <p><b>Threads</b>: it's fine for more than one thread to
- * call this at once. Only the first thread will attempt
- * the reopen; subsequent threads will see that another
- * thread is already handling reopen and will return
- * immediately. Note that this means if another thread
- * is already reopening then subsequent threads will
- * return right away without waiting for the reader
- * reopen to complete.</p>
- *
- * <p>This method returns true if a new reader was in
- * fact opened.</p>
+ /**
+ * You must call this, periodically, to perform a reopen. This calls
+ * {@link #openIfChanged(IndexReader)} with the underlying reader, and if that returns a
+ * new reader, it's warmed (if you provided a {@link SearcherWarmer} and then
+ * swapped into production.
+ *
+ * <p>
+ * <b>Threads</b>: it's fine for more than one thread to call this at once.
+ * Only the first thread will attempt the reopen; subsequent threads will see
+ * that another thread is already handling reopen and will return immediately.
+ * Note that this means if another thread is already reopening then subsequent
+ * threads will return right away without waiting for the reader reopen to
+ * complete.
+ * </p>
+ *
+ * <p>
+ * This method returns true if a new reader was in fact opened.
+ * </p>
*/
- public boolean maybeReopen()
- throws IOException {
-
- if (currentSearcher == null) {
- throw new AlreadyClosedException("this SearcherManager is closed");
- }
-
+ public boolean maybeReopen() throws IOException {
+ ensureOpen();
// Ensure only 1 thread does reopen at once; other
// threads just return immediately:
- if (reopening.tryAcquire()) {
+ if (reopenLock.tryAcquire()) {
try {
- IndexReader newReader = IndexReader.openIfChanged(currentSearcher.getIndexReader());
+ final IndexReader newReader = openIfChanged(currentSearcher.getIndexReader());
if (newReader != null) {
- IndexSearcher newSearcher = new IndexSearcher(newReader, es);
- if (warmer != null) {
- boolean success = false;
- try {
+ final IndexSearcher newSearcher = new IndexSearcher(newReader, es);
+ boolean success = false;
+ try {
+ if (warmer != null) {
warmer.warm(newSearcher);
- success = true;
- } finally {
- if (!success) {
- newReader.decRef();
- }
+ }
+ swapSearcher(newSearcher);
+ success = true;
+ } finally {
+ if (!success) {
+ release(newSearcher);
}
}
- swapSearcher(newSearcher);
return true;
} else {
return false;
}
} finally {
- reopening.release();
+ reopenLock.release();
}
} else {
return false;
}
}
- /** Obtain the current IndexSearcher. You must match
- * every call to acquire with one call to {@link #release};
- * it's best to do so in a finally clause. */
+ /**
+ * Returns <code>true</code> if no changes have occured since this searcher
+ * ie. reader was opened, otherwise <code>false</code>.
+ * @see IndexReader#isCurrent()
+ */
+ public boolean isSearcherCurrent() throws CorruptIndexException,
+ IOException {
+ final IndexSearcher searcher = acquire();
+ try {
+ return searcher.getIndexReader().isCurrent();
+ } finally {
+ release(searcher);
+ }
+ }
+
+ /**
+ * Release the searcher previously obtained with {@link #acquire}.
+ *
+ * <p>
+ * <b>NOTE</b>: it's safe to call this after {@link #close}.
+ */
+ public void release(IndexSearcher searcher) throws IOException {
+ assert searcher != null;
+ searcher.getIndexReader().decRef();
+ }
+
+ /**
+ * Close this SearcherManager to future searching. Any searches still in
+ * process in other threads won't be affected, and they should still call
+ * {@link #release} after they are done.
+ */
+ public synchronized void close() throws IOException {
+ if (currentSearcher != null) {
+ // make sure we can call this more than once
+ // closeable javadoc says:
+ // if this is already closed then invoking this method has no effect.
+ swapSearcher(null);
+ }
+ }
+
+ /**
+ * Obtain the current IndexSearcher. You must match every call to acquire with
+ * one call to {@link #release}; it's best to do so in a finally clause.
+ */
public IndexSearcher acquire() {
IndexSearcher searcher;
do {
@@ -176,35 +182,130 @@ public class SearcherManager implements
}
} while (!searcher.getIndexReader().tryIncRef());
return searcher;
- }
-
- /** Release the searcher previously obtained with {@link
- * #acquire}.
- *
- * <p><b>NOTE</b>: it's safe to call this after {@link
- * #close}. */
- public void release(IndexSearcher searcher)
- throws IOException {
- searcher.getIndexReader().decRef();
}
- // Replaces old searcher with new one - needs to be synced to make close() work
- private synchronized void swapSearcher(IndexSearcher newSearcher)
- throws IOException {
- IndexSearcher oldSearcher = currentSearcher;
- if (oldSearcher == null) {
+ private void ensureOpen() {
+ if (currentSearcher == null) {
throw new AlreadyClosedException("this SearcherManager is closed");
}
+ }
+
+ protected synchronized void swapSearcher(IndexSearcher newSearcher) throws IOException {
+ ensureOpen();
+ final IndexSearcher oldSearcher = currentSearcher;
currentSearcher = newSearcher;
release(oldSearcher);
}
- /** Close this SearcherManager to future searching. Any
- * searches still in process in other threads won't be
- * affected, and they should still call {@link #release}
- * after they are done. */
- @Override
- public void close() throws IOException {
- swapSearcher(null);
+ protected abstract IndexReader openIfChanged(IndexReader oldReader)
+ throws IOException;
+
+ /**
+ * Creates and returns a new SearcherManager from the given {@link IndexWriter}.
+ * @param writer the IndexWriter to open the IndexReader from.
+ * @param applyAllDeletes If <code>true</code>, all buffered deletes will
+ * be applied (made visible) in the {@link IndexSearcher} / {@link IndexReader}.
+ * If <code>false</code>, the deletes are not applied but remain buffered
+ * (in IndexWriter) so that they will be applied in the future.
+ * Applying deletes can be costly, so if your app can tolerate deleted documents
+ * being returned you might gain some performance by passing <code>false</code>.
+ * @param warmer An optional {@link SearcherWarmer}. Pass
+ * <code>null</code> if you don't require the searcher to warmed
+ * before going live. If this is <code>non-null</code> then a
+ * merged segment warmer is installed on the
+ * provided IndexWriter's config.
+ * @param es An optional {@link ExecutorService} so different segments can
+ * be searched concurrently (see {@link
+ * IndexSearcher#IndexSearcher(IndexReader,ExecutorService)}. Pass <code>null</code>
+ * to search segments sequentially.
+ *
+ * @see IndexReader#openIfChanged(IndexReader, IndexWriter, boolean)
+ * @throws IOException
+ */
+ public static SearcherManager open(IndexWriter writer, boolean applyAllDeletes,
+ SearcherWarmer warmer, ExecutorService es) throws IOException {
+ final IndexReader open = IndexReader.open(writer, true);
+ boolean success = false;
+ try {
+ SearcherManager manager = new NRTSearcherManager(writer, applyAllDeletes,
+ open, warmer, es);
+ success = true;
+ return manager;
+ } finally {
+ if (!success) {
+ open.close();
+ }
+ }
+ }
+
+ /**
+ * Creates and returns a new SearcherManager from the given {@link Directory}.
+ * @param dir the directory to open the IndexReader on.
+ * @param warmer An optional {@link SearcherWarmer}. Pass
+ * <code>null</code> if you don't require the searcher to warmed
+ * before going live. If this is <code>non-null</code> then a
+ * merged segment warmer is installed on the
+ * provided IndexWriter's config.
+ * @param es And optional {@link ExecutorService} so different segments can
+ * be searched concurrently (see {@link
+ * IndexSearcher#IndexSearcher(IndexReader,ExecutorService)}. Pass <code>null</code>
+ * to search segments sequentially.
+ *
+ * @throws IOException
+ */
+ public static SearcherManager open(Directory dir, SearcherWarmer warmer,
+ ExecutorService es) throws IOException {
+ final IndexReader open = IndexReader.open(dir, true);
+ boolean success = false;
+ try {
+ SearcherManager manager = new DirectorySearchManager(open, warmer, es);
+ success = true;
+ return manager;
+ } finally {
+ if (!success) {
+ open.close();
+ }
+ }
+ }
+
+ static final class NRTSearcherManager extends SearcherManager {
+ private final IndexWriter writer;
+ private final boolean applyDeletes;
+
+ NRTSearcherManager(final IndexWriter writer, final boolean applyDeletes,
+ final IndexReader openedReader, final SearcherWarmer warmer, final ExecutorService es)
+ throws IOException {
+ super(openedReader, warmer, es);
+ this.writer = writer;
+ this.applyDeletes = applyDeletes;
+ if (warmer != null) {
+ writer.getConfig().setMergedSegmentWarmer(
+ new IndexWriter.IndexReaderWarmer() {
+ @Override
+ public void warm(IndexReader reader) throws IOException {
+ warmer.warm(new IndexSearcher(reader, es));
+ }
+ });
+ }
+ }
+
+ @Override
+ protected IndexReader openIfChanged(IndexReader oldReader)
+ throws IOException {
+ return IndexReader.openIfChanged(oldReader, writer, applyDeletes);
+ }
+ }
+
+ static final class DirectorySearchManager extends SearcherManager {
+ DirectorySearchManager(IndexReader openedReader,
+ SearcherWarmer warmer, ExecutorService es) throws IOException {
+ super(openedReader, warmer, es);
+ }
+
+ @Override
+ protected IndexReader openIfChanged(IndexReader oldReader)
+ throws IOException {
+ return IndexReader.openIfChanged(oldReader, true);
+ }
}
}
Modified: lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java (original)
+++ lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java Mon Oct 10 19:58:24 2011
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.concurrent.ExecutorService;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.SearcherManager;
import org.apache.lucene.search.SearcherWarmer;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.store.Directory;
@@ -41,7 +42,8 @@ public class TestNRTManager extends Thre
if (VERBOSE) {
System.out.println("TEST: finalSearcher maxGen=" + maxGen);
}
- return nrt.get(maxGen, true);
+ final SearcherManager manager = nrt.waitForGeneration(maxGen, true);
+ return manager.acquire();
}
@Override
@@ -67,14 +69,15 @@ public class TestNRTManager extends Thre
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": nrt: verify " + id);
}
- final IndexSearcher s = nrt.get(gen, true);
+ SearcherManager manager = nrt.waitForGeneration(gen, true);
+ final IndexSearcher s = manager.acquire();
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": nrt: got searcher=" + s);
}
try {
assertEquals(docs.size(), s.search(new TermQuery(id), 10).totalHits);
} finally {
- nrt.release(s);
+ manager.release(s);
}
}
@@ -89,14 +92,15 @@ public class TestNRTManager extends Thre
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": nrt: verify " + id);
}
- final IndexSearcher s = nrt.get(gen, false);
+ final SearcherManager manager = nrt.waitForGeneration(gen, false);
+ final IndexSearcher s = manager.acquire();
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": nrt: got searcher=" + s);
}
try {
assertEquals(docs.size(), s.search(new TermQuery(id), 10).totalHits);
} finally {
- nrt.release(s);
+ manager.release(s);
}
}
lastGens.set(gen);
@@ -111,14 +115,15 @@ public class TestNRTManager extends Thre
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": nrt: verify " + id);
}
- final IndexSearcher s = nrt.get(gen, false);
+ final SearcherManager manager = nrt.waitForGeneration(gen, false);
+ final IndexSearcher s = manager.acquire();
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": nrt: got searcher=" + s);
}
try {
assertEquals(1, s.search(new TermQuery(id), 10).totalHits);
} finally {
- nrt.release(s);
+ manager.release(s);
}
}
lastGens.set(gen);
@@ -132,14 +137,15 @@ public class TestNRTManager extends Thre
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": nrt: verify " + id);
}
- final IndexSearcher s = nrt.get(gen, true);
+ final SearcherManager manager = nrt.waitForGeneration(gen, true);
+ final IndexSearcher s = manager.acquire();
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": nrt: got searcher=" + s);
}
try {
assertEquals(1, s.search(new TermQuery(id), 10).totalHits);
} finally {
- nrt.release(s);
+ manager.release(s);
}
}
lastGens.set(gen);
@@ -153,14 +159,15 @@ public class TestNRTManager extends Thre
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": nrt: verify del " + id);
}
- final IndexSearcher s = nrt.get(gen, true);
+ final SearcherManager manager = nrt.waitForGeneration(gen, true);
+ final IndexSearcher s = manager.acquire();
if (VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": nrt: got searcher=" + s);
}
try {
assertEquals(0, s.search(new TermQuery(id), 10).totalHits);
} finally {
- nrt.release(s);
+ manager.release(s);
}
}
lastGens.set(gen);
@@ -168,7 +175,6 @@ public class TestNRTManager extends Thre
private NRTManager nrt;
private NRTManagerReopenThread nrtThread;
-
@Override
protected void doAfterWriter(ExecutorService es) throws Exception {
final double minReopenSec = 0.01 + 0.05 * random.nextDouble();
@@ -185,7 +191,8 @@ public class TestNRTManager extends Thre
TestNRTManager.this.warmCalled = true;
s.search(new TermQuery(new Term("body", "united")), 10);
}
- });
+ }, false);
+
nrtThread = new NRTManagerReopenThread(nrt, maxReopenSec, minReopenSec);
nrtThread.setName("NRT Reopen Thread");
nrtThread.setPriority(Math.min(Thread.currentThread().getPriority()+2, Thread.MAX_PRIORITY));
@@ -214,12 +221,12 @@ public class TestNRTManager extends Thre
@Override
protected IndexSearcher getCurrentSearcher() throws Exception {
- return nrt.get(random.nextBoolean());
+ return nrt.getSearcherManager(false).acquire();
}
@Override
protected void releaseSearcher(IndexSearcher s) throws Exception {
- nrt.release(s);
+ nrt.getSearcherManager(false).release(s);
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java (original)
+++ lucene/dev/branches/solrcloud/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java Mon Oct 10 19:58:24 2011
@@ -18,10 +18,20 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
-
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.NamedThreadFactory;
import org.apache.lucene.util._TestUtil;
public class TestSearcherManager extends ThreadedIndexingAndSearchingTestCase {
@@ -34,25 +44,35 @@ public class TestSearcherManager extends
@Override
protected IndexSearcher getFinalSearcher() throws Exception {
- writer.commit();
- mgr.maybeReopen();
+ if (!isNRT) {
+ writer.commit();
+ }
+ assertTrue(mgr.maybeReopen() || mgr.isSearcherCurrent());
return mgr.acquire();
}
private SearcherManager mgr;
+ private boolean isNRT;
@Override
protected void doAfterWriter(ExecutorService es) throws Exception {
// SearcherManager needs to see empty commit:
- writer.commit();
- mgr = new SearcherManager(dir,
- new SearcherWarmer() {
- @Override
- public void warm(IndexSearcher s) throws IOException {
- TestSearcherManager.this.warmCalled = true;
- s.search(new TermQuery(new Term("body", "united")), 10);
- }
- }, es);
+ final SearcherWarmer warmer = new SearcherWarmer() {
+ @Override
+ public void warm(IndexSearcher s) throws IOException {
+ TestSearcherManager.this.warmCalled = true;
+ s.search(new TermQuery(new Term("body", "united")), 10);
+ }
+ };
+ if (random.nextBoolean()) {
+ mgr = SearcherManager.open(writer, true, warmer, es);
+ isNRT = true;
+ } else {
+ writer.commit();
+ mgr = SearcherManager.open(dir, warmer, es);
+ isNRT = false;
+ }
+
}
@Override
@@ -110,4 +130,78 @@ public class TestSearcherManager extends
}
mgr.close();
}
+
+ public void testIntermediateClose() throws IOException, InterruptedException {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
+ TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ writer.addDocument(new Document());
+ writer.commit();
+ final CountDownLatch awaitEnterWarm = new CountDownLatch(1);
+ final CountDownLatch awaitClose = new CountDownLatch(1);
+ final ExecutorService es = random.nextBoolean() ? null : Executors.newCachedThreadPool(new NamedThreadFactory("testIntermediateClose"));
+ final SearcherWarmer warmer = new SearcherWarmer() {
+ @Override
+ public void warm(IndexSearcher s) throws IOException {
+ try {
+ awaitEnterWarm.countDown();
+ awaitClose.await();
+ } catch (InterruptedException e) {
+ //
+ }
+ }
+ };
+ final SearcherManager searcherManager = random.nextBoolean() ? SearcherManager.open(dir,
+ warmer, es) : SearcherManager.open(writer, random.nextBoolean(), warmer, es);
+ IndexSearcher searcher = searcherManager.acquire();
+ try {
+ assertEquals(1, searcher.getIndexReader().numDocs());
+ } finally {
+ searcherManager.release(searcher);
+ }
+ writer.addDocument(new Document());
+ writer.commit();
+ final AtomicBoolean success = new AtomicBoolean(false);
+ final AtomicBoolean triedReopen = new AtomicBoolean(false);
+ final Throwable[] exc = new Throwable[1];
+ Thread thread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ triedReopen.set(true);
+ searcherManager.maybeReopen();
+ success.set(true);
+ } catch (AlreadyClosedException e) {
+ // expected
+ } catch (Throwable e) {
+ exc[0] = e;
+ // use success as the barrier here to make sure we see the write
+ success.set(false);
+
+ }
+ }
+ });
+ thread.start();
+ awaitEnterWarm.await();
+ for (int i = 0; i < 2; i++) {
+ searcherManager.close();
+ }
+ awaitClose.countDown();
+ thread.join();
+ try {
+ searcherManager.acquire();
+ fail("already closed");
+ } catch (AlreadyClosedException ex) {
+ // expected
+ }
+ assertFalse(success.get());
+ assertTrue(triedReopen.get());
+ assertNull("" + exc[0], exc[0]);
+ writer.close();
+ dir.close();
+ if (es != null) {
+ es.shutdown();
+ es.awaitTermination(1, TimeUnit.SECONDS);
+ }
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java Mon Oct 10 19:58:24 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.document;
import java.io.Reader;
import java.util.Comparator;
-import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.index.IndexableFieldType;
import org.apache.lucene.index.values.PerDocFieldValues;
import org.apache.lucene.index.values.ValueType;
@@ -317,21 +316,34 @@ public class IndexDocValuesField extends
final String value;
switch (type) {
case BYTES_FIXED_DEREF:
- case BYTES_FIXED_SORTED:
case BYTES_FIXED_STRAIGHT:
case BYTES_VAR_DEREF:
- case BYTES_VAR_SORTED:
case BYTES_VAR_STRAIGHT:
- value = "bytes:bytes.utf8ToString();";
+ case BYTES_FIXED_SORTED:
+ case BYTES_VAR_SORTED:
+ // don't use to unicode string this is not necessarily unicode here
+ value = "bytes: " + bytes.toString();
+ break;
+ case FIXED_INTS_16:
+ value = "int16: " + longValue;
+ break;
+ case FIXED_INTS_32:
+ value = "int32: " + longValue;
+ break;
+ case FIXED_INTS_64:
+ value = "int64: " + longValue;
+ break;
+ case FIXED_INTS_8:
+ value = "int8: " + longValue;
break;
case VAR_INTS:
- value = "int:" + longValue;
+ value = "vint: " + longValue;
break;
case FLOAT_32:
- value = "float32:" + doubleValue;
+ value = "float32: " + doubleValue;
break;
case FLOAT_64:
- value = "float64:" + doubleValue;
+ value = "float64: " + doubleValue;
break;
default:
throw new IllegalArgumentException("unknown type: " + type);
@@ -353,14 +365,18 @@ public class IndexDocValuesField extends
final IndexDocValuesField valField = new IndexDocValuesField(field.name(), field.fieldType(), field.stringValue());
switch (type) {
case BYTES_FIXED_DEREF:
- case BYTES_FIXED_SORTED:
case BYTES_FIXED_STRAIGHT:
case BYTES_VAR_DEREF:
- case BYTES_VAR_SORTED:
case BYTES_VAR_STRAIGHT:
+ case BYTES_FIXED_SORTED:
+ case BYTES_VAR_SORTED:
BytesRef ref = field.isBinary() ? field.binaryValue() : new BytesRef(field.stringValue());
valField.setBytes(ref, type);
break;
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
case VAR_INTS:
valField.setInt(Long.parseLong(field.stringValue()));
break;
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/CheckIndex.java Mon Oct 10 19:58:24 2011
@@ -41,7 +41,7 @@ import java.util.Map;
import org.apache.lucene.index.codecs.BlockTreeTermsReader;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.values.IndexDocValues;
-import org.apache.lucene.index.values.ValuesEnum;
+import org.apache.lucene.index.values.IndexDocValues.Source;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -1070,27 +1070,28 @@ public class CheckIndex {
if (docValues == null) {
continue;
}
- final ValuesEnum values = docValues.getEnum();
- while (values.nextDoc() != ValuesEnum.NO_MORE_DOCS) {
+ final Source values = docValues.getDirectSource();
+ final int maxDoc = reader.maxDoc();
+ for (int i = 0; i < maxDoc; i++) {
switch (fieldInfo.docValues) {
- case BYTES_FIXED_DEREF:
case BYTES_FIXED_SORTED:
+ case BYTES_VAR_SORTED:
+ case BYTES_FIXED_DEREF:
case BYTES_FIXED_STRAIGHT:
case BYTES_VAR_DEREF:
- case BYTES_VAR_SORTED:
case BYTES_VAR_STRAIGHT:
- values.bytes();
+ values.getBytes(i, new BytesRef());
break;
case FLOAT_32:
case FLOAT_64:
- values.getFloat();
+ values.getFloat(i);
break;
case VAR_INTS:
case FIXED_INTS_16:
case FIXED_INTS_32:
case FIXED_INTS_64:
case FIXED_INTS_8:
- values.getInt();
+ values.getInt(i);
break;
default:
throw new IllegalArgumentException("Field: " + fieldInfo.name
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java Mon Oct 10 19:58:24 2011
@@ -186,13 +186,41 @@ public abstract class DocumentsWriterPer
if (numThreadStatesActive < perThreads.length) {
final ThreadState threadState = perThreads[numThreadStatesActive];
threadState.lock(); // lock so nobody else will get this ThreadState
- numThreadStatesActive++; // increment will publish the ThreadState
- threadState.perThread.initialize();
- return threadState;
+ boolean unlock = true;
+ try {
+ if (threadState.isActive()) {
+ // unreleased thread states are deactivated during DW#close()
+ numThreadStatesActive++; // increment will publish the ThreadState
+ assert threadState.perThread != null;
+ threadState.perThread.initialize();
+ unlock = false;
+ return threadState;
+ }
+ // unlock since the threadstate is not active anymore - we are closed!
+ assert assertUnreleasedThreadStatesInactive();
+ return null;
+ } finally {
+ if (unlock) {
+ // in any case make sure we unlock if we fail
+ threadState.unlock();
+ }
+ }
}
return null;
}
+ private synchronized boolean assertUnreleasedThreadStatesInactive() {
+ for (int i = numThreadStatesActive; i < perThreads.length; i++) {
+ assert perThreads[i].tryLock() : "unreleased threadstate should not be locked";
+ try {
+ assert !perThreads[i].isActive() : "expected unreleased thread state to be inactive";
+ } finally {
+ perThreads[i].unlock();
+ }
+ }
+ return true;
+ }
+
/**
* Deactivate all unreleased threadstates
*/
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfo.java Mon Oct 10 19:58:24 2011
@@ -86,7 +86,7 @@ public final class FieldInfo {
public int getCodecId() {
return codecId;
}
-
+
@Override
public Object clone() {
FieldInfo clone = new FieldInfo(name, isIndexed, number, storeTermVector, storePositionWithTermVector,
@@ -132,6 +132,12 @@ public final class FieldInfo {
}
}
+ public void resetDocValues(ValueType v) {
+ if (docValues != null) {
+ docValues = v;
+ }
+ }
+
public boolean hasDocValues() {
return docValues != null;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/FieldInfos.java Mon Oct 10 19:58:24 2011
@@ -652,31 +652,30 @@ public final class FieldInfos implements
case BYTES_FIXED_DEREF:
b = 5;
break;
- case BYTES_FIXED_SORTED:
+ case BYTES_VAR_STRAIGHT:
b = 6;
break;
- case BYTES_VAR_STRAIGHT:
+ case BYTES_VAR_DEREF:
b = 7;
break;
- case BYTES_VAR_DEREF:
+ case FIXED_INTS_16:
b = 8;
break;
- case BYTES_VAR_SORTED:
+ case FIXED_INTS_32:
b = 9;
break;
- case FIXED_INTS_16:
+ case FIXED_INTS_64:
b = 10;
break;
- case FIXED_INTS_32:
+ case FIXED_INTS_8:
b = 11;
break;
- case FIXED_INTS_64:
+ case BYTES_FIXED_SORTED:
b = 12;
break;
- case FIXED_INTS_8:
+ case BYTES_VAR_SORTED:
b = 13;
break;
-
default:
throw new IllegalStateException("unhandled indexValues type " + fi.docValues);
}
@@ -754,29 +753,29 @@ public final class FieldInfos implements
docValuesType = ValueType.BYTES_FIXED_DEREF;
break;
case 6:
- docValuesType = ValueType.BYTES_FIXED_SORTED;
+ docValuesType = ValueType.BYTES_VAR_STRAIGHT;
break;
case 7:
- docValuesType = ValueType.BYTES_VAR_STRAIGHT;
+ docValuesType = ValueType.BYTES_VAR_DEREF;
break;
case 8:
- docValuesType = ValueType.BYTES_VAR_DEREF;
+ docValuesType = ValueType.FIXED_INTS_16;
break;
case 9:
- docValuesType = ValueType.BYTES_VAR_SORTED;
+ docValuesType = ValueType.FIXED_INTS_32;
break;
case 10:
- docValuesType = ValueType.FIXED_INTS_16;
+ docValuesType = ValueType.FIXED_INTS_64;
break;
case 11:
- docValuesType = ValueType.FIXED_INTS_32;
+ docValuesType = ValueType.FIXED_INTS_8;
break;
case 12:
- docValuesType = ValueType.FIXED_INTS_64;
+ docValuesType = ValueType.BYTES_FIXED_SORTED;
break;
case 13:
- docValuesType = ValueType.FIXED_INTS_8;
- break;
+ docValuesType = ValueType.BYTES_VAR_SORTED;
+ break;
default:
throw new IllegalStateException("unhandled indexValues type " + b);
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiPerDocValues.java Mon Oct 10 19:58:24 2011
@@ -128,7 +128,7 @@ public class MultiPerDocValues extends P
if (docsUpto != start) {
type = values.type();
docValuesIndex.add(new MultiIndexDocValues.DocValuesIndex(
- new MultiIndexDocValues.DummyDocValues(start, type), docsUpto, start
+ new MultiIndexDocValues.EmptyDocValues(start, type), docsUpto, start
- docsUpto));
}
docValuesIndex.add(new MultiIndexDocValues.DocValuesIndex(values, start,
@@ -137,7 +137,7 @@ public class MultiPerDocValues extends P
} else if (i + 1 == subs.length && !docValuesIndex.isEmpty()) {
docValuesIndex.add(new MultiIndexDocValues.DocValuesIndex(
- new MultiIndexDocValues.DummyDocValues(start, type), docsUpto, start
+ new MultiIndexDocValues.EmptyDocValues(start, type), docsUpto, start
- docsUpto));
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Mon Oct 10 19:58:24 2011
@@ -33,7 +33,6 @@ import org.apache.lucene.index.codecs.Fi
import org.apache.lucene.index.codecs.FieldsWriter;
import org.apache.lucene.index.codecs.MergeState;
import org.apache.lucene.index.codecs.PerDocConsumer;
-import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
@@ -141,6 +140,8 @@ final class SegmentMerger {
if (fieldInfos.hasVectors()) {
mergeVectors();
}
+ // write FIS once merge is done. IDV might change types or drops fields
+ fieldInfos.write(directory, segment + "." + IndexFileNames.FIELD_INFOS_EXTENSION);
return mergedDocs;
}
@@ -254,7 +255,6 @@ final class SegmentMerger {
}
}
final SegmentCodecs codecInfo = fieldInfos.buildSegmentCodecs(false);
- fieldInfos.write(directory, segment + "." + IndexFileNames.FIELD_INFOS_EXTENSION);
int docCount = 0;
@@ -584,28 +584,11 @@ final class SegmentMerger {
}
private void mergePerDoc() throws IOException {
- final List<PerDocValues> perDocProducers = new ArrayList<PerDocValues>();
- final List<ReaderUtil.Slice> perDocSlices = new ArrayList<ReaderUtil.Slice>();
- int docBase = 0;
- for (MergeState.IndexReaderAndLiveDocs r : readers) {
- final int maxDoc = r.reader.maxDoc();
- final PerDocValues producer = r.reader.perDocValues();
- if (producer != null) {
- perDocSlices.add(new ReaderUtil.Slice(docBase, maxDoc, perDocProducers
- .size()));
- perDocProducers.add(producer);
- }
- docBase += maxDoc;
- }
- if (!perDocSlices.isEmpty()) {
final PerDocConsumer docsConsumer = codec
.docsConsumer(new PerDocWriteState(segmentWriteState));
boolean success = false;
try {
- final MultiPerDocValues multiPerDocValues = new MultiPerDocValues(
- perDocProducers.toArray(PerDocValues.EMPTY_ARRAY),
- perDocSlices.toArray(ReaderUtil.Slice.EMPTY_ARRAY));
- docsConsumer.merge(mergeState, multiPerDocValues);
+ docsConsumer.merge(mergeState);
success = true;
} finally {
if (success) {
@@ -614,11 +597,8 @@ final class SegmentMerger {
IOUtils.closeWhileHandlingException(docsConsumer);
}
}
- }
- /* don't close the perDocProducers here since they are private segment producers
- * and will be closed once the SegmentReader goes out of scope */
}
-
+
private MergeState mergeState;
public boolean getAnyNonBulkMerges() {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/Codec.java Mon Oct 10 19:58:24 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.index.codecs;
*/
import java.io.IOException;
-import java.util.Comparator;
import java.util.Set;
import org.apache.lucene.index.PerDocWriteState;
@@ -26,7 +25,6 @@ import org.apache.lucene.index.SegmentIn
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
/** @lucene.experimental */
public abstract class Codec {
@@ -34,22 +32,9 @@ public abstract class Codec {
/** Unique name that's used to retrieve this codec when
* reading the index */
public final String name;
- protected final boolean dvUseCompoundFile;
- protected final Comparator<BytesRef> docValuesSortComparator;
protected Codec(String name) {
- this(name, true);
- }
-
- protected Codec(String name, boolean docValuesUseCompoundFile) {
- this(name, docValuesUseCompoundFile, BytesRef.getUTF8SortedAsUnicodeComparator());
- }
-
- protected Codec(String name, boolean docValuesUseCompoundFile,
- Comparator<BytesRef> docValuesSortComparator) {
this.name = name;
- this.dvUseCompoundFile = docValuesUseCompoundFile;
- this.docValuesSortComparator = docValuesSortComparator;
}
/** Writes a new segment */
@@ -87,38 +72,6 @@ public abstract class Codec {
/** Records all file extensions this codec uses */
public abstract void getExtensions(Set<String> extensions);
-
- /**
- * Returns <code>true</code> iff compound file should be used for
- * IndexDocValues, otherwise <code>false</code>. The default is
- * <code>true</code>.
- * <p>
- * NOTE: To change the default value you need to subclass a {@link Codec} with
- * a distinct name since this value is final and should not be changed to
- * prevent the risk of a index corruption. This setting is private to a
- * {@link Codec}. If you intend to change this value on an existing
- * {@link Codec} re-indexing is required.
- *
- * @return <code>true</code> iff compound file should be used for
- * IndexDocValues, otherwise <code>false</code>.
- */
- public boolean getDocValuesUseCFS() {
- return dvUseCompoundFile;
- }
-
- /**
- * Returns the {@link BytesRef} comparator for sorted IndexDocValue variants.
- * The default is {@link BytesRef#getUTF8SortedAsUnicodeComparator()}.
- * <p>
- * NOTE: To change the default value you need to subclass a {@link Codec} with
- * a distinct name since this value is final and should not be changed to
- * prevent the risk of a index corruption. This setting is private to a
- * {@link Codec}. If you intend to change this value on an existing
- * {@link Codec} re-indexing is required.
- */
- public Comparator<BytesRef> getDocValuesSortComparator() {
- return docValuesSortComparator;
- }
@Override
public String toString() {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java Mon Oct 10 19:58:24 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.index.codecs;
*/
import java.io.IOException;
-import java.util.Comparator;
import java.util.Set;
import org.apache.lucene.index.FieldInfo;
@@ -26,113 +25,51 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.PerDocWriteState;
import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.values.Writer;
+import org.apache.lucene.index.codecs.DocValuesWriterBase;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.Counter;
/**
- *
+ * Default PerDocConsumer implementation that uses compound file.
* @lucene.experimental
*/
-public class DefaultDocValuesConsumer extends PerDocConsumer {
- private final String segmentName;
- private final int codecId;
+public class DefaultDocValuesConsumer extends DocValuesWriterBase {
private final Directory directory;
- private final Counter bytesUsed;
- private final Comparator<BytesRef> comparator;
- private boolean useCompoundFile;
- private final IOContext context;
- public DefaultDocValuesConsumer(PerDocWriteState state, Comparator<BytesRef> comparator, boolean useCompoundFile) throws IOException {
- this.segmentName = state.segmentName;
- this.codecId = state.codecId;
- this.bytesUsed = state.bytesUsed;
- this.context = state.context;
+ public DefaultDocValuesConsumer(PerDocWriteState state) throws IOException {
+ super(state);
//TODO maybe we should enable a global CFS that all codecs can pull on demand to further reduce the number of files?
- this.directory = useCompoundFile ? new CompoundFileDirectory(state.directory,
- IndexFileNames.segmentFileName(segmentName, codecId,
- IndexFileNames.COMPOUND_FILE_EXTENSION), context, true) : state.directory;
- this.comparator = comparator;
- this.useCompoundFile = useCompoundFile;
+ this.directory = new CompoundFileDirectory(state.directory,
+ IndexFileNames.segmentFileName(state.segmentName, state.codecId,
+ IndexFileNames.COMPOUND_FILE_EXTENSION), state.context, true);
}
-
- public void close() throws IOException {
- if (useCompoundFile) {
- this.directory.close();
- }
+
+ @Override
+ protected Directory getDirectory() {
+ return directory;
}
@Override
- public DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
- return Writer.create(field.getDocValues(),
- docValuesId(segmentName, codecId, field.number),
- directory, comparator, bytesUsed, context);
+ public void close() throws IOException {
+ this.directory.close();
}
-
+
@SuppressWarnings("fallthrough")
- public static void files(Directory dir, SegmentInfo segmentInfo, int codecId,
- Set<String> files, boolean useCompoundFile) throws IOException {
+ public static void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set<String> files) throws IOException {
FieldInfos fieldInfos = segmentInfo.getFieldInfos();
for (FieldInfo fieldInfo : fieldInfos) {
if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) {
- String filename = docValuesId(segmentInfo.name, codecId,
- fieldInfo.number);
- if (useCompoundFile) {
- files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION));
- files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
- assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
- assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION));
- return;
- } else {
- switch (fieldInfo.getDocValues()) {
- case BYTES_FIXED_DEREF:
- case BYTES_VAR_DEREF:
- case BYTES_VAR_SORTED:
- case BYTES_FIXED_SORTED:
- case BYTES_VAR_STRAIGHT:
- files.add(IndexFileNames.segmentFileName(filename, "",
- Writer.INDEX_EXTENSION));
- assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
- Writer.INDEX_EXTENSION));
- // until here all types use an index
- case BYTES_FIXED_STRAIGHT:
- case FLOAT_32:
- case FLOAT_64:
- case VAR_INTS:
- case FIXED_INTS_16:
- case FIXED_INTS_32:
- case FIXED_INTS_64:
- case FIXED_INTS_8:
- files.add(IndexFileNames.segmentFileName(filename, "",
- Writer.DATA_EXTENSION));
- assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
- Writer.DATA_EXTENSION));
- break;
-
- default:
- assert false;
- }
- }
+ files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION));
+ files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
+ assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
+ assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION));
+ return;
}
}
}
-
- static String docValuesId(String segmentsName, int codecID, int fieldId) {
- return segmentsName + "_" + codecID + "-" + fieldId;
- }
-
- public static void getDocValuesExtensions(Set<String> extensions, boolean useCompoundFile) {
- if (useCompoundFile) {
- extensions.add(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION);
- extensions.add(IndexFileNames.COMPOUND_FILE_EXTENSION);
- } else {
- extensions.add(Writer.DATA_EXTENSION);
- extensions.add(Writer.INDEX_EXTENSION);
- }
+ public static void getExtensions(Set<String> extensions) {
+ extensions.add(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION);
+ extensions.add(IndexFileNames.COMPOUND_FILE_EXTENSION);
}
-
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java Mon Oct 10 19:58:24 2011
@@ -16,184 +16,50 @@ package org.apache.lucene.index.codecs;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
import java.io.Closeable;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Comparator;
+import java.util.Map;
import java.util.TreeMap;
-import org.apache.lucene.index.FieldInfo;
-import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.IndexFileNames;
-import org.apache.lucene.index.SegmentInfo;
-import org.apache.lucene.index.values.Bytes;
+import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.codecs.DocValuesReaderBase;
import org.apache.lucene.index.values.IndexDocValues;
-import org.apache.lucene.index.values.Floats;
-import org.apache.lucene.index.values.Ints;
-import org.apache.lucene.index.values.ValueType;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.store.IOContext;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
/**
- * Abstract base class for FieldsProducer implementations supporting
- * {@link IndexDocValues}.
- *
+ * Default PerDocValues implementation that uses compound file.
* @lucene.experimental
*/
-public class DefaultDocValuesProducer extends PerDocValues {
-
+public class DefaultDocValuesProducer extends DocValuesReaderBase {
protected final TreeMap<String, IndexDocValues> docValues;
- private final boolean useCompoundFile;
- private final Closeable cfs;
- private final Comparator<BytesRef> sortComparator;
+ private final Directory cfs;
/**
- *
* Creates a new {@link DefaultDocValuesProducer} instance and loads all
* {@link IndexDocValues} instances for this segment and codec.
- *
- * @param si
- * the segment info to load the {@link IndexDocValues} for.
- * @param dir
- * the directory to load the {@link IndexDocValues} from.
- * @param fieldInfo
- * the {@link FieldInfos}
- * @param codecId
- * the codec ID
- * @param useCompoundFile
- * if <code>true</code> this producer opens a compound file to read
- * IndexDocValues fields, otherwise each field defines its own set of
- * files.
- * @param sortComparator
- * defines the sort order for sorted IndexDocValues variants
- * @throws IOException
- * if an {@link IOException} occurs
*/
- public DefaultDocValuesProducer(SegmentInfo si, Directory dir,
- FieldInfos fieldInfo, int codecId, boolean useCompoundFile, Comparator<BytesRef> sortComparator, IOContext context) throws IOException {
- this.useCompoundFile = useCompoundFile;
- this.sortComparator = sortComparator;
- final Directory directory;
- if (useCompoundFile) {
- cfs = directory = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(si.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION), context, false);
- } else {
- cfs = null;
- directory = dir;
- }
- docValues = load(fieldInfo, si.name, si.docCount, directory, codecId, context);
- }
-
- /**
- * Returns a {@link IndexDocValues} instance for the given field name or
- * <code>null</code> if this field has no {@link IndexDocValues}.
- */
- @Override
- public IndexDocValues docValues(String field) throws IOException {
- return docValues.get(field);
- }
-
- // Only opens files... doesn't actually load any values
- protected TreeMap<String, IndexDocValues> load(FieldInfos fieldInfos,
- String segment, int docCount, Directory dir, int codecId, IOContext context)
- throws IOException {
- TreeMap<String, IndexDocValues> values = new TreeMap<String, IndexDocValues>();
- boolean success = false;
- try {
-
- for (FieldInfo fieldInfo : fieldInfos) {
- if (codecId == fieldInfo.getCodecId() && fieldInfo.hasDocValues()) {
- final String field = fieldInfo.name;
- // TODO can we have a compound file per segment and codec for
- // docvalues?
- final String id = DefaultDocValuesConsumer.docValuesId(segment,
- codecId, fieldInfo.number);
- values.put(field,
- loadDocValues(docCount, dir, id, fieldInfo.getDocValues(), sortComparator, context));
- }
- }
- success = true;
- } finally {
- if (!success) {
- // if we fail we must close all opened resources if there are any
- closeInternal(values.values());
- }
- }
- return values;
+ public DefaultDocValuesProducer(SegmentReadState state) throws IOException {
+ cfs = new CompoundFileDirectory(state.dir,
+ IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, IndexFileNames.COMPOUND_FILE_EXTENSION),
+ state.context, false);
+ docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, cfs, state.codecId, state.context);
}
-
- /**
- * Loads a {@link IndexDocValues} instance depending on the given {@link ValueType}.
- * Codecs that use different implementations for a certain {@link ValueType} can
- * simply override this method and return their custom implementations.
- *
- * @param docCount
- * number of documents in the segment
- * @param dir
- * the {@link Directory} to load the {@link IndexDocValues} from
- * @param id
- * the unique file ID within the segment
- * @param type
- * the type to load
- * @param sortComparator byte comparator used by sorted variants
- * @return a {@link IndexDocValues} instance for the given type
- * @throws IOException
- * if an {@link IOException} occurs
- * @throws IllegalArgumentException
- * if the given {@link ValueType} is not supported
- */
- protected IndexDocValues loadDocValues(int docCount, Directory dir, String id,
- ValueType type, Comparator<BytesRef> sortComparator, IOContext context) throws IOException {
- switch (type) {
- case FIXED_INTS_16:
- case FIXED_INTS_32:
- case FIXED_INTS_64:
- case FIXED_INTS_8:
- case VAR_INTS:
- return Ints.getValues(dir, id, docCount, type, context);
- case FLOAT_32:
- return Floats.getValues(dir, id, docCount, context);
- case FLOAT_64:
- return Floats.getValues(dir, id, docCount, context);
- case BYTES_FIXED_STRAIGHT:
- return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount, sortComparator, context);
- case BYTES_FIXED_DEREF:
- return Bytes.getValues(dir, id, Bytes.Mode.DEREF, true, docCount, sortComparator, context);
- case BYTES_FIXED_SORTED:
- return Bytes.getValues(dir, id, Bytes.Mode.SORTED, true, docCount, sortComparator, context);
- case BYTES_VAR_STRAIGHT:
- return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, false, docCount, sortComparator, context);
- case BYTES_VAR_DEREF:
- return Bytes.getValues(dir, id, Bytes.Mode.DEREF, false, docCount, sortComparator, context);
- case BYTES_VAR_SORTED:
- return Bytes.getValues(dir, id, Bytes.Mode.SORTED, false, docCount, sortComparator, context);
- default:
- throw new IllegalStateException("unrecognized index values mode " + type);
- }
- }
-
- public void close() throws IOException {
- closeInternal(docValues.values());
- }
-
- private void closeInternal(Collection<? extends Closeable> closeables) throws IOException {
- final Collection<? extends Closeable> toClose;
- if (useCompoundFile) {
- final ArrayList<Closeable> list = new ArrayList<Closeable>(closeables);
- list.add(cfs);
- toClose = list;
- } else {
- toClose = closeables;
- }
- IOUtils.close(toClose);
+ @Override
+ protected Map<String,IndexDocValues> docValues() {
+ return docValues;
}
@Override
- public Collection<String> fields() {
- return docValues.keySet();
+ protected void closeInternal(Collection<? extends Closeable> closeables) throws IOException {
+ final ArrayList<Closeable> list = new ArrayList<Closeable>(closeables);
+ list.add(cfs);
+ IOUtils.close(list);
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesConsumer.java Mon Oct 10 19:58:24 2011
@@ -92,48 +92,38 @@ public abstract class DocValuesConsumer
*
* @param mergeState
* the state to merge
- * @param values
- * the docValues to merge in
+ * @param docValues docValues array containing one instance per reader (
+ * {@link MergeState#readers}) or <code>null</code> if the reader has
+ * no {@link IndexDocValues} instance.
* @throws IOException
* if an {@link IOException} occurs
*/
- public void merge(org.apache.lucene.index.codecs.MergeState mergeState,
- IndexDocValues values) throws IOException {
+ public void merge(MergeState mergeState, IndexDocValues[] docValues) throws IOException {
assert mergeState != null;
- // TODO we need some kind of compatibility notation for values such
- // that two slightly different segments can be merged eg. fixed vs.
- // variable byte len or float32 vs. float64
- boolean merged = false;
- /*
- * We ignore the given DocValues here and merge from the subReaders directly
- * to support bulk copies on the DocValues Writer level. if this gets merged
- * with MultiDocValues the writer can not optimize for bulk-copyable data
- */
+ boolean hasMerged = false;
for(int readerIDX=0;readerIDX<mergeState.readers.size();readerIDX++) {
final org.apache.lucene.index.codecs.MergeState.IndexReaderAndLiveDocs reader = mergeState.readers.get(readerIDX);
- final IndexDocValues r = reader.reader.docValues(mergeState.fieldInfo.name);
- if (r != null) {
- merged = true;
- merge(new Writer.MergeState(r, mergeState.docBase[readerIDX], reader.reader.maxDoc(),
+ if (docValues[readerIDX] != null) {
+ hasMerged = true;
+ merge(new Writer.SingleSubMergeState(docValues[readerIDX], mergeState.docBase[readerIDX], reader.reader.maxDoc(),
reader.liveDocs));
}
}
- if (merged) {
+ // only finish if no exception is thrown!
+ if (hasMerged) {
finish(mergeState.mergedDocCount);
}
}
/**
- * Merges the given {@link MergeState} into this {@link DocValuesConsumer}.
- * {@link MergeState#docBase} must always be increasing. Merging segments out
- * of order is not supported.
+ * Merges the given {@link SingleSubMergeState} into this {@link DocValuesConsumer}.
*
* @param mergeState
- * the {@link MergeState} to merge
+ * the {@link SingleSubMergeState} to merge
* @throws IOException
* if an {@link IOException} occurs
*/
- protected abstract void merge(MergeState mergeState) throws IOException;
+ protected abstract void merge(SingleSubMergeState mergeState) throws IOException;
/**
* Specialized auxiliary MergeState is necessary since we don't want to
@@ -141,7 +131,7 @@ public abstract class DocValuesConsumer
* created for each merged low level {@link IndexReader} we are merging to
* support low level bulk copies.
*/
- public static class MergeState {
+ public static class SingleSubMergeState {
/**
* the source reader for this MergeState - merged values should be read from
* this instance
@@ -154,7 +144,7 @@ public abstract class DocValuesConsumer
/** the not deleted bits for this MergeState */
public final Bits liveDocs;
- public MergeState(IndexDocValues reader, int docBase, int docCount, Bits liveDocs) {
+ public SingleSubMergeState(IndexDocValues reader, int docBase, int docCount, Bits liveDocs) {
assert reader != null;
this.reader = reader;
this.docBase = docBase;
Copied: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java (from r1178296, lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java?p2=lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java&r1=1178296&r2=1181188&rev=1181188&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java Mon Oct 10 19:58:24 2011
@@ -58,11 +58,11 @@ public abstract class DocValuesReaderBas
public Collection<String> fields() {
return docValues().keySet();
}
-
+
public Comparator<BytesRef> getComparator() throws IOException {
return BytesRef.getUTF8SortedAsUnicodeComparator();
}
-
+
// Only opens files... doesn't actually load any values
protected TreeMap<String, IndexDocValues> load(FieldInfos fieldInfos,
String segment, int docCount, Directory dir, int codecId, IOContext context)
@@ -121,9 +121,9 @@ public abstract class DocValuesReaderBas
case VAR_INTS:
return Ints.getValues(dir, id, docCount, type, context);
case FLOAT_32:
- return Floats.getValues(dir, id, docCount, context);
+ return Floats.getValues(dir, id, docCount, context, type);
case FLOAT_64:
- return Floats.getValues(dir, id, docCount, context);
+ return Floats.getValues(dir, id, docCount, context, type);
case BYTES_FIXED_STRAIGHT:
return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount, getComparator(), context);
case BYTES_FIXED_DEREF:
Copied: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java (from r1178296, lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java?p2=lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java&r1=1178296&r2=1181188&rev=1181188&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java Mon Oct 10 19:58:24 2011
@@ -54,7 +54,7 @@ public abstract class DocValuesWriterBas
@Override
public DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
return Writer.create(field.getDocValues(),
- docValuesId(segmentName, codecId, field.number),
+ docValuesId(segmentName, codecId, field.number),
getDirectory(), getComparator(), bytesUsed, context);
}
@@ -62,6 +62,7 @@ public abstract class DocValuesWriterBas
return segmentsName + "_" + codecID + "-" + fieldId;
}
+
public Comparator<BytesRef> getComparator() throws IOException {
return BytesRef.getUTF8SortedAsUnicodeComparator();
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java Mon Oct 10 19:58:24 2011
@@ -19,7 +19,10 @@ import java.io.Closeable;
import java.io.IOException;
import org.apache.lucene.index.FieldInfo;
+import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.values.IndexDocValues;
+import org.apache.lucene.index.values.TypePromoter;
+import org.apache.lucene.index.values.ValueType;
/**
* Abstract API that consumes per document values. Concrete implementations of
@@ -40,28 +43,75 @@ public abstract class PerDocConsumer imp
* Consumes and merges the given {@link PerDocValues} producer
* into this consumers format.
*/
- public void merge(MergeState mergeState, PerDocValues producer)
+ public void merge(MergeState mergeState)
throws IOException {
- Iterable<String> fields = producer.fields();
- for (String field : fields) {
- mergeState.fieldInfo = mergeState.fieldInfos.fieldInfo(field);
- assert mergeState.fieldInfo != null : "FieldInfo for field is null: "
- + field;
- if (mergeState.fieldInfo.hasDocValues()) {
- final IndexDocValues docValues = producer.docValues(field);
- if (docValues == null) {
- /*
- * It is actually possible that a fieldInfo has a values type but no
- * values are actually available. this can happen if there are already
- * segments without values around.
- */
+ final FieldInfos fieldInfos = mergeState.fieldInfos;
+ final IndexDocValues[] docValues = new IndexDocValues[mergeState.readers.size()];
+ final PerDocValues[] perDocValues = new PerDocValues[mergeState.readers.size()];
+ // pull all PerDocValues
+ for (int i = 0; i < perDocValues.length; i++) {
+ perDocValues[i] = mergeState.readers.get(i).reader.perDocValues();
+ }
+ for (FieldInfo fieldInfo : fieldInfos) {
+ mergeState.fieldInfo = fieldInfo;
+ TypePromoter currentPromoter = TypePromoter.getIdentityPromoter();
+ if (fieldInfo.hasDocValues()) {
+ for (int i = 0; i < perDocValues.length; i++) {
+ if (perDocValues[i] != null) { // get all IDV to merge
+ docValues[i] = perDocValues[i].docValues(fieldInfo.name);
+ if (docValues[i] != null) {
+ currentPromoter = promoteValueType(fieldInfo, docValues[i], currentPromoter);
+ if (currentPromoter == null) {
+ break;
+ }
+ }
+ }
+ }
+
+ if (currentPromoter == null) {
+ fieldInfo.resetDocValues(null);
continue;
}
+ assert currentPromoter != TypePromoter.getIdentityPromoter();
+ if (fieldInfo.getDocValues() != currentPromoter.type()) {
+ // reset the type if we got promoted
+ fieldInfo.resetDocValues(currentPromoter.type());
+ }
+
final DocValuesConsumer docValuesConsumer = addValuesField(mergeState.fieldInfo);
assert docValuesConsumer != null;
docValuesConsumer.merge(mergeState, docValues);
}
}
+ /* NOTE: don't close the perDocProducers here since they are private segment producers
+ * and will be closed once the SegmentReader goes out of scope */
+ }
+ protected TypePromoter promoteValueType(final FieldInfo fieldInfo, final IndexDocValues docValues,
+ TypePromoter currentPromoter) {
+ assert currentPromoter != null;
+ final TypePromoter incomingPromoter = TypePromoter.create(docValues.type(), docValues.getValueSize());
+ assert incomingPromoter != null;
+ final TypePromoter newPromoter = currentPromoter.promote(incomingPromoter);
+ return newPromoter == null ? handleIncompatibleValueType(fieldInfo, incomingPromoter, currentPromoter) : newPromoter;
+ }
+
+ /**
+ * Resolves a conflicts of incompatible {@link TypePromoter}s. The default
+ * implementation promotes incompatible types to
+ * {@link ValueType#BYTES_VAR_STRAIGHT} and preserves all values. If this
+ * method returns <code>null</code> all docvalues for the given
+ * {@link FieldInfo} are dropped and all values are lost.
+ *
+ * @param incomingPromoter
+ * the incompatible incoming promoter
+ * @param currentPromoter
+ * the current promoter
+ * @return a promoted {@link TypePromoter} or <code>null</code> iff this index
+ * docvalues should be dropped for this field.
+ */
+ protected TypePromoter handleIncompatibleValueType(FieldInfo fieldInfo, TypePromoter incomingPromoter, TypePromoter currentPromoter) {
+ return TypePromoter.create(ValueType.BYTES_VAR_STRAIGHT, TypePromoter.VAR_TYPE_VALUE_SIZE);
}
+
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java Mon Oct 10 19:58:24 2011
@@ -51,5 +51,4 @@ public abstract class PerDocValues imple
* Returns all fields this {@link PerDocValues} contains values for.
*/
public abstract Collection<String> fields();
-
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java Mon Oct 10 19:58:24 2011
@@ -787,22 +787,22 @@ public class MemoryCodec extends Codec {
@Override
public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, EXTENSION));
- DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS());
+ DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
}
@Override
public void getExtensions(Set<String> extensions) {
extensions.add(EXTENSION);
- DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
+ DefaultDocValuesConsumer.getExtensions(extensions);
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+ return new DefaultDocValuesConsumer(state);
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), IOContext.READONCE);
+ return new DefaultDocValuesProducer(state);
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java Mon Oct 10 19:58:24 2011
@@ -134,22 +134,22 @@ public class PulsingCodec extends Codec
public void files(Directory dir, SegmentInfo segmentInfo, int codecID, Set<String> files) throws IOException {
StandardPostingsReader.files(dir, segmentInfo, codecID, files);
BlockTreeTermsReader.files(dir, segmentInfo, codecID, files);
- DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files, getDocValuesUseCFS());
+ DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files);
}
@Override
public void getExtensions(Set<String> extensions) {
StandardCodec.getStandardExtensions(extensions);
- DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
+ DefaultDocValuesConsumer.getExtensions(extensions);
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+ return new DefaultDocValuesConsumer(state);
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+ return new DefaultDocValuesProducer(state);
}
}
Copied: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java (from r1178296, lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java?p2=lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java&r1=1178296&r2=1181188&rev=1181188&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java Mon Oct 10 19:58:24 2011
@@ -56,9 +56,9 @@ public class SepDocValuesConsumer extend
switch (fieldInfo.getDocValues()) {
case BYTES_FIXED_DEREF:
case BYTES_VAR_DEREF:
- case BYTES_VAR_SORTED:
- case BYTES_FIXED_SORTED:
case BYTES_VAR_STRAIGHT:
+ case BYTES_FIXED_SORTED:
+ case BYTES_VAR_SORTED:
files.add(IndexFileNames.segmentFileName(filename, "",
Writer.INDEX_EXTENSION));
assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
@@ -77,7 +77,6 @@ public class SepDocValuesConsumer extend
assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
Writer.DATA_EXTENSION));
break;
-
default:
assert false;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java Mon Oct 10 19:58:24 2011
@@ -69,23 +69,23 @@ public class SimpleTextCodec extends Cod
@Override
public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
files.add(getPostingsFileName(segmentInfo.name, id));
- DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS());
+ DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
}
@Override
public void getExtensions(Set<String> extensions) {
extensions.add(POSTINGS_EXTENSION);
- DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
+ DefaultDocValuesConsumer.getExtensions(extensions);
}
// TODO: would be great if these used a plain text impl
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+ return new DefaultDocValuesConsumer(state);
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+ return new DefaultDocValuesProducer(state);
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java Mon Oct 10 19:58:24 2011
@@ -110,19 +110,19 @@ public class StandardCodec extends Codec
public void files(Directory dir, SegmentInfo segmentInfo, int codecID, Set<String> files) throws IOException {
StandardPostingsReader.files(dir, segmentInfo, codecID, files);
BlockTreeTermsReader.files(dir, segmentInfo, codecID, files);
- DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files, getDocValuesUseCFS());
+ DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files);
}
@Override
public void getExtensions(Set<String> extensions) {
getStandardExtensions(extensions);
- DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
}
public static void getStandardExtensions(Set<String> extensions) {
extensions.add(FREQ_EXTENSION);
extensions.add(PROX_EXTENSION);
BlockTreeTermsReader.getExtensions(extensions);
+ DefaultDocValuesConsumer.getExtensions(extensions);
}
@Override
@@ -132,11 +132,11 @@ public class StandardCodec extends Codec
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+ return new DefaultDocValuesConsumer(state);
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+ return new DefaultDocValuesProducer(state);
}
}