You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2011/12/15 06:11:30 UTC
svn commit: r1214623 [4/17] - in /lucene/dev/branches/solrcloud: ./
dev-tools/idea/lucene/contrib/ lucene/ lucene/contrib/
lucene/contrib/demo/src/java/org/apache/lucene/demo/
lucene/contrib/highlighter/src/java/org/apache/lucene/search/vectorhighlight...
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexReader.java Thu Dec 15 05:11:14 2011
@@ -24,19 +24,19 @@ import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.DocumentStoredFieldVisitor;
-import org.apache.lucene.index.codecs.PerDocValues;
-import org.apache.lucene.index.values.IndexDocValues;
-import org.apache.lucene.search.FieldCache; // javadocs
import org.apache.lucene.search.SearcherManager; // javadocs
-import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.*;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.CommandLineUtil;
+import org.apache.lucene.util.MapBackedSet;
import org.apache.lucene.util.ReaderUtil; // for javadocs
/** IndexReader is an abstract class, providing an interface for accessing an
@@ -45,7 +45,7 @@ import org.apache.lucene.util.ReaderUtil
<p> Concrete subclasses of IndexReader are usually constructed with a call to
one of the static <code>open()</code> methods, e.g. {@link
- #open(Directory, boolean)}.
+ #open(Directory)}.
<p> For efficiency, in this API documents are often referred to via
<i>document numbers</i>, non-negative integers which each name a unique
@@ -53,9 +53,6 @@ import org.apache.lucene.util.ReaderUtil
as documents are added to and deleted from an index. Clients should thus not
rely on a given document having the same number between sessions.
- <p> An IndexReader can be opened on a directory for which an IndexWriter is
- opened already, but it cannot be used to delete documents from the index then.
-
<p>
<b>NOTE</b>: for backwards API compatibility, several methods are not listed
as abstract, but have no useful implementations in this base class and
@@ -66,13 +63,6 @@ import org.apache.lucene.util.ReaderUtil
<p>
- <b>NOTE</b>: as of 2.4, it's possible to open a read-only
- IndexReader using the static open methods that accept the
- boolean readOnly parameter. Such a reader may have better
- concurrency. You must specify false if you want to
- make changes with the resulting IndexReader.
- </p>
-
<a name="thread-safety"></a><p><b>NOTE</b>: {@link
IndexReader} instances are completely thread
safe, meaning multiple threads can call any of its methods,
@@ -85,62 +75,40 @@ public abstract class IndexReader implem
/**
* A custom listener that's invoked when the IndexReader
- * is finished.
- *
- * <p>For a SegmentReader, this listener is called only
- * once all SegmentReaders sharing the same core are
- * closed. At this point it is safe for apps to evict
- * this reader from any caches keyed on {@link
- * #getCoreCacheKey}. This is the same interface that
- * {@link FieldCache} uses, internally, to evict
- * entries.</p>
- *
- * <p>For other readers, this listener is called when they
- * are closed.</p>
+ * is closed.
*
* @lucene.experimental
*/
- public static interface ReaderFinishedListener {
- public void finished(IndexReader reader);
+ public static interface ReaderClosedListener {
+ public void onClose(IndexReader reader);
}
- // Impls must set this if they may call add/removeReaderFinishedListener:
- protected volatile Collection<ReaderFinishedListener> readerFinishedListeners;
+ private final Set<ReaderClosedListener> readerClosedListeners =
+ new MapBackedSet<ReaderClosedListener>(new ConcurrentHashMap<ReaderClosedListener, Boolean>());
- /** Expert: adds a {@link ReaderFinishedListener}. The
- * provided listener is also added to any sub-readers, if
- * this is a composite reader. Also, any reader reopened
- * or cloned from this one will also copy the listeners at
- * the time of reopen.
+ /** Expert: adds a {@link ReaderClosedListener}. The
+ * provided listener will be invoked when this reader is closed.
*
* @lucene.experimental */
- public void addReaderFinishedListener(ReaderFinishedListener listener) {
+ public final void addReaderClosedListener(ReaderClosedListener listener) {
ensureOpen();
- readerFinishedListeners.add(listener);
+ readerClosedListeners.add(listener);
}
- /** Expert: remove a previously added {@link ReaderFinishedListener}.
+ /** Expert: remove a previously added {@link ReaderClosedListener}.
*
* @lucene.experimental */
- public void removeReaderFinishedListener(ReaderFinishedListener listener) {
+ public final void removeReaderClosedListener(ReaderClosedListener listener) {
ensureOpen();
- readerFinishedListeners.remove(listener);
+ readerClosedListeners.remove(listener);
}
- protected void notifyReaderFinishedListeners() {
- // Defensive (should never be null -- all impls must set
- // this):
- if (readerFinishedListeners != null) {
- for(ReaderFinishedListener listener : readerFinishedListeners) {
- listener.finished(this);
- }
+ private final void notifyReaderClosedListeners() {
+ for(ReaderClosedListener listener : readerClosedListeners) {
+ listener.onClose(this);
}
}
- protected void readerFinished() {
- notifyReaderFinishedListeners();
- }
-
/**
* Constants describing field properties, for example used for
* {@link IndexReader#getFieldNames(FieldOption)}.
@@ -175,14 +143,13 @@ public abstract class IndexReader implem
}
private volatile boolean closed;
- protected boolean hasChanges;
private final AtomicInteger refCount = new AtomicInteger();
static int DEFAULT_TERMS_INDEX_DIVISOR = 1;
/** Expert: returns the current refCount for this reader */
- public int getRefCount() {
+ public final int getRefCount() {
return refCount.get();
}
@@ -201,7 +168,7 @@ public abstract class IndexReader implem
* @see #decRef
* @see #tryIncRef
*/
- public void incRef() {
+ public final void incRef() {
ensureOpen();
refCount.incrementAndGet();
}
@@ -229,7 +196,7 @@ public abstract class IndexReader implem
* @see #decRef
* @see #incRef
*/
- public boolean tryIncRef() {
+ public final boolean tryIncRef() {
int count;
while ((count = refCount.get()) > 0) {
if (refCount.compareAndSet(count, count+1)) {
@@ -243,9 +210,6 @@ public abstract class IndexReader implem
@Override
public String toString() {
final StringBuilder buffer = new StringBuilder();
- if (hasChanges) {
- buffer.append('*');
- }
buffer.append(getClass().getSimpleName());
buffer.append('(');
final IndexReader[] subReaders = getSequentialSubReaders();
@@ -261,22 +225,20 @@ public abstract class IndexReader implem
/**
* Expert: decreases the refCount of this IndexReader
- * instance. If the refCount drops to 0, then pending
- * changes (if any) are committed to the index and this
+ * instance. If the refCount drops to 0, then this
* reader is closed. If an exception is hit, the refCount
* is unchanged.
*
- * @throws IOException in case an IOException occurs in commit() or doClose()
+ * @throws IOException in case an IOException occurs in doClose()
*
* @see #incRef
*/
- public void decRef() throws IOException {
+ public final void decRef() throws IOException {
ensureOpen();
- final int rc = refCount.getAndDecrement();
- if (rc == 1) {
+ final int rc = refCount.decrementAndGet();
+ if (rc == 0) {
boolean success = false;
try {
- commit();
doClose();
success = true;
} finally {
@@ -285,9 +247,9 @@ public abstract class IndexReader implem
refCount.incrementAndGet();
}
}
- readerFinished();
- } else if (rc <= 0) {
- throw new IllegalStateException("too many decRef calls: refCount was " + rc + " before decrement");
+ notifyReaderClosedListeners();
+ } else if (rc < 0) {
+ throw new IllegalStateException("too many decRef calls: refCount is " + rc + " after decrement");
}
}
@@ -305,27 +267,33 @@ public abstract class IndexReader implem
}
/** Returns a IndexReader reading the index in the given
- * Directory, with readOnly=true.
+ * Directory
* @param directory the index directory
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
public static IndexReader open(final Directory directory) throws CorruptIndexException, IOException {
- return open(directory, null, null, true, DEFAULT_TERMS_INDEX_DIVISOR);
+ return DirectoryReader.open(directory, null, DEFAULT_TERMS_INDEX_DIVISOR);
}
-
- /** Returns an IndexReader reading the index in the given
- * Directory. You should pass readOnly=true, since it
- * gives much better concurrent performance, unless you
- * intend to do write operations (delete documents or
- * change norms) with the reader.
+
+ /** Expert: Returns a IndexReader reading the index in the given
+ * Directory with the given termInfosIndexDivisor.
* @param directory the index directory
- * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader
+ * @param termInfosIndexDivisor Subsamples which indexed
+ * terms are loaded into RAM. This has the same effect as {@link
+ * IndexWriterConfig#setTermIndexInterval} except that setting
+ * must be done at indexing time while this setting can be
+ * set per reader. When set to N, then one in every
+ * N*termIndexInterval terms in the index is loaded into
+ * memory. By setting this to a value > 1 you can reduce
+ * memory usage, at the expense of higher latency when
+ * loading a TermInfo. The default value is 1. Set this
+ * to -1 to skip loading the terms index entirely.
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public static IndexReader open(final Directory directory, boolean readOnly) throws CorruptIndexException, IOException {
- return open(directory, null, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR);
+ public static IndexReader open(final Directory directory, int termInfosIndexDivisor) throws CorruptIndexException, IOException {
+ return DirectoryReader.open(directory, null, termInfosIndexDivisor);
}
/**
@@ -352,48 +320,19 @@ public abstract class IndexReader implem
}
/** Expert: returns an IndexReader reading the index in the given
- * {@link IndexCommit}. You should pass readOnly=true, since it
- * gives much better concurrent performance, unless you
- * intend to do write operations (delete documents or
- * change norms) with the reader.
+ * {@link IndexCommit}.
* @param commit the commit point to open
- * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public static IndexReader open(final IndexCommit commit, boolean readOnly) throws CorruptIndexException, IOException {
- return open(commit.getDirectory(), null, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR);
+ public static IndexReader open(final IndexCommit commit) throws CorruptIndexException, IOException {
+ return DirectoryReader.open(commit.getDirectory(), commit, DEFAULT_TERMS_INDEX_DIVISOR);
}
- /** Expert: returns an IndexReader reading the index in
- * the given Directory, with a custom {@link
- * IndexDeletionPolicy}. You should pass readOnly=true,
- * since it gives much better concurrent performance,
- * unless you intend to do write operations (delete
- * documents or change norms) with the reader.
- * @param directory the index directory
- * @param deletionPolicy a custom deletion policy (only used
- * if you use this reader to perform deletes or to set
- * norms); see {@link IndexWriter} for details.
- * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader
- * @throws CorruptIndexException if the index is corrupt
- * @throws IOException if there is a low-level IO error
- */
- public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly) throws CorruptIndexException, IOException {
- return open(directory, deletionPolicy, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR);
- }
- /** Expert: returns an IndexReader reading the index in
- * the given Directory, with a custom {@link
- * IndexDeletionPolicy}. You should pass readOnly=true,
- * since it gives much better concurrent performance,
- * unless you intend to do write operations (delete
- * documents or change norms) with the reader.
- * @param directory the index directory
- * @param deletionPolicy a custom deletion policy (only used
- * if you use this reader to perform deletes or to set
- * norms); see {@link IndexWriter} for details.
- * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader
+ /** Expert: returns an IndexReader reading the index in the given
+ * {@link IndexCommit} and termInfosIndexDivisor.
+ * @param commit the commit point to open
* @param termInfosIndexDivisor Subsamples which indexed
* terms are loaded into RAM. This has the same effect as {@link
* IndexWriterConfig#setTermIndexInterval} except that setting
@@ -407,65 +346,8 @@ public abstract class IndexReader implem
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException {
- return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor);
- }
-
- /** Expert: returns an IndexReader reading the index in
- * the given Directory, using a specific commit and with
- * a custom {@link IndexDeletionPolicy}. You should pass
- * readOnly=true, since it gives much better concurrent
- * performance, unless you intend to do write operations
- * (delete documents or change norms) with the reader.
- * @param commit the specific {@link IndexCommit} to open;
- * see {@link IndexReader#listCommits} to list all commits
- * in a directory
- * @param deletionPolicy a custom deletion policy (only used
- * if you use this reader to perform deletes or to set
- * norms); see {@link IndexWriter} for details.
- * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader
- * @throws CorruptIndexException if the index is corrupt
- * @throws IOException if there is a low-level IO error
- */
- public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly) throws CorruptIndexException, IOException {
- return open(commit.getDirectory(), deletionPolicy, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR);
- }
-
- /** Expert: returns an IndexReader reading the index in
- * the given Directory, using a specific commit and with
- * a custom {@link IndexDeletionPolicy}. You should pass
- * readOnly=true, since it gives much better concurrent
- * performance, unless you intend to do write operations
- * (delete documents or change norms) with the reader.
- * @param commit the specific {@link IndexCommit} to open;
- * see {@link IndexReader#listCommits} to list all commits
- * in a directory
- * @param deletionPolicy a custom deletion policy (only used
- * if you use this reader to perform deletes or to set
- * norms); see {@link IndexWriter} for details.
- * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader
- * @param termInfosIndexDivisor Subsamples which indexed
- * terms are loaded into RAM. This has the same effect as {@link
- * IndexWriterConfig#setTermIndexInterval} except that setting
- * must be done at indexing time while this setting can be
- * set per reader. When set to N, then one in every
- * N*termIndexInterval terms in the index is loaded into
- * memory. By setting this to a value > 1 you can reduce
- * memory usage, at the expense of higher latency when
- * loading a TermInfo. The default value is 1. Set this
- * to -1 to skip loading the terms index entirely. This is only useful in
- * advanced situations when you will only .next() through all terms;
- * attempts to seek will hit an exception.
- *
- * @throws CorruptIndexException if the index is corrupt
- * @throws IOException if there is a low-level IO error
- */
- public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException {
- return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor);
- }
-
- private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException {
- return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor);
+ public static IndexReader open(final IndexCommit commit, int termInfosIndexDivisor) throws CorruptIndexException, IOException {
+ return DirectoryReader.open(commit.getDirectory(), commit, termInfosIndexDivisor);
}
/**
@@ -487,11 +369,6 @@ public abstract class IndexReader implem
* reader while other threads are still using it; see
* {@link SearcherManager} to simplify managing this.
*
- * <p>If a new reader is returned, it's safe to make changes
- * (deletions, norms) with it. All shared mutable state
- * with the old reader uses "copy on write" semantics to
- * ensure the changes are not seen by other readers.
- *
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
* @return null if there are no changes; else, a new
@@ -504,28 +381,12 @@ public abstract class IndexReader implem
}
/**
- * If the index has changed since the provided reader was
- * opened, open and return a new reader, with the
- * specified <code>readOnly</code>; else, return
- * null.
- *
- * @see #openIfChanged(IndexReader)
- */
- public static IndexReader openIfChanged(IndexReader oldReader, boolean readOnly) throws IOException {
- final IndexReader newReader = oldReader.doOpenIfChanged(readOnly);
- assert newReader != oldReader;
- return newReader;
- }
-
- /**
* If the IndexCommit differs from what the
- * provided reader is searching, or the provided reader is
- * not already read-only, open and return a new
- * <code>readOnly=true</code> reader; else, return null.
+ * provided reader is searching, open and return a new
+ * reader; else, return null.
*
* @see #openIfChanged(IndexReader)
*/
- // TODO: should you be able to specify readOnly?
public static IndexReader openIfChanged(IndexReader oldReader, IndexCommit commit) throws IOException {
final IndexReader newReader = oldReader.doOpenIfChanged(commit);
assert newReader != oldReader;
@@ -535,7 +396,7 @@ public abstract class IndexReader implem
/**
* Expert: If there changes (committed or not) in the
* {@link IndexWriter} versus what the provided reader is
- * searching, then open and return a new read-only
+ * searching, then open and return a new
* IndexReader searching both committed and uncommitted
* changes from the writer; else, return null (though, the
* current implementation never returns null).
@@ -613,16 +474,6 @@ public abstract class IndexReader implem
* If the index has changed since it was opened, open and return a new reader;
* else, return {@code null}.
*
- * @see #openIfChanged(IndexReader, boolean)
- */
- protected IndexReader doOpenIfChanged(boolean openReadOnly) throws CorruptIndexException, IOException {
- throw new UnsupportedOperationException("This reader does not support reopen().");
- }
-
- /**
- * If the index has changed since it was opened, open and return a new reader;
- * else, return {@code null}.
- *
* @see #openIfChanged(IndexReader, IndexCommit)
*/
protected IndexReader doOpenIfChanged(final IndexCommit commit) throws CorruptIndexException, IOException {
@@ -642,34 +493,11 @@ public abstract class IndexReader implem
/**
* Efficiently clones the IndexReader (sharing most
* internal state).
- * <p>
- * On cloning a reader with pending changes (deletions,
- * norms), the original reader transfers its write lock to
- * the cloned reader. This means only the cloned reader
- * may make further changes to the index, and commit the
- * changes to the index on close, but the old reader still
- * reflects all changes made up until it was cloned.
- * <p>
- * Like {@link #openIfChanged(IndexReader)}, it's safe to make changes to
- * either the original or the cloned reader: all shared
- * mutable state obeys "copy on write" semantics to ensure
- * the changes are not seen by other readers.
- * <p>
*/
@Override
public synchronized Object clone() {
throw new UnsupportedOperationException("This reader does not implement clone()");
}
-
- /**
- * Clones the IndexReader and optionally changes readOnly. A readOnly
- * reader cannot open a writeable reader.
- * @throws CorruptIndexException if the index is corrupt
- * @throws IOException if there is a low-level IO error
- */
- public synchronized IndexReader clone(boolean openReadOnly) throws CorruptIndexException, IOException {
- throw new UnsupportedOperationException("This reader does not implement clone()");
- }
/**
* Returns the directory associated with this index. The Default
@@ -807,14 +635,14 @@ public abstract class IndexReader implem
* term vectors were not indexed. The returned Fields
* instance acts like a single-document inverted index
* (the docID will be 0). */
- abstract public Fields getTermVectors(int docID)
+ public abstract Fields getTermVectors(int docID)
throws IOException;
/** Retrieve term vector for this document and field, or
* null if term vectors were not indexed. The returned
* Fields instance acts like a single-document inverted
* index (the docID will be 0). */
- public Terms getTermVector(int docID, String field)
+ public final Terms getTermVector(int docID, String field)
throws IOException {
Fields vectors = getTermVectors(docID);
if (vectors == null) {
@@ -848,7 +676,7 @@ public abstract class IndexReader implem
public abstract int maxDoc();
/** Returns the number of deleted documents. */
- public int numDeletedDocs() {
+ public final int numDeletedDocs() {
return maxDoc() - numDocs();
}
@@ -881,16 +709,24 @@ public abstract class IndexReader implem
// TODO: we need a separate StoredField, so that the
// Document returned here contains that class not
// IndexableField
- public Document document(int docID) throws CorruptIndexException, IOException {
+ public final Document document(int docID) throws CorruptIndexException, IOException {
ensureOpen();
- if (docID < 0 || docID >= maxDoc()) {
- throw new IllegalArgumentException("docID must be >= 0 and < maxDoc=" + maxDoc() + " (got docID=" + docID + ")");
- }
final DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
document(docID, visitor);
return visitor.getDocument();
}
+ /**
+ * Like {@link #document(int)} but only loads the specified
+ * fields. Note that this is simply sugar for {@link
+ * DocumentStoredFieldVisitor#DocumentStoredFieldVisitor(Set)}.
+ */
+ public final Document document(int docID, Set<String> fieldsToLoad) throws CorruptIndexException, IOException {
+ final DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor(fieldsToLoad);
+ document(docID, visitor);
+ return visitor.getDocument();
+ }
+
/** Returns true if any documents have been deleted */
public abstract boolean hasDeletions();
@@ -910,39 +746,6 @@ public abstract class IndexReader implem
*/
public abstract byte[] norms(String field) throws IOException;
- /** Expert: Resets the normalization factor for the named field of the named
- * document. By default, the norm represents the product of the field's {@link
- * org.apache.lucene.document.Field#setBoost(float) boost} and its
- * length normalization}. Thus, to preserve the length normalization
- * values when resetting this, one should base the new value upon the old.
- *
- * <b>NOTE:</b> If this field does not index norms, then
- * this method throws {@link IllegalStateException}.
- *
- * @see #norms(String)
- * @see Similarity#computeNorm(FieldInvertState)
- * @see org.apache.lucene.search.similarities.DefaultSimilarity#decodeNormValue(byte)
- * @throws StaleReaderException if the index has changed
- * since this reader was opened
- * @throws CorruptIndexException if the index is corrupt
- * @throws LockObtainFailedException if another writer
- * has this index open (<code>write.lock</code> could not
- * be obtained)
- * @throws IOException if there is a low-level IO error
- * @throws IllegalStateException if the field does not index norms
- */
- public synchronized void setNorm(int doc, String field, byte value)
- throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException {
- ensureOpen();
- acquireWriteLock();
- hasChanges = true;
- doSetNorm(doc, field, value);
- }
-
- /** Implements setNorm in subclass.*/
- protected abstract void doSetNorm(int doc, String field, byte value)
- throws CorruptIndexException, IOException;
-
/**
* Returns {@link Fields} for this reader.
* This method may return null if the reader has no
@@ -958,22 +761,7 @@ public abstract class IndexReader implem
* through them yourself. */
public abstract Fields fields() throws IOException;
- /**
- * Returns {@link PerDocValues} for this reader.
- * This method may return null if the reader has no per-document
- * values stored.
- *
- * <p><b>NOTE</b>: if this is a multi reader ({@link
- * #getSequentialSubReaders} is not null) then this
- * method will throw UnsupportedOperationException. If
- * you really need {@link PerDocValues} for such a reader,
- * use {@link MultiPerDocValues#getPerDocs(IndexReader)}. However, for
- * performance reasons, it's best to get all sub-readers
- * using {@link ReaderUtil#gatherSubReaders} and iterate
- * through them yourself. */
- public abstract PerDocValues perDocValues() throws IOException;
-
- public int docFreq(Term term) throws IOException {
+ public final int docFreq(Term term) throws IOException {
return docFreq(term.field(), term.bytes());
}
@@ -1004,7 +792,7 @@ public abstract class IndexReader implem
* field does not exists. This method does not take into
* account deleted documents that have not yet been merged
* away. */
- public long totalTermFreq(String field, BytesRef term) throws IOException {
+ public final long totalTermFreq(String field, BytesRef term) throws IOException {
final Fields fields = fields();
if (fields == null) {
return 0;
@@ -1022,7 +810,7 @@ public abstract class IndexReader implem
}
/** This may return null if the field does not exist.*/
- public Terms terms(String field) throws IOException {
+ public final Terms terms(String field) throws IOException {
final Fields fields = fields();
if (fields == null) {
return null;
@@ -1033,7 +821,7 @@ public abstract class IndexReader implem
/** Returns {@link DocsEnum} for the specified field &
* term. This may return null, if either the field or
* term does not exist. */
- public DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, boolean needsFreqs) throws IOException {
+ public final DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, boolean needsFreqs) throws IOException {
assert field != null;
assert term != null;
final Fields fields = fields();
@@ -1053,7 +841,7 @@ public abstract class IndexReader implem
* field & term. This may return null, if either the
* field or term does not exist, or, positions were not
* indexed for this field. */
- public DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term) throws IOException {
+ public final DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term) throws IOException {
assert field != null;
assert term != null;
final Fields fields = fields();
@@ -1074,7 +862,7 @@ public abstract class IndexReader implem
* {@link TermState}. This may return null, if either the field or the term
* does not exists or the {@link TermState} is invalid for the underlying
* implementation.*/
- public DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, TermState state, boolean needsFreqs) throws IOException {
+ public final DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, TermState state, boolean needsFreqs) throws IOException {
assert state != null;
assert field != null;
final Fields fields = fields();
@@ -1094,7 +882,7 @@ public abstract class IndexReader implem
* {@link TermState}. This may return null, if either the field or the term
* does not exists, the {@link TermState} is invalid for the underlying
* implementation, or positions were not indexed for this field. */
- public DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term, TermState state) throws IOException {
+ public final DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term, TermState state) throws IOException {
assert state != null;
assert field != null;
final Fields fields = fields();
@@ -1109,159 +897,6 @@ public abstract class IndexReader implem
return null;
}
-
- /** Deletes the document numbered <code>docNum</code>. Once a document is
- * deleted it will not appear in TermDocs or TermPositions enumerations.
- * Attempts to read its field with the {@link #document}
- * method will result in an error. The presence of this document may still be
- * reflected in the {@link #docFreq} statistic, though
- * this will be corrected eventually as the index is further modified.
- *
- * @throws StaleReaderException if the index has changed
- * since this reader was opened
- * @throws CorruptIndexException if the index is corrupt
- * @throws LockObtainFailedException if another writer
- * has this index open (<code>write.lock</code> could not
- * be obtained)
- * @throws IOException if there is a low-level IO error
- */
- public synchronized void deleteDocument(int docNum) throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException {
- ensureOpen();
- acquireWriteLock();
- hasChanges = true;
- doDelete(docNum);
- }
-
-
- /** Implements deletion of the document numbered <code>docNum</code>.
- * Applications should call {@link #deleteDocument(int)} or {@link #deleteDocuments(Term)}.
- */
- protected abstract void doDelete(int docNum) throws CorruptIndexException, IOException;
-
-
- /** Deletes all documents that have a given <code>term</code> indexed.
- * This is useful if one uses a document field to hold a unique ID string for
- * the document. Then to delete such a document, one merely constructs a
- * term with the appropriate field and the unique ID string as its text and
- * passes it to this method.
- * See {@link #deleteDocument(int)} for information about when this deletion will
- * become effective.
- *
- * @return the number of documents deleted
- * @throws StaleReaderException if the index has changed
- * since this reader was opened
- * @throws CorruptIndexException if the index is corrupt
- * @throws LockObtainFailedException if another writer
- * has this index open (<code>write.lock</code> could not
- * be obtained)
- * @throws IOException if there is a low-level IO error
- */
- public int deleteDocuments(Term term) throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException {
- ensureOpen();
- DocsEnum docs = MultiFields.getTermDocsEnum(this,
- MultiFields.getLiveDocs(this),
- term.field(),
- term.bytes(),
- false);
- if (docs == null) {
- return 0;
- }
- int n = 0;
- int doc;
- while ((doc = docs.nextDoc()) != DocsEnum.NO_MORE_DOCS) {
- deleteDocument(doc);
- n++;
- }
- return n;
- }
-
- /** Undeletes all documents currently marked as deleted in
- * this index.
- *
- * <p>NOTE: this method can only recover documents marked
- * for deletion but not yet removed from the index; when
- * and how Lucene removes deleted documents is an
- * implementation detail, subject to change from release
- * to release. However, you can use {@link
- * #numDeletedDocs} on the current IndexReader instance to
- * see how many documents will be un-deleted.
- *
- * @throws StaleReaderException if the index has changed
- * since this reader was opened
- * @throws LockObtainFailedException if another writer
- * has this index open (<code>write.lock</code> could not
- * be obtained)
- * @throws CorruptIndexException if the index is corrupt
- * @throws IOException if there is a low-level IO error
- */
- public synchronized void undeleteAll() throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException {
- ensureOpen();
- acquireWriteLock();
- hasChanges = true;
- doUndeleteAll();
- }
-
- /** Implements actual undeleteAll() in subclass. */
- protected abstract void doUndeleteAll() throws CorruptIndexException, IOException;
-
- /** Does nothing by default. Subclasses that require a write lock for
- * index modifications must implement this method. */
- protected synchronized void acquireWriteLock() throws IOException {
- /* NOOP */
- }
-
- /**
- *
- * @throws IOException
- */
- public final synchronized void flush() throws IOException {
- ensureOpen();
- commit();
- }
-
- /**
- * @param commitUserData Opaque Map (String -> String)
- * that's recorded into the segments file in the index,
- * and retrievable by {@link
- * IndexReader#getCommitUserData}.
- * @throws IOException
- */
- public final synchronized void flush(Map<String, String> commitUserData) throws IOException {
- ensureOpen();
- commit(commitUserData);
- }
-
- /**
- * Commit changes resulting from delete, undeleteAll, or
- * setNorm operations
- *
- * If an exception is hit, then either no changes or all
- * changes will have been committed to the index
- * (transactional semantics).
- * @throws IOException if there is a low-level IO error
- */
- protected final synchronized void commit() throws IOException {
- commit(null);
- }
-
- /**
- * Commit changes resulting from delete, undeleteAll, or
- * setNorm operations
- *
- * If an exception is hit, then either no changes or all
- * changes will have been committed to the index
- * (transactional semantics).
- * @throws IOException if there is a low-level IO error
- */
- public final synchronized void commit(Map<String, String> commitUserData) throws IOException {
- // Don't call ensureOpen since we commit() on close
- doCommit(commitUserData);
- hasChanges = false;
- }
-
- /** Implements commit. */
- protected abstract void doCommit(Map<String, String> commitUserData) throws IOException;
-
/**
* Closes files associated with this index.
* Also saves any new deletions to disk.
@@ -1322,17 +957,28 @@ public abstract class IndexReader implem
public static void main(String [] args) {
String filename = null;
boolean extract = false;
+ String dirImpl = null;
- for (int i = 0; i < args.length; ++i) {
- if (args[i].equals("-extract")) {
+ int j = 0;
+ while(j < args.length) {
+ String arg = args[j];
+ if ("-extract".equals(arg)) {
extract = true;
+ } else if ("-dir-impl".equals(arg)) {
+ if (j == args.length - 1) {
+ System.out.println("ERROR: missing value for -dir-impl option");
+ System.exit(1);
+ }
+ j++;
+ dirImpl = args[j];
} else if (filename == null) {
- filename = args[i];
+ filename = arg;
}
+ j++;
}
if (filename == null) {
- System.out.println("Usage: org.apache.lucene.index.IndexReader [-extract] <cfsfile>");
+ System.out.println("Usage: org.apache.lucene.index.IndexReader [-extract] [-dir-impl X] <cfsfile>");
return;
}
@@ -1344,7 +990,12 @@ public abstract class IndexReader implem
File file = new File(filename);
String dirname = file.getAbsoluteFile().getParent();
filename = file.getName();
- dir = FSDirectory.open(new File(dirname));
+ if (dirImpl == null) {
+ dir = FSDirectory.open(new File(dirname));
+ } else {
+ dir = CommandLineUtil.newFSDirectory(dirImpl, new File(dirname));
+ }
+
cfr = new CompoundFileDirectory(dir, filename, IOContext.DEFAULT, false);
String [] files = cfr.listAll();
@@ -1397,7 +1048,7 @@ public abstract class IndexReader implem
* one commit point. But if you're using a custom {@link
* IndexDeletionPolicy} then there could be many commits.
* Once you have a given commit, you can open a reader on
- * it by calling {@link IndexReader#open(IndexCommit,boolean)}
+ * it by calling {@link IndexReader#open(IndexCommit)}
* There must be at least one commit in
* the Directory, else this method throws {@link
* IndexNotFoundException}. Note that if a commit is in
@@ -1418,7 +1069,7 @@ public abstract class IndexReader implem
* that has no sub readers).
* <p>
* NOTE: You should not try using sub-readers returned by
- * this method to make any changes (setNorm, deleteDocument,
+ * this method to make any changes (deleteDocument,
* etc.). While this might succeed for one composite reader
* (like MultiReader), it will most likely lead to index
* corruption for other readers (like DirectoryReader obtained
@@ -1444,24 +1095,29 @@ public abstract class IndexReader implem
* top-level context holds a <code>null</code> {@link CompositeReaderContext#leaves}
* reference. Only the top-level context maintains the convenience leaf-view
* for performance reasons.
- * <p>
- * NOTE: You should not try using sub-readers returned by this method to make
- * any changes (setNorm, deleteDocument, etc.). While this might succeed for
- * one composite reader (like MultiReader), it will most likely lead to index
- * corruption for other readers (like DirectoryReader obtained through
- * {@link #open}. Use the top-level context's reader directly.
*
* @lucene.experimental
*/
public abstract ReaderContext getTopReaderContext();
- /** Expert */
+ /** Expert: Returns a key for this IndexReader, so FieldCache/CachingWrapperFilter can find
+ * it again.
+ * This key must not have equals()/hashCode() methods, so "equals" means "identical". */
public Object getCoreCacheKey() {
// Don't can ensureOpen since FC calls this (to evict)
// on close
return this;
}
+ /** Expert: Returns a key for this IndexReader that also includes deletions,
+ * so FieldCache/CachingWrapperFilter can find it again.
+ * This key must not have equals()/hashCode() methods, so "equals" means "identical". */
+ public Object getCombinedCoreAndDeletesKey() {
+ // Don't can ensureOpen since FC calls this (to evict)
+ // on close
+ return this;
+ }
+
/** Returns the number of unique terms (across all fields)
* in this reader.
*
@@ -1470,7 +1126,10 @@ public abstract class IndexReader implem
* Instead, you should call {@link
* #getSequentialSubReaders} and ask each sub reader for
* its unique term count. */
- public long getUniqueTermCount() throws IOException {
+ public final long getUniqueTermCount() throws IOException {
+ if (!getTopReaderContext().isAtomic) {
+ return -1;
+ }
final Fields fields = fields();
if (fields == null) {
return 0;
@@ -1487,14 +1146,20 @@ public abstract class IndexReader implem
throw new UnsupportedOperationException("This reader does not support this method.");
}
- public final IndexDocValues docValues(String field) throws IOException {
- ensureOpen();
- final PerDocValues perDoc = perDocValues();
- if (perDoc == null) {
- return null;
- }
- return perDoc.docValues(field);
- }
+ /**
+ * Returns {@link DocValues} for this field.
+ * This method may return null if the reader has no per-document
+ * values stored.
+ *
+ * <p><b>NOTE</b>: if this is a multi reader ({@link
+ * #getSequentialSubReaders} is not null) then this
+ * method will throw UnsupportedOperationException. If
+ * you really need {@link DocValues} for such a reader,
+ * use {@link MultiDocValues#getDocValues(IndexReader,String)}. However, for
+ * performance reasons, it's best to get all sub-readers
+ * using {@link ReaderUtil#gatherSubReaders} and iterate
+ * through them yourself. */
+ public abstract DocValues docValues(String field) throws IOException;
private volatile Fields fields;
@@ -1510,21 +1175,6 @@ public abstract class IndexReader implem
return fields;
}
- private volatile PerDocValues perDocValues;
-
- /** @lucene.internal */
- void storePerDoc(PerDocValues perDocValues) {
- ensureOpen();
- this.perDocValues = perDocValues;
- }
-
- /** @lucene.internal */
- PerDocValues retrievePerDoc() {
- ensureOpen();
- return perDocValues;
- }
-
-
/**
* A struct like class that represents a hierarchical relationship between
* {@link IndexReader} instances.
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java Thu Dec 15 05:11:14 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.util.CommandLineUtil;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.Version;
@@ -54,36 +55,56 @@ public final class IndexUpgrader {
private static void printUsage() {
System.err.println("Upgrades an index so all segments created with a previous Lucene version are rewritten.");
System.err.println("Usage:");
- System.err.println(" java " + IndexUpgrader.class.getName() + " [-delete-prior-commits] [-verbose] indexDir");
+ System.err.println(" java " + IndexUpgrader.class.getName() + " [-delete-prior-commits] [-verbose] [-dir-impl X] indexDir");
System.err.println("This tool keeps only the last commit in an index; for this");
System.err.println("reason, if the incoming index has more than one commit, the tool");
System.err.println("refuses to run by default. Specify -delete-prior-commits to override");
System.err.println("this, allowing the tool to delete all but the last commit.");
+ System.err.println("Specify a " + FSDirectory.class.getSimpleName() +
+ " implementation through the -dir-impl option to force its use. If no package is specified the "
+ + FSDirectory.class.getPackage().getName() + " package will be used.");
System.err.println("WARNING: This tool may reorder document IDs!");
System.exit(1);
}
@SuppressWarnings("deprecation")
public static void main(String[] args) throws IOException {
- String dir = null;
+ String path = null;
boolean deletePriorCommits = false;
PrintStream out = null;
- for (String arg : args) {
+ String dirImpl = null;
+ int i = 0;
+ while (i<args.length) {
+ String arg = args[i];
if ("-delete-prior-commits".equals(arg)) {
deletePriorCommits = true;
} else if ("-verbose".equals(arg)) {
out = System.out;
- } else if (dir == null) {
- dir = arg;
- } else {
+ } else if (path == null) {
+ path = arg;
+ } else if ("-dir-impl".equals(arg)) {
+ if (i == args.length - 1) {
+ System.out.println("ERROR: missing value for -dir-impl option");
+ System.exit(1);
+ }
+ i++;
+ dirImpl = args[i];
+ }else {
printUsage();
}
+ i++;
}
- if (dir == null) {
+ if (path == null) {
printUsage();
}
- new IndexUpgrader(FSDirectory.open(new File(dir)), Version.LUCENE_CURRENT, out, deletePriorCommits).upgrade();
+ Directory dir = null;
+ if (dirImpl == null) {
+ dir = FSDirectory.open(new File(path));
+ } else {
+ dir = CommandLineUtil.newFSDirectory(dirImpl, new File(path));
+ }
+ new IndexUpgrader(dir, Version.LUCENE_CURRENT, out, deletePriorCommits).upgrade();
}
private final Directory dir;
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexWriter.java Thu Dec 15 05:11:14 2011
@@ -29,7 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.regex.Pattern;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment;
@@ -52,7 +52,6 @@ import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.ThreadInterruptedException;
-import org.apache.lucene.util.MapBackedSet;
import org.apache.lucene.util.TwoPhaseCommit;
/**
@@ -331,7 +330,9 @@ public class IndexWriter implements Clos
final long tStart = System.currentTimeMillis();
- infoStream.message("IW", "flush at getReader");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "flush at getReader");
+ }
// Do this up front before flushing so that the readers
// obtained during this flush are pooled, the first time
// this method is called:
@@ -361,7 +362,7 @@ public class IndexWriter implements Clos
// just like we do when loading segments_N
synchronized(this) {
maybeApplyDeletes(applyAllDeletes);
- r = new DirectoryReader(this, segmentInfos, applyAllDeletes);
+ r = DirectoryReader.open(this, segmentInfos, applyAllDeletes);
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "return reader version=" + r.getVersion() + " reader=" + r);
}
@@ -371,8 +372,10 @@ public class IndexWriter implements Clos
// never reached but javac disagrees:
return null;
} finally {
- if (!success && infoStream.isEnabled("IW")) {
- infoStream.message("IW", "hit exception during NRT reader");
+ if (!success) {
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "hit exception during NRT reader");
+ }
}
// Done: finish the full flush!
docWriter.finishFullFlush(success);
@@ -388,13 +391,6 @@ public class IndexWriter implements Clos
return r;
}
- // Used for all SegmentReaders we open
- private final Collection<IndexReader.ReaderFinishedListener> readerFinishedListeners = new MapBackedSet<IndexReader.ReaderFinishedListener>(new ConcurrentHashMap<IndexReader.ReaderFinishedListener,Boolean>());
-
- Collection<IndexReader.ReaderFinishedListener> getReaderFinishedListeners() throws IOException {
- return readerFinishedListeners;
- }
-
/** Holds shared SegmentReader instances. IndexWriter uses
* SegmentReaders for 1) applying deletes, 2) doing
* merges, 3) handing out a real-time reader. This pool
@@ -618,7 +614,7 @@ public class IndexWriter implements Clos
SegmentReader sr = ent.getValue();
if (sr.hasChanges) {
assert infoIsLive(sr.getSegmentInfo(), "key=" + ent.getKey());
- sr.doCommit(null);
+ sr.doCommit();
// Must checkpoint w/ deleter, because this
// segment reader will have created new _X_N.del
@@ -650,7 +646,7 @@ public class IndexWriter implements Clos
final SegmentReader sr = readerMap.get(new SegmentCacheKey(info, IOContext.Context.READ));
if (sr != null && sr.hasChanges) {
assert infoIsLive(info);
- sr.doCommit(null);
+ sr.doCommit();
// Must checkpoint w/ deleter, because this
// segment reader will have created new _X_N.del
// file.
@@ -697,8 +693,7 @@ public class IndexWriter implements Clos
// TODO: we may want to avoid doing this while
// synchronized
// Returns a ref, which we xfer to readerMap:
- sr = SegmentReader.get(false, info.dir, info, doOpenStores, context.context == IOContext.Context.MERGE ? -1 : config.getReaderTermsIndexDivisor(), context);
- sr.readerFinishedListeners = readerFinishedListeners;
+ sr = SegmentReader.getRW(info, doOpenStores, context.context == IOContext.Context.MERGE ? -1 : config.getReaderTermsIndexDivisor(), context);
if (info.dir == directory) {
// Only pool if reader is not external
@@ -872,8 +867,9 @@ public class IndexWriter implements Clos
segmentInfos.replace(oldInfos);
changeCount++;
segmentInfos.changed();
- if (infoStream.isEnabled("IW"))
+ if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "init: loaded commit \"" + commit.getSegmentsFileName() + "\"");
+ }
}
}
@@ -909,7 +905,9 @@ public class IndexWriter implements Clos
} finally {
if (!success) {
- infoStream.message("IW", "init: hit exception on init; releasing write lock");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "init: hit exception on init; releasing write lock");
+ }
try {
writeLock.release();
} catch (Throwable t) {
@@ -1077,14 +1075,17 @@ public class IndexWriter implements Clos
mergeScheduler.close();
- infoStream.message("IW", "now call final commit()");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "now call final commit()");
+ }
if (!hitOOM) {
commitInternal(null);
}
- if (infoStream.isEnabled("IW"))
+ if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "at close: " + segString());
+ }
// used by assert below
final DocumentsWriter oldWriter = docWriter;
synchronized(this) {
@@ -1108,7 +1109,9 @@ public class IndexWriter implements Clos
closing = false;
notifyAll();
if (!closed) {
- infoStream.message("IW", "hit exception while closing");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "hit exception while closing");
+ }
}
}
}
@@ -1337,8 +1340,10 @@ public class IndexWriter implements Clos
anySegmentFlushed = docWriter.updateDocuments(docs, analyzer, delTerm);
success = true;
} finally {
- if (!success && infoStream.isEnabled("IW")) {
- infoStream.message("IW", "hit exception updating document");
+ if (!success) {
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "hit exception updating document");
+ }
}
}
if (anySegmentFlushed) {
@@ -1484,8 +1489,10 @@ public class IndexWriter implements Clos
anySegmentFlushed = docWriter.updateDocument(doc, analyzer, term);
success = true;
} finally {
- if (!success && infoStream.isEnabled("IW")) {
- infoStream.message("IW", "hit exception updating document");
+ if (!success) {
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "hit exception updating document");
+ }
}
}
@@ -1743,8 +1750,9 @@ public class IndexWriter implements Clos
flush(true, true);
- if (infoStream.isEnabled("IW"))
+ if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "forceMergeDeletes: index now " + segString());
+ }
MergePolicy.MergeSpecification spec;
@@ -1937,7 +1945,9 @@ public class IndexWriter implements Clos
boolean success = false;
- infoStream.message("IW", "rollback");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "rollback");
+ }
try {
synchronized(this) {
@@ -1945,7 +1955,9 @@ public class IndexWriter implements Clos
stopMerges = true;
}
- infoStream.message("IW", "rollback: done finish merges");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "rollback: done finish merges");
+ }
// Must pre-close these two, in case they increment
// changeCount so that we can then set it to false
@@ -1997,7 +2009,9 @@ public class IndexWriter implements Clos
if (!success) {
closing = false;
notifyAll();
- infoStream.message("IW", "hit exception during rollback");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "hit exception during rollback");
+ }
}
}
}
@@ -2049,8 +2063,10 @@ public class IndexWriter implements Clos
} catch (OutOfMemoryError oom) {
handleOOM(oom, "deleteAll");
} finally {
- if (!success && infoStream.isEnabled("IW")) {
- infoStream.message("IW", "hit exception during deleteAll");
+ if (!success) {
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "hit exception during deleteAll");
+ }
}
}
}
@@ -2062,16 +2078,18 @@ public class IndexWriter implements Clos
// Abort all pending & running merges:
for (final MergePolicy.OneMerge merge : pendingMerges) {
- if (infoStream.isEnabled("IW"))
+ if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "now abort pending merge " + merge.segString(directory));
+ }
merge.abort();
mergeFinish(merge);
}
pendingMerges.clear();
for (final MergePolicy.OneMerge merge : runningMerges) {
- if (infoStream.isEnabled("IW"))
+ if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "now abort running merge " + merge.segString(directory));
+ }
merge.abort();
}
@@ -2081,8 +2099,9 @@ public class IndexWriter implements Clos
// because the merge threads periodically check if
// they are aborted.
while(runningMerges.size() > 0) {
- if (infoStream.isEnabled("IW"))
+ if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "now wait for " + runningMerges.size() + " running merge to abort");
+ }
doWait();
}
@@ -2091,7 +2110,9 @@ public class IndexWriter implements Clos
assert 0 == mergingSegments.size();
- infoStream.message("IW", "all running merges have aborted");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "all running merges have aborted");
+ }
} else {
// waitForMerges() will ensure any running addIndexes finishes.
@@ -2111,7 +2132,9 @@ public class IndexWriter implements Clos
*/
public synchronized void waitForMerges() {
ensureOpen(false);
- infoStream.message("IW", "waitForMerges");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "waitForMerges");
+ }
while(pendingMerges.size() > 0 || runningMerges.size() > 0) {
doWait();
}
@@ -2119,7 +2142,9 @@ public class IndexWriter implements Clos
// sanity check
assert 0 == mergingSegments.size();
- infoStream.message("IW", "waitForMerges done");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "waitForMerges done");
+ }
}
/**
@@ -2246,7 +2271,9 @@ public class IndexWriter implements Clos
FrozenBufferedDeletes packet, FrozenBufferedDeletes globalPacket) throws IOException {
// Lock order IW -> BDS
synchronized (bufferedDeletesStream) {
- infoStream.message("IW", "publishFlushedSegment");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "publishFlushedSegment");
+ }
if (globalPacket != null && globalPacket.any()) {
bufferedDeletesStream.push(globalPacket);
@@ -2340,7 +2367,10 @@ public class IndexWriter implements Clos
noDupDirs(dirs);
try {
- infoStream.message("IW", "flush at addIndexes(Directory...)");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "flush at addIndexes(Directory...)");
+ }
+
flush(false, true);
List<SegmentInfo> infos = new ArrayList<SegmentInfo>();
@@ -2409,7 +2439,9 @@ public class IndexWriter implements Clos
int numDocs = 0;
try {
- infoStream.message("IW", "flush at addIndexes(IndexReader...)");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "flush at addIndexes(IndexReader...)");
+ }
flush(false, true);
String mergedName = newSegmentName();
@@ -2637,7 +2669,9 @@ public class IndexWriter implements Clos
success = true;
} finally {
if (!success) {
- infoStream.message("IW", "hit exception during prepareCommit");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "hit exception during prepareCommit");
+ }
}
// Done: finish the full flush!
docWriter.finishFullFlush(flushSuccess);
@@ -2721,16 +2755,24 @@ public class IndexWriter implements Clos
private final void commitInternal(Map<String,String> commitUserData) throws CorruptIndexException, IOException {
- infoStream.message("IW", "commit: start");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "commit: start");
+ }
synchronized(commitLock) {
- infoStream.message("IW", "commit: enter lock");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "commit: enter lock");
+ }
if (pendingCommit == null) {
- infoStream.message("IW", "commit: now prepare");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "commit: now prepare");
+ }
prepareCommit(commitUserData);
} else {
- infoStream.message("IW", "commit: already prepared");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "commit: already prepared");
+ }
}
finishCommit();
@@ -2741,10 +2783,13 @@ public class IndexWriter implements Clos
if (pendingCommit != null) {
try {
- infoStream.message("IW", "commit: pendingCommit != null");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "commit: pendingCommit != null");
+ }
pendingCommit.finishCommit(directory, codec);
- if (infoStream.isEnabled("IW"))
+ if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "commit: wrote segments file \"" + pendingCommit.getCurrentSegmentFileName() + "\"");
+ }
lastCommitChangeCount = pendingCommitChangeCount;
segmentInfos.updateGeneration(pendingCommit);
segmentInfos.setUserData(pendingCommit.getUserData());
@@ -2759,10 +2804,14 @@ public class IndexWriter implements Clos
}
} else {
- infoStream.message("IW", "commit: pendingCommit == null; skip");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "commit: pendingCommit == null; skip");
+ }
}
- infoStream.message("IW", "commit: done");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "commit: done");
+ }
}
// Ensures only one flush() is actually flushing segments
@@ -2832,14 +2881,19 @@ public class IndexWriter implements Clos
// never hit
return false;
} finally {
- if (!success)
- infoStream.message("IW", "hit exception during flush");
+ if (!success) {
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "hit exception during flush");
+ }
+ }
}
}
final synchronized void maybeApplyDeletes(boolean applyAllDeletes) throws IOException {
if (applyAllDeletes) {
- infoStream.message("IW", "apply all deletes during flush");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "apply all deletes during flush");
+ }
applyAllDeletes();
} else if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "don't apply deletes now delTermCount=" + bufferedDeletesStream.numTerms() + " bytesUsed=" + bufferedDeletesStream.bytesUsed());
@@ -2919,8 +2973,9 @@ public class IndexWriter implements Clos
final List<SegmentInfo> sourceSegments = merge.segments;
- if (infoStream.isEnabled("IW"))
+ if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "commitMergeDeletes " + merge.segString(directory));
+ }
// Carefully merge deletes that occurred after we
// started merging:
@@ -3017,8 +3072,9 @@ public class IndexWriter implements Clos
throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot complete merge");
}
- if (infoStream.isEnabled("IW"))
+ if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "commitMerge: " + merge.segString(directory) + " index=" + segString());
+ }
assert merge.registerDone;
@@ -3029,8 +3085,9 @@ public class IndexWriter implements Clos
// file that current segments does not reference), we
// abort this merge
if (merge.isAborted()) {
- if (infoStream.isEnabled("IW"))
+ if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "commitMerge: skipping merge " + merge.segString(directory) + ": it was aborted");
+ }
return false;
}
@@ -3045,8 +3102,10 @@ public class IndexWriter implements Clos
final boolean allDeleted = mergedReader.numDocs() == 0;
- if (allDeleted && infoStream.isEnabled("IW")) {
- infoStream.message("IW", "merged segment " + merge.info + " is 100% deleted" + (keepFullyDeletedSegments ? "" : "; skipping insert"));
+ if (allDeleted) {
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "merged segment " + merge.info + " is 100% deleted" + (keepFullyDeletedSegments ? "" : "; skipping insert"));
+ }
}
final boolean dropSegment = allDeleted && !keepFullyDeletedSegments;
@@ -3132,8 +3191,9 @@ public class IndexWriter implements Clos
try {
mergeInit(merge);
- if (infoStream.isEnabled("IW"))
+ if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "now merge\n merge=" + merge.segString(directory) + "\n index=" + segString());
+ }
mergeMiddle(merge);
mergeSuccess(merge);
@@ -3146,9 +3206,12 @@ public class IndexWriter implements Clos
mergeFinish(merge);
if (!success) {
- infoStream.message("IW", "hit exception during merge");
- if (merge.info != null && !segmentInfos.contains(merge.info))
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "hit exception during merge");
+ }
+ if (merge.info != null && !segmentInfos.contains(merge.info)) {
deleter.refresh(merge.info.name);
+ }
}
// This merge (and, generally, any change to the
@@ -3162,8 +3225,10 @@ public class IndexWriter implements Clos
} catch (OutOfMemoryError oom) {
handleOOM(oom, "merge");
}
- if (merge.info != null && infoStream.isEnabled("IW")) {
- infoStream.message("IW", "merge time " + (System.currentTimeMillis()-t0) + " msec for " + merge.info.docCount + " docs");
+ if (merge.info != null) {
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "merge time " + (System.currentTimeMillis()-t0) + " msec for " + merge.info.docCount + " docs");
+ }
}
//System.out.println(Thread.currentThread().getName() + ": merge end");
}
@@ -3208,8 +3273,9 @@ public class IndexWriter implements Clos
pendingMerges.add(merge);
- if (infoStream.isEnabled("IW"))
+ if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "add merge to pendingMerges: " + merge.segString(directory) + " [total " + pendingMerges.size() + " pending]");
+ }
merge.mergeGen = mergeGen;
merge.isExternal = isExternal;
@@ -3226,7 +3292,9 @@ public class IndexWriter implements Clos
builder.append("]");
// don't call mergingSegments.toString() could lead to ConcurrentModException
// since merge updates the segments FieldInfos
- infoStream.message("IW", builder.toString());
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", builder.toString());
+ }
}
for(SegmentInfo info : merge.segments) {
if (infoStream.isEnabled("IW")) {
@@ -3250,7 +3318,9 @@ public class IndexWriter implements Clos
success = true;
} finally {
if (!success) {
- infoStream.message("IW", "hit exception in mergeInit");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "hit exception in mergeInit");
+ }
mergeFinish(merge);
}
}
@@ -3543,7 +3613,9 @@ public class IndexWriter implements Clos
handleMergeException(t, merge);
} finally {
if (!success) {
- infoStream.message("IW", "hit exception creating compound file during merge");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "hit exception creating compound file during merge");
+ }
synchronized(this) {
deleter.deleteFile(compoundFileName);
@@ -3562,7 +3634,9 @@ public class IndexWriter implements Clos
deleter.deleteNewFiles(merge.info.files());
if (merge.isAborted()) {
- infoStream.message("IW", "abort merge after building CFS");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "abort merge after building CFS");
+ }
deleter.deleteFile(compoundFileName);
return 0;
}
@@ -3745,14 +3819,18 @@ public class IndexWriter implements Clos
try {
- infoStream.message("IW", "startCommit(): start");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "startCommit(): start");
+ }
synchronized(this) {
assert lastCommitChangeCount <= changeCount;
if (pendingCommitChangeCount == lastCommitChangeCount) {
- infoStream.message("IW", " skip startCommit(): no changes pending");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", " skip startCommit(): no changes pending");
+ }
deleter.decRef(filesToCommit);
filesToCommit = null;
return;
@@ -3795,7 +3873,9 @@ public class IndexWriter implements Clos
pendingCommit = toSync;
}
- infoStream.message("IW", "done all syncs");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "done all syncs");
+ }
assert testPoint("midStartCommitSuccess");
@@ -3808,7 +3888,9 @@ public class IndexWriter implements Clos
segmentInfos.updateGeneration(toSync);
if (!pendingCommitSet) {
- infoStream.message("IW", "hit exception committing segments file");
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "hit exception committing segments file");
+ }
// Hit exception
deleter.decRef(filesToCommit);
@@ -3980,7 +4062,7 @@ public class IndexWriter implements Clos
for (String file : files) {
assert !IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION)
: ".del file is not allowed in .cfs: " + file;
- assert !IndexFileNames.isSeparateNormsFile(file)
+ assert !isSeparateNormsFile(file)
: "separate norms file (.s[0-9]+) is not allowed in .cfs: " + file;
directory.copy(cfsDir, file, file, context);
checkAbort.work(directory.fileLength(file));
@@ -3991,4 +4073,18 @@ public class IndexWriter implements Clos
return files;
}
+
+
+ /**
+ * Returns true if the given filename ends with the separate norms file
+ * pattern: {@code SEPARATE_NORMS_EXTENSION + "[0-9]+"}.
+ * @deprecated only for asserting
+ */
+ @Deprecated
+ private static boolean isSeparateNormsFile(String filename) {
+ int idx = filename.lastIndexOf('.');
+ if (idx == -1) return false;
+ String ext = filename.substring(idx + 1);
+ return Pattern.matches("s[0-9]+", ext);
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableField.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableField.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/IndexableField.java Thu Dec 15 05:11:14 2011
@@ -23,8 +23,7 @@ import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.NumericField;
-import org.apache.lucene.index.values.PerDocFieldValues;
-import org.apache.lucene.index.values.ValueType;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.util.BytesRef;
// TODO: how to handle versioning here...?
@@ -77,10 +76,10 @@ public interface IndexableField {
public IndexableFieldType fieldType();
/** Non-null if doc values should be indexed */
- public PerDocFieldValues docValues();
+ public DocValue docValue();
- /** DocValues type; only used if docValues is non-null */
- public ValueType docValuesType();
+ /** DocValues type; only used if docValue is non-null */
+ public DocValues.Type docValueType();
/**
* Creates the TokenStream used for indexing this field. If appropriate,
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java Thu Dec 15 05:11:14 2011
@@ -23,7 +23,6 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
-import org.apache.lucene.util.InfoStream;
/** <p>This class implements a {@link MergePolicy} that tries
* to merge segments into levels of exponentially
@@ -424,8 +423,9 @@ public abstract class LogMergePolicy ext
final List<SegmentInfo> segments = segmentInfos.asList();
final int numSegments = segments.size();
- if (verbose())
+ if (verbose()) {
message("findForcedDeleteMerges: " + numSegments + " segments");
+ }
MergeSpecification spec = new MergeSpecification();
int firstSegmentWithDeletions = -1;
@@ -435,15 +435,17 @@ public abstract class LogMergePolicy ext
final SegmentInfo info = segmentInfos.info(i);
int delCount = w.numDeletedDocs(info);
if (delCount > 0) {
- if (verbose())
+ if (verbose()) {
message(" segment " + info.name + " has deletions");
+ }
if (firstSegmentWithDeletions == -1)
firstSegmentWithDeletions = i;
else if (i - firstSegmentWithDeletions == mergeFactor) {
// We've seen mergeFactor segments in a row with
// deletions, so force a merge now:
- if (verbose())
+ if (verbose()) {
message(" add merge " + firstSegmentWithDeletions + " to " + (i-1) + " inclusive");
+ }
spec.add(new OneMerge(segments.subList(firstSegmentWithDeletions, i)));
firstSegmentWithDeletions = i;
}
@@ -451,16 +453,18 @@ public abstract class LogMergePolicy ext
// End of a sequence of segments with deletions, so,
// merge those past segments even if it's fewer than
// mergeFactor segments
- if (verbose())
+ if (verbose()) {
message(" add merge " + firstSegmentWithDeletions + " to " + (i-1) + " inclusive");
+ }
spec.add(new OneMerge(segments.subList(firstSegmentWithDeletions, i)));
firstSegmentWithDeletions = -1;
}
}
if (firstSegmentWithDeletions != -1) {
- if (verbose())
+ if (verbose()) {
message(" add merge " + firstSegmentWithDeletions + " to " + (numSegments-1) + " inclusive");
+ }
spec.add(new OneMerge(segments.subList(firstSegmentWithDeletions, numSegments)));
}
@@ -500,8 +504,9 @@ public abstract class LogMergePolicy ext
public MergeSpecification findMerges(SegmentInfos infos) throws IOException {
final int numSegments = infos.size();
- if (verbose())
+ if (verbose()) {
message("findMerges: " + numSegments + " segments");
+ }
// Compute levels, which is just log (base mergeFactor)
// of the size of each segment
@@ -582,8 +587,9 @@ public abstract class LogMergePolicy ext
}
upto--;
}
- if (verbose())
+ if (verbose()) {
message(" level " + levelBottom + " to " + maxLevel + ": " + (1+upto-start) + " segments");
+ }
// Finally, record all merges that are viable at this level:
int end = start + mergeFactor;
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java?rev=1214623&r1=1214622&r2=1214623&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java Thu Dec 15 05:11:14 2011
@@ -17,7 +17,6 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.index.values.MultiIndexDocValues;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.ReaderUtil;
@@ -39,14 +38,9 @@ public final class MultiFieldsEnum exte
// Holds sub-readers containing field we are currently
// on, popped from queue.
private final FieldsEnumWithSlice[] top;
- private final FieldsEnumWithSlice[] enumWithSlices;
private int numTop;
- // Re-used TermsEnum
- private final MultiTermsEnum terms;
- private final MultiIndexDocValues docValues;
-
private final Fields fields;
private String currentField;
@@ -55,9 +49,7 @@ public final class MultiFieldsEnum exte
* (ie, {@link FieldsEnum#next} has not been called. */
public MultiFieldsEnum(MultiFields fields, FieldsEnum[] subs, ReaderUtil.Slice[] subSlices) throws IOException {
this.fields = fields;
- terms = new MultiTermsEnum(subSlices);
queue = new FieldMergeQueue(subs.length);
- docValues = new MultiIndexDocValues();
top = new FieldsEnumWithSlice[subs.length];
List<FieldsEnumWithSlice> enumWithSlices = new ArrayList<FieldsEnumWithSlice>();
@@ -73,8 +65,6 @@ public final class MultiFieldsEnum exte
queue.add(sub);
}
}
- this.enumWithSlices = enumWithSlices.toArray(FieldsEnumWithSlice.EMPTY_ARRAY);
-
}
@Override