You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/03/30 11:17:42 UTC
svn commit: r1086876 [7/18] - in /lucene/dev/branches/realtime_search: ./
dev-tools/eclipse/ dev-tools/idea/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/ dev-tools/idea/solr/
dev-tools/idea/solr/contrib/analysis-extras/ dev-tools/idea/solr/co...
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriter.java Wed Mar 30 09:17:25 2011
@@ -36,8 +36,10 @@ import java.util.concurrent.ConcurrentHa
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment;
+import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.PayloadProcessorProvider.DirPayloadProcessor;
+import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.search.Query;
import org.apache.lucene.store.AlreadyClosedException;
@@ -45,7 +47,6 @@ import org.apache.lucene.store.BufferedI
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.ThreadInterruptedException;
@@ -55,22 +56,22 @@ import org.apache.lucene.util.MapBackedS
An <code>IndexWriter</code> creates and maintains an index.
<p>The <code>create</code> argument to the {@link
- #IndexWriter(Directory, IndexWriterConfig) constructor} determines
+ #IndexWriter(Directory, IndexWriterConfig) constructor} determines
whether a new index is created, or whether an existing index is
opened. Note that you can open an index with <code>create=true</code>
- even while readers are using the index. The old readers will
- continue to search the "point in time" snapshot they had opened,
+ even while readers are using the index. The old readers will
+ continue to search the "point in time" snapshot they had opened,
and won't see the newly created index until they re-open. There are
also {@link #IndexWriter(Directory, IndexWriterConfig) constructors}
with no <code>create</code> argument which will create a new index
- if there is not already an index at the provided path and otherwise
+ if there is not already an index at the provided path and otherwise
open the existing index.</p>
<p>In either case, documents are added with {@link #addDocument(Document)
addDocument} and removed with {@link #deleteDocuments(Term)} or {@link
#deleteDocuments(Query)}. A document can be updated with {@link
#updateDocument(Term, Document) updateDocument} (which just deletes
- and then adds the entire document). When finished adding, deleting
+ and then adds the entire document). When finished adding, deleting
and updating documents, {@link #close() close} should be called.</p>
<a name="flush"></a>
@@ -103,7 +104,7 @@ import org.apache.lucene.util.MapBackedS
{@link LockObtainFailedException}. The {@link LockObtainFailedException}
is also thrown if an IndexReader on the same directory is used to delete documents
from the index.</p>
-
+
<a name="deletionPolicy"></a>
<p>Expert: <code>IndexWriter</code> allows an optional
{@link IndexDeletionPolicy} implementation to be
@@ -153,7 +154,7 @@ import org.apache.lucene.util.MapBackedS
synchronize on the <code>IndexWriter</code> instance as
this may cause deadlock; use your own (non-Lucene) objects
instead. </p>
-
+
<p><b>NOTE</b>: If you call
<code>Thread.interrupt()</code> on a thread that's within
IndexWriter, IndexWriter will try to catch this (eg, if
@@ -222,7 +223,7 @@ public class IndexWriter implements Clos
volatile long pendingCommitChangeCount;
final SegmentInfos segmentInfos; // the segments
- final FieldInfos fieldInfos;
+ final FieldNumberBiMap globalFieldNumberMap;
private DocumentsWriter docWriter;
final IndexFileDeleter deleter;
@@ -232,8 +233,6 @@ public class IndexWriter implements Clos
private Lock writeLock;
- private final int termIndexInterval;
-
private boolean closed;
private boolean closing;
@@ -643,7 +642,7 @@ public class IndexWriter implements Clos
/**
* Obtain the number of deleted docs for a pooled reader.
- * If the reader isn't being pooled, the segmentInfo's
+ * If the reader isn't being pooled, the segmentInfo's
* delCount is returned.
*/
public int numDeletedDocs(SegmentInfo info) throws IOException {
@@ -691,15 +690,11 @@ public class IndexWriter implements Clos
/**
* Constructs a new IndexWriter per the settings given in <code>conf</code>.
- * Note that the passed in {@link IndexWriterConfig} is cloned and thus making
- * changes to it after IndexWriter has been instantiated will not affect
- * IndexWriter. Additionally, calling {@link #getConfig()} and changing the
- * parameters does not affect that IndexWriter instance.
+ * Note that the passed in {@link IndexWriterConfig} is
+ * privately cloned; if you need to make subsequent "live"
+ * changes to the configuration use {@link #getConfig}.
* <p>
- * <b>NOTE:</b> by default, {@link IndexWriterConfig#getMaxFieldLength()}
- * returns {@link IndexWriterConfig#UNLIMITED_FIELD_LENGTH}. Pay attention to
- * whether this setting fits your application.
- *
+ *
* @param d
* the index directory. The index is either created or appended
* according <code>conf.getOpenMode()</code>.
@@ -723,11 +718,9 @@ public class IndexWriter implements Clos
directory = d;
analyzer = conf.getAnalyzer();
infoStream = defaultInfoStream;
- termIndexInterval = conf.getTermIndexInterval();
mergePolicy = conf.getMergePolicy();
mergePolicy.setIndexWriter(this);
mergeScheduler = conf.getMergeScheduler();
- mergedSegmentWarmer = conf.getMergedSegmentWarmer();
codecs = conf.getCodecProvider();
bufferedDeletesStream = new BufferedDeletesStream(messageID);
@@ -797,9 +790,8 @@ public class IndexWriter implements Clos
setRollbackSegmentInfos(segmentInfos);
// start with previous field numbers, but new FieldInfos
- fieldInfos = getCurrentFieldInfos();
- docWriter = new DocumentsWriter(directory, this, conf.getIndexingChain(), conf.getIndexerThreadPool(),
- fieldInfos.newFieldInfosWithGlobalFieldNumberMap(), bufferedDeletesStream);
+ globalFieldNumberMap = segmentInfos.getOrLoadGlobalFieldNumberMap(directory);
+ docWriter = new DocumentsWriter(config, directory, this, globalFieldNumberMap, bufferedDeletesStream);
docWriter.setInfoStream(infoStream);
// Default deleter (for backwards compatibility) is
@@ -817,10 +809,6 @@ public class IndexWriter implements Clos
segmentInfos.changed();
}
- docWriter.setRAMBufferSizeMB(conf.getRAMBufferSizeMB());
- docWriter.setMaxBufferedDocs(conf.getMaxBufferedDocs());
- pushMaxBufferedDocs();
-
if (infoStream != null) {
message("init: create=" + create);
messageState();
@@ -842,76 +830,26 @@ public class IndexWriter implements Clos
}
}
}
-
- private FieldInfos getFieldInfos(SegmentInfo info) throws IOException {
- Directory cfsDir = null;
- try {
- if (info.getUseCompoundFile()) {
- cfsDir = new CompoundFileReader(directory, IndexFileNames.segmentFileName(info.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION));
- } else {
- cfsDir = directory;
- }
- return new FieldInfos(cfsDir, IndexFileNames.segmentFileName(info.name, "", IndexFileNames.FIELD_INFOS_EXTENSION));
- } finally {
- if (info.getUseCompoundFile() && cfsDir != null) {
- cfsDir.close();
- }
- }
- }
-
- private FieldInfos getCurrentFieldInfos() throws IOException {
- final FieldInfos fieldInfos;
- if (segmentInfos.size() > 0) {
- fieldInfos = new FieldInfos();
- for(SegmentInfo info : segmentInfos) {
- final FieldInfos segFieldInfos = getFieldInfos(info);
- for (FieldInfo fi : segFieldInfos) {
- fieldInfos.add(fi);
- }
- }
- } else {
- fieldInfos = new FieldInfos();
- }
- return fieldInfos;
- }
-
+
private synchronized void setRollbackSegmentInfos(SegmentInfos infos) {
rollbackSegmentInfos = (SegmentInfos) infos.clone();
}
/**
- * Returns the {@link IndexWriterConfig} that was passed to
- * {@link #IndexWriter(Directory, IndexWriterConfig)}. This allows querying
- * IndexWriter's settings.
+ * Returns the private {@link IndexWriterConfig}, cloned
+ * from the {@link IndexWriterConfig} passed to
+ * {@link #IndexWriter(Directory, IndexWriterConfig)}.
* <p>
- * <b>NOTE:</b> setting any parameter on the returned instance has not effect
- * on the IndexWriter instance. If you need to change those settings after
- * IndexWriter has been created, you need to instantiate a new IndexWriter.
+ * <b>NOTE:</b> some settings may be changed on the
+ * returned {@link IndexWriterConfig}, and will take
+ * effect in the current IndexWriter instance. See the
+ * javadocs for the specific setters in {@link
+ * IndexWriterConfig} for details.
*/
public IndexWriterConfig getConfig() {
return config;
}
- /**
- * If we are flushing by doc count (not by RAM usage), and
- * using LogDocMergePolicy then push maxBufferedDocs down
- * as its minMergeDocs, to keep backwards compatibility.
- */
- private void pushMaxBufferedDocs() {
- if (docWriter.getMaxBufferedDocs() != IndexWriterConfig.DISABLE_AUTO_FLUSH) {
- final MergePolicy mp = mergePolicy;
- if (mp instanceof LogDocMergePolicy) {
- LogDocMergePolicy lmp = (LogDocMergePolicy) mp;
- final int maxBufferedDocs = docWriter.getMaxBufferedDocs();
- if (lmp.getMinMergeDocs() != maxBufferedDocs) {
- if (infoStream != null)
- message("now push maxBufferedDocs " + maxBufferedDocs + " to LogDocMergePolicy");
- lmp.setMinMergeDocs(maxBufferedDocs);
- }
- }
- }
- }
-
/** If non-null, this will be the default infoStream used
* by a newly instantiated IndexWriter.
* @see #setInfoStream
@@ -1197,9 +1135,7 @@ public class IndexWriter implements Clos
}
/**
- * Adds a document to this index. If the document contains more than
- * {@link IndexWriterConfig#setMaxFieldLength(int)} terms for a given field,
- * the remainder are discarded.
+ * Adds a document to this index.
*
* <p> Note that if an Exception is hit (for example disk full)
* then the index will be consistent, but this document
@@ -1246,9 +1182,7 @@ public class IndexWriter implements Clos
/**
* Adds a document to this index, using the provided analyzer instead of the
- * value of {@link #getAnalyzer()}. If the document contains more than
- * {@link IndexWriterConfig#setMaxFieldLength(int)} terms for a given field, the remainder are
- * discarded.
+ * value of {@link #getAnalyzer()}.
*
* <p>See {@link #addDocument(Document)} for details on
* index and IndexWriter state after an Exception, and
@@ -1430,6 +1364,11 @@ public class IndexWriter implements Clos
}
// for test purpose
+ final synchronized Collection<String> getIndexFileNames() throws IOException {
+ return segmentInfos.files(directory, true);
+ }
+
+ // for test purpose
final synchronized int getDocCount(int i) {
if (i >= 0 && i < segmentInfos.size()) {
return segmentInfos.info(i).docCount;
@@ -1465,8 +1404,8 @@ public class IndexWriter implements Clos
/** If non-null, information about merges will be printed to this.
*/
- private PrintStream infoStream = null;
- private static PrintStream defaultInfoStream = null;
+ private PrintStream infoStream;
+ private static PrintStream defaultInfoStream;
/**
* Requests an "optimize" operation on an index, priming the index
@@ -2295,8 +2234,6 @@ public class IndexWriter implements Clos
}
// Update SI appropriately
- // if this call is removed in the future we need to make
- // sure that info.clearFiles() is called here
info.setDocStore(info.getDocStoreOffset(), newDsName, info.getDocStoreIsCompoundFile());
info.dir = directory;
info.name = newSegName;
@@ -2316,28 +2253,21 @@ public class IndexWriter implements Clos
}
}
- /**
- * Merges the provided indexes into this index.
- * <p>
- * After this completes, the index is optimized.
- * </p>
- * <p>
- * The provided IndexReaders are not closed.
- * </p>
- *
- * <p>
- * <b>NOTE:</b> while this is running, any attempts to add or delete documents
- * (with another thread) will be paused until this method completes.
- *
- * <p>
- * See {@link #addIndexes} for details on transactional semantics, temporary
- * free space required in the Directory, and non-CFS segments on an Exception.
- * </p>
+ /** Merges the provided indexes into this index.
+ * <p>After this completes, the index is optimized. </p>
+ * <p>The provided IndexReaders are not closed.</p>
+ *
+ * <p><b>NOTE:</b> while this is running, any attempts to
+ * add or delete documents (with another thread) will be
+ * paused until this method completes.
+ *
+ * <p>See {@link #addIndexes} for details on transactional
+ * semantics, temporary free space required in the Directory,
+ * and non-CFS segments on an Exception.</p>
*
- * <p>
- * <b>NOTE</b>: if this method hits an OutOfMemoryError you should immediately
- * close the writer. See <a href="#OOME">above</a> for details.
- * </p>
+ * <p><b>NOTE</b>: if this method hits an OutOfMemoryError
+ * you should immediately close the writer. See <a
+ * href="#OOME">above</a> for details.</p>
*
* <p><b>NOTE</b>: if you call {@link #close(boolean)}
* with <tt>false</tt>, which aborts all running merges,
@@ -2351,19 +2281,25 @@ public class IndexWriter implements Clos
ensureOpen();
try {
+ if (infoStream != null)
+ message("flush at addIndexes(IndexReader...)");
+ flush(false, true);
+
String mergedName = newSegmentName();
- SegmentMerger merger = new SegmentMerger(directory, termIndexInterval,
+ SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(),
mergedName, null, codecs, payloadProcessorProvider,
- fieldInfos.newFieldInfosWithGlobalFieldNumberMap());
+ globalFieldNumberMap.newFieldInfos(SegmentCodecsBuilder.create(codecs)));
for (IndexReader reader : readers) // add new indexes
merger.add(reader);
int docCount = merger.merge(); // merge 'em
+ final FieldInfos fieldInfos = merger.fieldInfos();
SegmentInfo info = new SegmentInfo(mergedName, docCount, directory,
- false, merger.getSegmentCodecs(),
- merger.fieldInfos());
+ false, fieldInfos.hasProx(), merger.getSegmentCodecs(),
+ fieldInfos.hasVectors(),
+ fieldInfos);
setDiagnostics(info, "addIndexes(IndexReader...)");
boolean useCompoundFile;
@@ -2851,7 +2787,7 @@ public class IndexWriter implements Clos
message("merged segment " + merge.info + " is 100% deleted" + (keepFullyDeletedSegments ? "" : "; skipping insert"));
}
- final Set mergedAway = new HashSet<SegmentInfo>(merge.segments);
+ final Set<SegmentInfo> mergedAway = new HashSet<SegmentInfo>(merge.segments);
int segIdx = 0;
int newSegIdx = 0;
boolean inserted = false;
@@ -3095,7 +3031,7 @@ public class IndexWriter implements Clos
// Bind a new segment name here so even with
// ConcurrentMergePolicy we keep deterministic segment
// names.
- merge.info = new SegmentInfo(newSegmentName(), 0, directory, false, null, fieldInfos.newFieldInfosWithGlobalFieldNumberMap());
+ merge.info = new SegmentInfo(newSegmentName(), 0, directory, false, false, null, false, globalFieldNumberMap.newFieldInfos(SegmentCodecsBuilder.create(codecs)));
// Lock order: IW -> BD
final BufferedDeletesStream.ApplyDeletesResult result = bufferedDeletesStream.applyDeletes(readerPool, merge.segments);
@@ -3244,7 +3180,7 @@ public class IndexWriter implements Clos
SegmentInfos sourceSegments = merge.segments;
- SegmentMerger merger = new SegmentMerger(directory, termIndexInterval, mergedName, merge,
+ SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(), mergedName, merge,
codecs, payloadProcessorProvider,
merge.info.getFieldInfos());
@@ -3255,9 +3191,6 @@ public class IndexWriter implements Clos
merge.readers = new ArrayList<SegmentReader>();
merge.readerClones = new ArrayList<SegmentReader>();
- merge.info.clearFilesCache();
-
-
// This is try/finally to make sure merger's readers are
// closed:
boolean success = false;
@@ -3281,10 +3214,10 @@ public class IndexWriter implements Clos
final SegmentReader clone = (SegmentReader) reader.clone(true);
merge.readerClones.add(clone);
- if (reader.numDocs() > 0) {
+ if (clone.numDocs() > 0) {
merger.add(clone);
+ totDocCount += clone.numDocs();
}
- totDocCount += clone.numDocs();
segUpto++;
}
@@ -3299,12 +3232,14 @@ public class IndexWriter implements Clos
// Record which codec was used to write the segment
merge.info.setSegmentCodecs(merger.getSegmentCodecs());
+ // Record if we have merged vectors
+ merge.info.setHasVectors(merger.fieldInfos().hasVectors());
if (infoStream != null) {
message("merge segmentCodecs=" + merger.getSegmentCodecs());
message("merge store matchedCount=" + merger.getMatchedSubReaderCount() + " vs " + merge.readers.size());
}
- anyNonBulkMerges |= merger.getMatchedSubReaderCount() != merge.readers.size();
+ anyNonBulkMerges |= merger.getAnyNonBulkMerges();
assert mergedDocCount == totDocCount: "mergedDocCount=" + mergedDocCount + " vs " + totDocCount;
@@ -3312,7 +3247,7 @@ public class IndexWriter implements Clos
// because codec must know if prox was written for
// this segment:
//System.out.println("merger set hasProx=" + merger.hasProx() + " seg=" + merge.info.name);
- merge.info.clearFilesCache();
+ merge.info.setHasProx(merger.fieldInfos().hasProx());
boolean useCompoundFile;
synchronized (this) { // Guard segmentInfos
@@ -3374,10 +3309,12 @@ public class IndexWriter implements Clos
merge.info.setUseCompoundFile(true);
}
+ final IndexReaderWarmer mergedSegmentWarmer = config.getMergedSegmentWarmer();
+
final int termsIndexDivisor;
final boolean loadDocStores;
- if (poolReaders && mergedSegmentWarmer != null) {
+ if (mergedSegmentWarmer != null) {
// Load terms index & doc stores so the segment
// warmer can run searches, load documents/term
// vectors
@@ -3470,7 +3407,7 @@ public class IndexWriter implements Clos
// NOTE: the callers of this method should in theory
// be able to do simply wait(), but, as a defense
// against thread timing hazards where notifyAll()
- // falls to be called, we wait for at most 1 second
+ // fails to be called, we wait for at most 1 second
// and then return so caller can check if wait
// conditions are satisfied:
try {
@@ -3654,8 +3591,6 @@ public class IndexWriter implements Clos
public abstract void warm(IndexReader reader) throws IOException;
}
- private IndexReaderWarmer mergedSegmentWarmer;
-
private void handleOOM(OutOfMemoryError oom, String location) {
if (infoStream != null) {
message("hit OutOfMemoryError inside " + location);
@@ -3703,10 +3638,10 @@ public class IndexWriter implements Clos
* IndexWriter, you'll see the unused files linger. If
* that's a problem, call this method to delete them
* (once you've closed the open readers that were
- * preventing their deletion).
- *
- * <p> In addition, you can call this method to delete
- * unreferenced index commits. This might be useful if you
+ * preventing their deletion).
+ *
+ * <p> In addition, you can call this method to delete
+ * unreferenced index commits. This might be useful if you
* are using an {@link IndexDeletionPolicy} which holds
* onto index commits until some criteria are met, but those
* commits are no longer needed. Otherwise, those commits will
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java Wed Mar 30 09:17:25 2011
@@ -26,13 +26,17 @@ import org.apache.lucene.search.Similari
import org.apache.lucene.util.Version;
/**
- * Holds all the configuration of {@link IndexWriter}. This object is only used
- * while constructing a new IndexWriter. Those settings cannot be changed
- * afterwards, except instantiating a new IndexWriter.
+ * Holds all the configuration of {@link IndexWriter}. You
+ * should instantiate this class, call the setters to set
+ * your configuration, then pass it to {@link IndexWriter}.
+ * Note that {@link IndexWriter} makes a private clone; if
+ * you need to subsequently change settings use {@link
+ * IndexWriter#getConfig}.
+ *
* <p>
* All setter methods return {@link IndexWriterConfig} to allow chaining
- * settings conveniently. Thus someone can do:
- *
+ * settings conveniently, for example:
+ *
* <pre>
* IndexWriterConfig conf = new IndexWriterConfig(analyzer);
* conf.setter1().setter2();
@@ -108,26 +112,25 @@ public final class IndexWriterConfig imp
return WRITE_LOCK_TIMEOUT;
}
- private Analyzer analyzer;
- private IndexDeletionPolicy delPolicy;
- private IndexCommit commit;
- private OpenMode openMode;
- private SimilarityProvider similarityProvider;
- private int termIndexInterval; // TODO: this should be private to the codec, not settable here
- private MergeScheduler mergeScheduler;
- private long writeLockTimeout;
- private int maxBufferedDeleteTerms;
- private double ramBufferSizeMB;
- private int maxBufferedDocs;
- private IndexingChain indexingChain;
- private IndexReaderWarmer mergedSegmentWarmer;
- private CodecProvider codecProvider;
- private MergePolicy mergePolicy;
- private boolean readerPooling;
- private DocumentsWriterPerThreadPool indexerThreadPool;
- private int readerTermsIndexDivisor;
+ private final Analyzer analyzer;
+ private volatile IndexDeletionPolicy delPolicy;
+ private volatile IndexCommit commit;
+ private volatile OpenMode openMode;
+ private volatile SimilarityProvider similarityProvider;
+ private volatile int termIndexInterval; // TODO: this should be private to the codec, not settable here
+ private volatile MergeScheduler mergeScheduler;
+ private volatile long writeLockTimeout;
+ private volatile int maxBufferedDeleteTerms;
+ private volatile double ramBufferSizeMB;
+ private volatile int maxBufferedDocs;
+ private volatile IndexingChain indexingChain;
+ private volatile IndexReaderWarmer mergedSegmentWarmer;
+ private volatile CodecProvider codecProvider;
+ private volatile MergePolicy mergePolicy;
+ private volatile DocumentsWriterPerThreadPool indexerThreadPool;
+ private volatile boolean readerPooling;
+ private volatile int readerTermsIndexDivisor;
- // required for clone
private Version matchVersion;
/**
@@ -162,7 +165,7 @@ public final class IndexWriterConfig imp
@Override
public Object clone() {
// Shallow clone is the only thing that's possible, since parameters like
- // analyzer, index commit etc. do not implemnt Cloneable.
+ // analyzer, index commit etc. do not implement Cloneable.
try {
return super.clone();
} catch (CloneNotSupportedException e) {
@@ -176,7 +179,9 @@ public final class IndexWriterConfig imp
return analyzer;
}
- /** Specifies {@link OpenMode} of that index. */
+ /** Specifies {@link OpenMode} of the index.
+ *
+ * <p>Only takes effect when IndexWriter is first created. */
public IndexWriterConfig setOpenMode(OpenMode openMode) {
this.openMode = openMode;
return this;
@@ -201,6 +206,8 @@ public final class IndexWriterConfig imp
* <p>
* <b>NOTE:</b> the deletion policy cannot be null. If <code>null</code> is
* passed, the deletion policy will be set to the default.
+ *
+ * <p>Only takes effect when IndexWriter is first created.
*/
public IndexWriterConfig setIndexDeletionPolicy(IndexDeletionPolicy delPolicy) {
this.delPolicy = delPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : delPolicy;
@@ -219,7 +226,8 @@ public final class IndexWriterConfig imp
/**
* Expert: allows to open a certain commit point. The default is null which
* opens the latest commit point.
- */
+ *
+ * <p>Only takes effect when IndexWriter is first created. */
public IndexWriterConfig setIndexCommit(IndexCommit commit) {
this.commit = commit;
return this;
@@ -239,7 +247,8 @@ public final class IndexWriterConfig imp
* <p>
* <b>NOTE:</b> the similarity provider cannot be null. If <code>null</code> is passed,
* the similarity provider will be set to the default implementation (unspecified).
- */
+ *
+ * <p>Only takes effect when IndexWriter is first created. */
public IndexWriterConfig setSimilarityProvider(SimilarityProvider similarityProvider) {
this.similarityProvider = similarityProvider == null ? IndexSearcher.getDefaultSimilarityProvider() : similarityProvider;
return this;
@@ -274,7 +283,9 @@ public final class IndexWriterConfig imp
* must be scanned for each random term access.
*
* @see #DEFAULT_TERM_INDEX_INTERVAL
- */
+ *
+ * <p>Takes effect immediately, but only applies to newly
+ * flushed/merged segments. */
public IndexWriterConfig setTermIndexInterval(int interval) { // TODO: this should be private to the codec, not settable here
this.termIndexInterval = interval;
return this;
@@ -295,7 +306,8 @@ public final class IndexWriterConfig imp
* <p>
* <b>NOTE:</b> the merge scheduler cannot be null. If <code>null</code> is
* passed, the merge scheduler will be set to the default.
- */
+ *
+ * <p>Only takes effect when IndexWriter is first created. */
public IndexWriterConfig setMergeScheduler(MergeScheduler mergeScheduler) {
this.mergeScheduler = mergeScheduler == null ? new ConcurrentMergeScheduler() : mergeScheduler;
return this;
@@ -313,7 +325,8 @@ public final class IndexWriterConfig imp
* Sets the maximum time to wait for a write lock (in milliseconds) for this
* instance. You can change the default value for all instances by calling
* {@link #setDefaultWriteLockTimeout(long)}.
- */
+ *
+ * <p>Only takes effect when IndexWriter is first created. */
public IndexWriterConfig setWriteLockTimeout(long writeLockTimeout) {
this.writeLockTimeout = writeLockTimeout;
return this;
@@ -339,6 +352,9 @@ public final class IndexWriterConfig imp
* @throws IllegalArgumentException if maxBufferedDeleteTerms
* is enabled but smaller than 1
* @see #setRAMBufferSizeMB
+ *
+ * <p>Takes effect immediately, but only the next time a
+ * document is added, updated or deleted.
*/
public IndexWriterConfig setMaxBufferedDeleteTerms(int maxBufferedDeleteTerms) {
if (maxBufferedDeleteTerms != DISABLE_AUTO_FLUSH
@@ -391,6 +407,9 @@ public final class IndexWriterConfig imp
* <p>
* The default value is {@link #DEFAULT_RAM_BUFFER_SIZE_MB}.
*
+ * <p>Takes effect immediately, but only the next time a
+ * document is added, updated or deleted.
+ *
* @throws IllegalArgumentException
* if ramBufferSize is enabled but non-positive, or it disables
* ramBufferSize when maxBufferedDocs is already disabled
@@ -430,6 +449,9 @@ public final class IndexWriterConfig imp
* <p>
* Disabled by default (writer flushes by RAM usage).
*
+ * <p>Takes effect immediately, but only the next time a
+ * document is added, updated or deleted.
+ *
* @see #setRAMBufferSizeMB(double)
*
* @throws IllegalArgumentException
@@ -458,7 +480,9 @@ public final class IndexWriterConfig imp
return maxBufferedDocs;
}
- /** Set the merged segment warmer. See {@link IndexReaderWarmer}. */
+ /** Set the merged segment warmer. See {@link IndexReaderWarmer}.
+ *
+ * <p>Takes effect on the next merge. */
public IndexWriterConfig setMergedSegmentWarmer(IndexReaderWarmer mergeSegmentWarmer) {
this.mergedSegmentWarmer = mergeSegmentWarmer;
return this;
@@ -475,13 +499,16 @@ public final class IndexWriterConfig imp
* and return a {@link MergePolicy.MergeSpecification} describing the merges.
* It also selects merges to do for optimize(). (The default is
* {@link LogByteSizeMergePolicy}.
- */
+ *
+ * <p>Only takes effect when IndexWriter is first created. */
public IndexWriterConfig setMergePolicy(MergePolicy mergePolicy) {
this.mergePolicy = mergePolicy == null ? new LogByteSizeMergePolicy() : mergePolicy;
return this;
}
- /** Set the CodecProvider. See {@link CodecProvider}. */
+ /** Set the CodecProvider. See {@link CodecProvider}.
+ *
+ * <p>Only takes effect when IndexWriter is first created. */
public IndexWriterConfig setCodecProvider(CodecProvider codecProvider) {
this.codecProvider = codecProvider;
return this;
@@ -507,7 +534,8 @@ public final class IndexWriterConfig imp
* at once in IndexWriter. Values < 1 are invalid and if passed
* <code>maxThreadStates</code> will be set to
* {@link #DEFAULT_MAX_THREAD_STATES}.
- */
+ *
+ * <p>Only takes effect when IndexWriter is first created. */
public IndexWriterConfig setIndexerThreadPool(DocumentsWriterPerThreadPool threadPool) {
this.indexerThreadPool = threadPool;
return this;
@@ -530,7 +558,9 @@ public final class IndexWriterConfig imp
* This method lets you enable pooling without getting a
* near-real-time reader. NOTE: if you set this to
* false, IndexWriter will still pool readers once
- * {@link IndexWriter#getReader} is called. */
+ * {@link IndexWriter#getReader} is called.
+ *
+ * <p>Only takes effect when IndexWriter is first created. */
public IndexWriterConfig setReaderPooling(boolean readerPooling) {
this.readerPooling = readerPooling;
return this;
@@ -542,7 +572,9 @@ public final class IndexWriterConfig imp
return readerPooling;
}
- /** Expert: sets the {@link DocConsumer} chain to be used to process documents. */
+ /** Expert: sets the {@link DocConsumer} chain to be used to process documents.
+ *
+ * <p>Only takes effect when IndexWriter is first created. */
IndexWriterConfig setIndexingChain(IndexingChain indexingChain) {
this.indexingChain = indexingChain == null ? DocumentsWriterPerThread.defaultIndexingChain : indexingChain;
return this;
@@ -559,7 +591,10 @@ public final class IndexWriterConfig imp
* IndexWriter#getReader}. If you pass -1, the terms index
* won't be loaded by the readers. This is only useful in
* advanced situations when you will only .next() through
- * all terms; attempts to seek will hit an exception. */
+ * all terms; attempts to seek will hit an exception.
+ *
+ * <p>Takes effect immediately, but only applies to
+ * readers opened after this call */
public IndexWriterConfig setReaderTermsIndexDivisor(int divisor) {
if (divisor <= 0 && divisor != -1) {
throw new IllegalArgumentException("divisor must be >= 1, or -1 (got " + divisor + ")");
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java Wed Mar 30 09:17:25 2011
@@ -129,7 +129,7 @@ public final class MultiFieldsEnum exte
private final static class FieldMergeQueue extends PriorityQueue<FieldsEnumWithSlice> {
FieldMergeQueue(int size) {
- initialize(size);
+ super(size);
}
@Override
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java Wed Mar 30 09:17:25 2011
@@ -427,7 +427,7 @@ public final class MultiTermsEnum extend
private final static class TermMergeQueue extends PriorityQueue<TermsEnumWithSlice> {
Comparator<BytesRef> termComp;
TermMergeQueue(int size) {
- initialize(size);
+ super(size);
}
@Override
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/NormsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/NormsWriter.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/NormsWriter.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/NormsWriter.java Wed Mar 30 09:17:25 2011
@@ -56,8 +56,7 @@ final class NormsWriter extends Inverted
int normCount = 0;
for (FieldInfo fi : state.fieldInfos) {
- NormsWriterPerField toWrite = (NormsWriterPerField) fieldsToFlush.get(fi);
-
+ final NormsWriterPerField toWrite = (NormsWriterPerField) fieldsToFlush.get(fi);
int upto = 0;
if (toWrite != null && toWrite.upto > 0) {
normCount++;
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java Wed Mar 30 09:17:25 2011
@@ -67,6 +67,7 @@ final class PerFieldCodecWrapper extends
@Override
public TermsConsumer addField(FieldInfo field) throws IOException {
+ assert field.getCodecId() != FieldInfo.UNASSIGNED_CODEC_ID;
final FieldsConsumer fields = consumers.get(field.getCodecId());
return fields.addField(field);
}
@@ -106,6 +107,7 @@ final class PerFieldCodecWrapper extends
for (FieldInfo fi : fieldInfos) {
if (fi.isIndexed) { // TODO this does not work for non-indexed fields
fields.add(fi.name);
+ assert fi.getCodecId() != FieldInfo.UNASSIGNED_CODEC_ID;
Codec codec = segmentCodecs.codecs[fi.getCodecId()];
if (!producers.containsKey(codec)) {
producers.put(codec, codec.fieldsProducer(new SegmentReadState(dir,
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java Wed Mar 30 09:17:25 2011
@@ -38,17 +38,16 @@ import org.apache.lucene.store.IndexOutp
* {@link SegmentWriteState} for each flush and is maintained in the
* corresponding {@link SegmentInfo} until it is committed.
* <p>
- * {@link SegmentCodecs#build(FieldInfos, CodecProvider)} should be used to
- * create a {@link SegmentCodecs} instance during {@link IndexWriter} sessions
- * which creates the ordering of distinct codecs and assigns the
- * {@link FieldInfo#codecId} or in other words, the ord of the codec maintained
- * inside {@link SegmentCodecs}, to the {@link FieldInfo}. This ord is valid
- * only until the current segment is flushed and {@link FieldInfos} for that
- * segment are written including the ord for each field. This ord is later used
- * to get the right codec when the segment is opened in a reader. The
- * {@link Codec} returned from {@link SegmentCodecs#codec()} in turn uses
- * {@link SegmentCodecs} internal structure to select and initialize the right
- * codec for a fields when it is written.
+ * During indexing {@link FieldInfos} uses {@link SegmentCodecsBuilder} to incrementally
+ * build the {@link SegmentCodecs} mapping. Once a segment is flushed
+ * DocumentsWriter creates a {@link SegmentCodecs} instance from
+ * {@link FieldInfos#buildSegmentCodecs(boolean)} The {@link FieldInfo#codecId}
+ * assigned by {@link SegmentCodecsBuilder} refers to the codecs ordinal
+ * maintained inside {@link SegmentCodecs}. This ord is later used to get the
+ * right codec when the segment is opened in a reader.The {@link Codec} returned
+ * from {@link SegmentCodecs#codec()} in turn uses {@link SegmentCodecs}
+ * internal structure to select and initialize the right codec for a fields when
+ * it is written.
* <p>
* Once a flush succeeded the {@link SegmentCodecs} is maintained inside the
* {@link SegmentInfo} for the flushed segment it was created for.
@@ -64,36 +63,19 @@ final class SegmentCodecs implements Clo
* internal structure to map codecs to fields - don't modify this from outside
* of this class!
*/
- Codec[] codecs;
+ final Codec[] codecs;
final CodecProvider provider;
private final Codec codec = new PerFieldCodecWrapper(this);
-
+
+ SegmentCodecs(CodecProvider provider, IndexInput input) throws IOException {
+ this(provider, read(input, provider));
+ }
+
SegmentCodecs(CodecProvider provider, Codec... codecs) {
this.provider = provider;
this.codecs = codecs;
}
- static SegmentCodecs build(FieldInfos infos, CodecProvider provider) {
- final Map<Codec, Integer> codecRegistry = new IdentityHashMap<Codec, Integer>();
- final ArrayList<Codec> codecs = new ArrayList<Codec>();
-
- for (FieldInfo fi : infos) {
- if (fi.isIndexed) {
- final Codec fieldCodec = provider.lookup(provider
- .getFieldCodec(fi.name));
- Integer ord = codecRegistry.get(fieldCodec);
- if (ord == null) {
- ord = Integer.valueOf(codecs.size());
- codecRegistry.put(fieldCodec, ord);
- codecs.add(fieldCodec);
- }
- fi.setCodecId(ord.intValue());
- }
- }
- return new SegmentCodecs(provider, codecs.toArray(Codec.EMPTY));
-
- }
-
Codec codec() {
return codec;
}
@@ -105,7 +87,7 @@ final class SegmentCodecs implements Clo
}
}
- void read(IndexInput in) throws IOException {
+ private static Codec[] read(IndexInput in, CodecProvider provider) throws IOException {
final int size = in.readVInt();
final ArrayList<Codec> list = new ArrayList<Codec>();
for (int i = 0; i < size; i++) {
@@ -113,7 +95,7 @@ final class SegmentCodecs implements Clo
final Codec lookup = provider.lookup(codecName);
list.add(i, lookup);
}
- codecs = list.toArray(Codec.EMPTY);
+ return list.toArray(Codec.EMPTY);
}
void files(Directory dir, SegmentInfo info, Set<String> files)
@@ -129,4 +111,58 @@ final class SegmentCodecs implements Clo
public String toString() {
return "SegmentCodecs [codecs=" + Arrays.toString(codecs) + ", provider=" + provider + "]";
}
+
+ /**
+ * Used in {@link FieldInfos} to incrementally build the codec ID mapping for
+ * {@link FieldInfo} instances.
+ * <p>
+ * Note: this class is not thread-safe
+ * </p>
+ * @see FieldInfo#getCodecId()
+ */
+ final static class SegmentCodecsBuilder {
+ private final Map<Codec, Integer> codecRegistry = new IdentityHashMap<Codec, Integer>();
+ private final ArrayList<Codec> codecs = new ArrayList<Codec>();
+ private final CodecProvider provider;
+
+ private SegmentCodecsBuilder(CodecProvider provider) {
+ this.provider = provider;
+ }
+
+ static SegmentCodecsBuilder create(CodecProvider provider) {
+ return new SegmentCodecsBuilder(provider);
+ }
+
+ SegmentCodecsBuilder tryAddAndSet(FieldInfo fi) {
+ if (fi.getCodecId() == FieldInfo.UNASSIGNED_CODEC_ID) {
+ final Codec fieldCodec = provider.lookup(provider
+ .getFieldCodec(fi.name));
+ Integer ord = codecRegistry.get(fieldCodec);
+ if (ord == null) {
+ ord = Integer.valueOf(codecs.size());
+ codecRegistry.put(fieldCodec, ord);
+ codecs.add(fieldCodec);
+ }
+ fi.setCodecId(ord.intValue());
+ }
+ return this;
+ }
+
+ SegmentCodecsBuilder addAll(FieldInfos infos) {
+ for (FieldInfo fieldInfo : infos) {
+ tryAddAndSet(fieldInfo);
+ }
+ return this;
+ }
+
+ SegmentCodecs build() {
+ return new SegmentCodecs(provider, codecs.toArray(Codec.EMPTY));
+ }
+
+ SegmentCodecsBuilder clear() {
+ codecRegistry.clear();
+ codecs.clear();
+ return this;
+ }
+ }
}
\ No newline at end of file
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Wed Mar 30 09:17:25 2011
@@ -42,9 +42,6 @@ import org.apache.lucene.util.Constants;
*/
public final class SegmentInfo {
- @Deprecated
- // remove with hasVector and hasProx
- static final int CHECK_FIELDINFOS = -2; // hasVector and hasProx use this for bw compatibility
static final int NO = -1; // e.g. no norms; no deletes;
static final int YES = 1; // e.g. have norms; have deletes;
static final int WITHOUT_GEN = 0; // a file name that has no GEN in it.
@@ -84,14 +81,10 @@ public final class SegmentInfo {
private int delCount; // How many deleted docs in this segment
- @Deprecated
- // remove when we don't have to support old indexes anymore that had this field
- private int hasProx = CHECK_FIELDINFOS; // True if this segment has any fields with omitTermFreqAndPositions==false
-
- @Deprecated
- // remove when we don't have to support old indexes anymore that had this field
- private int hasVectors = CHECK_FIELDINFOS; // True if this segment wrote term vectors
+ private boolean hasProx; // True if this segment has any fields with omitTermFreqAndPositions==false
+ private boolean hasVectors; // True if this segment wrote term vectors
+
private FieldInfos fieldInfos;
private SegmentCodecs segmentCodecs;
@@ -110,7 +103,7 @@ public final class SegmentInfo {
private long bufferedDeletesGen;
public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile,
- SegmentCodecs segmentCodecs, FieldInfos fieldInfos) {
+ boolean hasProx, SegmentCodecs segmentCodecs, boolean hasVectors, FieldInfos fieldInfos) {
this.name = name;
this.docCount = docCount;
this.dir = dir;
@@ -118,7 +111,9 @@ public final class SegmentInfo {
this.isCompoundFile = isCompoundFile;
this.docStoreOffset = -1;
this.docStoreSegment = name;
+ this.hasProx = hasProx;
this.segmentCodecs = segmentCodecs;
+ this.hasVectors = hasVectors;
delCount = 0;
version = Constants.LUCENE_MAIN_VERSION;
this.fieldInfos = fieldInfos;
@@ -146,7 +141,7 @@ public final class SegmentInfo {
normGen = new HashMap<Integer, Long>(src.normGen.size());
for (Entry<Integer,Long> entry : src.normGen.entrySet()) {
normGen.put(entry.getKey(), entry.getValue());
- }
+ }
}
isCompoundFile = src.isCompoundFile;
delCount = src.delCount;
@@ -203,44 +198,30 @@ public final class SegmentInfo {
int fieldNumber = j;
if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) {
fieldNumber = input.readInt();
- }
+ }
normGen.put(fieldNumber, input.readLong());
- }
- }
- isCompoundFile = input.readByte() == YES;
-
- Directory dir0 = dir;
- if (isCompoundFile) {
- dir0 = new CompoundFileReader(dir, IndexFileNames.segmentFileName(name, "", IndexFileNames.COMPOUND_FILE_EXTENSION));
- }
-
- try {
- fieldInfos = new FieldInfos(dir0, IndexFileNames.segmentFileName(name, "", IndexFileNames.FIELD_INFOS_EXTENSION));
- } finally {
- if (dir != dir0) {
- dir0.close();
}
}
+ isCompoundFile = input.readByte() == YES;
delCount = input.readInt();
assert delCount <= docCount;
- hasProx = input.readByte();
+ hasProx = input.readByte() == YES;
// System.out.println(Thread.currentThread().getName() + ": si.read hasProx=" + hasProx + " seg=" + name);
- segmentCodecs = new SegmentCodecs(codecs);
if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) {
- segmentCodecs.read(input);
+ segmentCodecs = new SegmentCodecs(codecs, input);
} else {
// codec ID on FieldInfo is 0 so it will simply use the first codec available
// TODO what todo if preflex is not available in the provider? register it or fail?
- segmentCodecs.codecs = new Codec[] { codecs.lookup("PreFlex")};
+ segmentCodecs = new SegmentCodecs(codecs, new Codec[] { codecs.lookup("PreFlex")});
}
diagnostics = input.readStringStringMap();
if (format <= DefaultSegmentInfosWriter.FORMAT_HAS_VECTORS) {
- hasVectors = input.readByte();
+ hasVectors = input.readByte() == 1;
} else {
final String storesSegment;
final String ext;
@@ -261,11 +242,7 @@ public final class SegmentInfo {
dirToTest = dir;
}
try {
- if (dirToTest.fileExists(IndexFileNames.segmentFileName(storesSegment, "", IndexFileNames.VECTORS_INDEX_EXTENSION))) {
- hasVectors = YES;
- } else {
- hasVectors = NO;
- }
+ hasVectors = dirToTest.fileExists(IndexFileNames.segmentFileName(storesSegment, "", IndexFileNames.VECTORS_INDEX_EXTENSION));
} finally {
if (isCompoundFile) {
dirToTest.close();
@@ -273,6 +250,24 @@ public final class SegmentInfo {
}
}
}
+
+ synchronized void loadFieldInfos(Directory dir, boolean checkCompoundFile) throws IOException {
+ if (fieldInfos == null) {
+ Directory dir0 = dir;
+ if (isCompoundFile && checkCompoundFile) {
+ dir0 = new CompoundFileReader(dir, IndexFileNames.segmentFileName(name,
+ "", IndexFileNames.COMPOUND_FILE_EXTENSION));
+ }
+ try {
+ fieldInfos = new FieldInfos(dir0, IndexFileNames.segmentFileName(name,
+ "", IndexFileNames.FIELD_INFOS_EXTENSION));
+ } finally {
+ if (dir != dir0) {
+ dir0.close();
+ }
+ }
+ }
+ }
/**
* Returns total size in bytes of all of files used by this segment (if
@@ -310,12 +305,17 @@ public final class SegmentInfo {
}
}
- public boolean getHasVectors() {
- return hasVectors == CHECK_FIELDINFOS ?
- (fieldInfos == null ? true : fieldInfos.hasVectors()) : hasVectors == YES;
+ public boolean getHasVectors() throws IOException {
+ return hasVectors;
+ }
+
+ public void setHasVectors(boolean v) {
+ hasVectors = v;
+ clearFilesCache();
}
- public FieldInfos getFieldInfos() {
+ public FieldInfos getFieldInfos() throws IOException {
+ loadFieldInfos(dir, true);
return fieldInfos;
}
@@ -344,7 +344,7 @@ public final class SegmentInfo {
@Override
public Object clone() {
- SegmentInfo si = new SegmentInfo(name, docCount, dir, isCompoundFile, segmentCodecs,
+ final SegmentInfo si = new SegmentInfo(name, docCount, dir, isCompoundFile, hasProx, segmentCodecs, hasVectors,
fieldInfos == null ? null : (FieldInfos) fieldInfos.clone());
si.docStoreOffset = docStoreOffset;
si.docStoreSegment = docStoreSegment;
@@ -356,10 +356,8 @@ public final class SegmentInfo {
si.normGen = new HashMap<Integer, Long>();
for (Entry<Integer,Long> entry : normGen.entrySet()) {
si.normGen.put(entry.getKey(), entry.getValue());
+ }
}
- }
- si.hasProx = hasProx;
- si.hasVectors = hasVectors;
si.version = version;
return si;
}
@@ -382,7 +380,7 @@ public final class SegmentInfo {
public boolean hasSeparateNorms(int fieldNumber) {
if (normGen == null) {
return false;
- }
+ }
Long gen = normGen.get(fieldNumber);
return gen != null && gen.longValue() != NO;
@@ -538,15 +536,19 @@ public final class SegmentInfo {
output.writeByte((byte) (isCompoundFile ? YES : NO));
output.writeInt(delCount);
- output.writeByte((byte) hasProx);
+ output.writeByte((byte) (hasProx ? 1:0));
segmentCodecs.write(output);
output.writeStringStringMap(diagnostics);
- output.writeByte((byte) hasVectors);
+ output.writeByte((byte) (hasVectors ? 1 : 0));
+ }
+
+ void setHasProx(boolean hasProx) {
+ this.hasProx = hasProx;
+ clearFilesCache();
}
public boolean getHasProx() {
- return hasProx == CHECK_FIELDINFOS ?
- (fieldInfos == null ? true : fieldInfos.hasProx()) : hasProx == YES;
+ return hasProx;
}
/** Can only be called once. */
@@ -602,7 +604,7 @@ public final class SegmentInfo {
} else {
fileSet.add(IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.FIELDS_INDEX_EXTENSION));
fileSet.add(IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.FIELDS_EXTENSION));
- if (getHasVectors()) {
+ if (hasVectors) {
fileSet.add(IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.VECTORS_INDEX_EXTENSION));
fileSet.add(IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION));
fileSet.add(IndexFileNames.segmentFileName(docStoreSegment, "", IndexFileNames.VECTORS_FIELDS_EXTENSION));
@@ -611,7 +613,7 @@ public final class SegmentInfo {
} else if (!useCompoundFile) {
fileSet.add(IndexFileNames.segmentFileName(name, "", IndexFileNames.FIELDS_INDEX_EXTENSION));
fileSet.add(IndexFileNames.segmentFileName(name, "", IndexFileNames.FIELDS_EXTENSION));
- if (getHasVectors()) {
+ if (hasVectors) {
fileSet.add(IndexFileNames.segmentFileName(name, "", IndexFileNames.VECTORS_INDEX_EXTENSION));
fileSet.add(IndexFileNames.segmentFileName(name, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION));
fileSet.add(IndexFileNames.segmentFileName(name, "", IndexFileNames.VECTORS_FIELDS_EXTENSION));
@@ -640,7 +642,7 @@ public final class SegmentInfo {
/* Called whenever any change is made that affects which
* files this segment has. */
- void clearFilesCache() {
+ private void clearFilesCache() {
files = null;
sizeInBytesNoStore = -1;
sizeInBytesWithStore = -1;
@@ -675,7 +677,7 @@ public final class SegmentInfo {
if (this.dir != dir) {
s.append('x');
}
- if (getHasVectors()) {
+ if (hasVectors) {
s.append('v');
}
s.append(docCount);
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentInfos.java Wed Mar 30 09:17:25 2011
@@ -28,7 +28,9 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Vector;
+import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter;
import org.apache.lucene.index.codecs.SegmentInfosReader;
import org.apache.lucene.index.codecs.SegmentInfosWriter;
import org.apache.lucene.store.Directory;
@@ -64,6 +66,11 @@ public final class SegmentInfos extends
* starting with the current time in milliseconds forces to create unique version numbers.
*/
public long version = System.currentTimeMillis();
+
+ private long globalFieldMapVersion = 0; // version of the GFNM for the next commit
+ private long lastGlobalFieldMapVersion = 0; // version of the GFNM file we last successfully read or wrote
+ private long pendingMapVersion = -1; // version of the GFNM itself that we have last successfully written
+ // or -1 if we it was not written. This is set during prepareCommit
private long generation = 0; // generation of the "segments_N" for the next commit
private long lastGeneration = 0; // generation of the "segments_N" file we last successfully read
@@ -75,6 +82,8 @@ public final class SegmentInfos extends
private CodecProvider codecs;
private int format;
+
+ private FieldNumberBiMap globalFieldNumberMap; // this segments global field number map - lazy loaded on demand
/**
* If non-null, information about loading segments_N files
@@ -171,6 +180,15 @@ public final class SegmentInfos extends
"",
lastGeneration);
}
+
+ private String getGlobalFieldNumberName(long version) {
+ /*
+ * This creates a file name ${version}.fnx without a leading underscore
+ * since this file might belong to more than one segment (global map) and
+ * could otherwise easily be confused with a per-segment file.
+ */
+ return IndexFileNames.segmentFileName(""+ version, "", IndexFileNames.GLOBAL_FIELD_NUM_MAP_EXTENSION);
+ }
/**
* Parse the generation off the segments file name and
@@ -261,6 +279,8 @@ public final class SegmentInfos extends
return null;
}
}.run();
+ // either we are on 4.0 or we don't have a lastGlobalFieldMapVersion i.e. its still set to 0
+ assert DefaultSegmentInfosWriter.FORMAT_4_0 <= format || (DefaultSegmentInfosWriter.FORMAT_4_0 > format && lastGlobalFieldMapVersion == 0);
}
// Only non-null after prepareCommit has been called and
@@ -270,15 +290,24 @@ public final class SegmentInfos extends
private void write(Directory directory) throws IOException {
String segmentFileName = getNextSegmentFileName();
-
+ final String globalFieldMapFile;
+ if (globalFieldNumberMap != null && globalFieldNumberMap.isDirty()) {
+ globalFieldMapFile = getGlobalFieldNumberName(++globalFieldMapVersion);
+ pendingMapVersion = writeGlobalFieldMap(globalFieldNumberMap, directory, globalFieldMapFile);
+ } else {
+ globalFieldMapFile = null;
+ }
+
+
// Always advance the generation on write:
if (generation == -1) {
generation = 1;
} else {
generation++;
}
-
+
IndexOutput segnOutput = null;
+
boolean success = false;
@@ -304,6 +333,16 @@ public final class SegmentInfos extends
} catch (Throwable t) {
// Suppress so we keep throwing the original exception
}
+ if (globalFieldMapFile != null) { // delete if written here
+ try {
+ // Try not to leave global field map in
+ // the index:
+ directory.deleteFile(globalFieldMapFile);
+ } catch (Throwable t) {
+ // Suppress so we keep throwing the original exception
+ }
+ }
+ pendingMapVersion = -1;
}
}
}
@@ -719,6 +758,8 @@ public final class SegmentInfos extends
void updateGeneration(SegmentInfos other) {
lastGeneration = other.lastGeneration;
generation = other.generation;
+ lastGlobalFieldMapVersion = other.lastGlobalFieldMapVersion;
+ globalFieldMapVersion = other.globalFieldMapVersion;
}
final void rollbackCommit(Directory dir) throws IOException {
@@ -742,6 +783,16 @@ public final class SegmentInfos extends
// in our caller
}
pendingSegnOutput = null;
+ if (pendingMapVersion != -1) {
+ try {
+ final String fieldMapName = getGlobalFieldNumberName(globalFieldMapVersion--);
+ dir.deleteFile(fieldMapName);
+ } catch (Throwable t) {
+ // Suppress so we keep throwing the original exception
+ // in our caller
+ }
+ pendingMapVersion = -1;
+ }
}
}
@@ -760,6 +811,40 @@ public final class SegmentInfos extends
throw new IllegalStateException("prepareCommit was already called");
write(dir);
}
+
+ private final long writeGlobalFieldMap(FieldNumberBiMap map, Directory dir, String name) throws IOException {
+ final IndexOutput output = dir.createOutput(name);
+ boolean success = false;
+ long version;
+ try {
+ version = map.write(output);
+ success = true;
+ } finally {
+ try {
+ output.close();
+ } catch (Throwable t) {
+ // throw orig excp
+ }
+ if (!success) {
+ try {
+ dir.deleteFile(name);
+ } catch (Throwable t) {
+ // throw orig excp
+ }
+ }
+ }
+ return version;
+ }
+
+ private void readGlobalFieldMap(FieldNumberBiMap map, Directory dir) throws IOException {
+ final String name = getGlobalFieldNumberName(lastGlobalFieldMapVersion);
+ final IndexInput input = dir.openInput(name);
+ try {
+ map.read(input);
+ } finally {
+ input.close();
+ }
+ }
/** Returns all file names referenced by SegmentInfo
* instances matching the provided Directory (ie files
@@ -769,7 +854,17 @@ public final class SegmentInfos extends
public Collection<String> files(Directory dir, boolean includeSegmentsFile) throws IOException {
HashSet<String> files = new HashSet<String>();
if (includeSegmentsFile) {
- files.add(getCurrentSegmentFileName());
+ final String segmentFileName = getCurrentSegmentFileName();
+ if (segmentFileName != null) {
+ /*
+ * TODO: if lastGen == -1 we get might get null here it seems wrong to
+ * add null to the files set
+ */
+ files.add(segmentFileName);
+ }
+ if (lastGlobalFieldMapVersion > 0) {
+ files.add(getGlobalFieldNumberName(lastGlobalFieldMapVersion));
+ }
}
final int size = size();
for(int i=0;i<size;i++) {
@@ -821,6 +916,17 @@ public final class SegmentInfos extends
}
lastGeneration = generation;
+ if (pendingMapVersion != -1) {
+ /*
+ * TODO is it possible that the commit does not succeed here? if another
+ * commit happens at the same time and we lost the race between the
+ * prepareCommit and finishCommit the latest version is already
+ * incremented.
+ */
+ globalFieldNumberMap.commitLastVersion(pendingMapVersion);
+ pendingMapVersion = -1;
+ lastGlobalFieldMapVersion = globalFieldMapVersion;
+ }
try {
IndexOutput genOutput = dir.createOutput(IndexFileNames.SEGMENTS_GEN);
@@ -848,6 +954,7 @@ public final class SegmentInfos extends
prepareCommit(dir);
finishCommit(dir);
}
+
public synchronized String toString(Directory directory) {
StringBuilder buffer = new StringBuilder();
@@ -883,6 +990,8 @@ public final class SegmentInfos extends
clear();
addAll(other);
lastGeneration = other.lastGeneration;
+ lastGlobalFieldMapVersion = other.lastGlobalFieldMapVersion;
+ format = other.format;
}
/** Returns sum of all segment's docCounts. Note that
@@ -900,4 +1009,49 @@ public final class SegmentInfos extends
public void changed() {
version++;
}
+
+ /**
+ * Loads or returns the already loaded the global field number map for this {@link SegmentInfos}.
+ * If this {@link SegmentInfos} has no global field number map the returned instance is empty
+ */
+ synchronized FieldNumberBiMap getOrLoadGlobalFieldNumberMap(Directory dir) throws IOException {
+ if (globalFieldNumberMap != null) {
+ return globalFieldNumberMap;
+ }
+ final FieldNumberBiMap map = new FieldNumberBiMap();
+
+ if (lastGlobalFieldMapVersion > 0) {
+ // if we don't have a global map or this is a SI from a earlier version we just return the empty map;
+ readGlobalFieldMap(map, dir);
+ }
+ if (size() > 0) {
+ if (format > DefaultSegmentInfosWriter.FORMAT_4_0) {
+ assert lastGlobalFieldMapVersion == 0;
+ // build the map up if we open a pre 4.0 index
+ for (SegmentInfo info : this) {
+ final FieldInfos segFieldInfos = info.getFieldInfos();
+ for (FieldInfo fi : segFieldInfos) {
+ map.addOrGet(fi.name, fi.number);
+ }
+ }
+ }
+ }
+ return globalFieldNumberMap = map;
+ }
+
+ /**
+ * Called by {@link SegmentInfosReader} when reading the global field map version
+ */
+ public void setGlobalFieldMapVersion(long version) {
+ lastGlobalFieldMapVersion = globalFieldMapVersion = version;
+ }
+
+ public long getGlobalFieldMapVersion() {
+ return globalFieldMapVersion;
+ }
+
+ // for testing
+ long getLastGlobalFieldMapVersion() {
+ return lastGlobalFieldMapVersion;
+ }
}
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Wed Mar 30 09:17:25 2011
@@ -148,7 +148,7 @@ final class SegmentMerger {
boolean storePayloads, boolean omitTFAndPositions)
throws IOException {
for (String field : names) {
- fInfos.add(field, true, storeTermVectors,
+ fInfos.addOrUpdate(field, true, storeTermVectors,
storePositionWithTermVector, storeOffsetWithTermVector, !reader
.hasNorms(field), storePayloads, omitTFAndPositions);
}
@@ -217,10 +217,10 @@ final class SegmentMerger {
addIndexed(reader, fieldInfos, reader.getFieldNames(FieldOption.OMIT_TERM_FREQ_AND_POSITIONS), false, false, false, false, true);
addIndexed(reader, fieldInfos, reader.getFieldNames(FieldOption.STORES_PAYLOADS), false, false, false, true, false);
addIndexed(reader, fieldInfos, reader.getFieldNames(FieldOption.INDEXED), false, false, false, false, false);
- fieldInfos.add(reader.getFieldNames(FieldOption.UNINDEXED), false);
+ fieldInfos.addOrUpdate(reader.getFieldNames(FieldOption.UNINDEXED), false);
}
}
- final SegmentCodecs codecInfo = SegmentCodecs.build(fieldInfos, this.codecs);
+ final SegmentCodecs codecInfo = fieldInfos.buildSegmentCodecs(false);
fieldInfos.write(directory, segment + "." + IndexFileNames.FIELD_INFOS_EXTENSION);
int docCount = 0;
@@ -568,6 +568,11 @@ final class SegmentMerger {
return mergeState.delCounts;
}
+ public boolean getAnyNonBulkMerges() {
+ assert matchedCount <= readers.size();
+ return matchedCount != readers.size();
+ }
+
private void mergeNorms() throws IOException {
IndexOutput output = null;
try {
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentReader.java Wed Mar 30 09:17:25 2011
@@ -118,7 +118,7 @@ public class SegmentReader extends Index
dir0 = cfsReader;
}
cfsDir = dir0;
-
+ si.loadFieldInfos(cfsDir, false); // prevent opening the CFS to load fieldInfos
fieldInfos = si.getFieldInfos();
this.termsIndexDivisor = termsIndexDivisor;
@@ -700,7 +700,7 @@ public class SegmentReader extends Index
}
}
- private void commitChanges(Map<String,String> commitUserData) throws IOException {
+ private synchronized void commitChanges(Map<String,String> commitUserData) throws IOException {
if (deletedDocsDirty) { // re-write deleted
si.advanceDelGen();
@@ -920,26 +920,21 @@ public class SegmentReader extends Index
return fieldSet;
}
-
@Override
- public synchronized boolean hasNorms(String field) {
+ public boolean hasNorms(String field) {
ensureOpen();
return norms.containsKey(field);
}
- // can return null if norms aren't stored
- protected synchronized byte[] getNorms(String field) throws IOException {
- Norm norm = norms.get(field);
- if (norm == null) return null; // not indexed, or norms not stored
- return norm.bytes();
- }
-
- // returns fake norms if norms aren't available
@Override
- public synchronized byte[] norms(String field) throws IOException {
+ public byte[] norms(String field) throws IOException {
ensureOpen();
- byte[] bytes = getNorms(field);
- return bytes;
+ final Norm norm = norms.get(field);
+ if (norm == null) {
+ // not indexed, or norms not stored
+ return null;
+ }
+ return norm.bytes();
}
@Override
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/TermsHashPerField.java Wed Mar 30 09:17:25 2011
@@ -40,6 +40,7 @@ final class TermsHashPerField extends In
final DocumentsWriterPerThread.DocState docState;
final FieldInvertState fieldState;
TermToBytesRefAttribute termAtt;
+ BytesRef termBytesRef;
// Copied from our perThread
final IntBlockPool intPool;
@@ -54,7 +55,6 @@ final class TermsHashPerField extends In
final BytesRefHash bytesHash;
ParallelPostingsArray postingsArray;
- private final BytesRef termBytesRef;
private final AtomicLong bytesUsed;
public TermsHashPerField(DocInverterPerField docInverterPerField, final TermsHash termsHash, final TermsHash nextTermsHash, final FieldInfo fieldInfo) {
@@ -71,8 +71,6 @@ final class TermsHashPerField extends In
bytesHash = new BytesRefHash(termBytePool, HASH_INIT_SIZE, byteStarts);
streamCount = consumer.getStreamCount();
numPostingInt = 2*streamCount;
-
- termBytesRef = termsHash.termBytesRef;
this.fieldInfo = fieldInfo;
if (nextTermsHash != null)
nextPerField = (TermsHashPerField) nextTermsHash.addField(docInverterPerField, fieldInfo);
@@ -120,6 +118,7 @@ final class TermsHashPerField extends In
@Override
void start(Fieldable f) {
termAtt = fieldState.attributeSource.getAttribute(TermToBytesRefAttribute.class);
+ termBytesRef = termAtt.getBytesRef();
consumer.start(f);
if (nextPerField != null) {
nextPerField.start(f);
@@ -181,7 +180,7 @@ final class TermsHashPerField extends In
// Get the text & hash of this term.
int termID;
try{
- termID = bytesHash.add(termBytesRef, termAtt.toBytesRef(termBytesRef));
+ termID = bytesHash.add(termBytesRef, termAtt.fillBytesRef());
}catch (MaxBytesLengthExceededException e) {
// Not enough room in current block
// Just skip this term, to remain as robust as
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java Wed Mar 30 09:17:25 2011
@@ -56,7 +56,9 @@ public class DefaultSegmentInfosReader e
infos.version = input.readLong(); // read version
infos.counter = input.readInt(); // read counter
-
+ if (infos.getFormat() <= DefaultSegmentInfosWriter.FORMAT_4_0) {
+ infos.setGlobalFieldMapVersion(input.readLong());
+ }
for (int i = input.readInt(); i > 0; i--) { // read segmentInfos
SegmentInfo si = new SegmentInfo(directory, format, input, codecs);
if (si.getVersion() == null) {
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java Wed Mar 30 09:17:25 2011
@@ -59,6 +59,7 @@ public class DefaultSegmentInfosWriter e
out.writeInt(FORMAT_CURRENT); // write FORMAT
out.writeLong(infos.version);
out.writeInt(infos.counter); // write counter
+ out.writeLong(infos.getGlobalFieldMapVersion());
out.writeInt(infos.size()); // write infos
for (SegmentInfo si : infos) {
si.write(out);
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java Wed Mar 30 09:17:25 2011
@@ -30,15 +30,14 @@ import org.apache.lucene.index.codecs.Po
import org.apache.lucene.index.codecs.standard.StandardPostingsReader;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
-import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
-import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
+import org.apache.lucene.index.codecs.VariableGapTermsIndexReader;
+import org.apache.lucene.index.codecs.VariableGapTermsIndexWriter;
import org.apache.lucene.index.codecs.BlockTermsReader;
import org.apache.lucene.index.codecs.BlockTermsWriter;
import org.apache.lucene.index.codecs.TermsIndexReaderBase;
import org.apache.lucene.index.codecs.TermsIndexWriterBase;
import org.apache.lucene.index.codecs.standard.StandardCodec;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
/** This codec "inlines" the postings for terms that have
* low docFreq. It wraps another codec, which is used for
@@ -78,7 +77,7 @@ public class PulsingCodec extends Codec
TermsIndexWriterBase indexWriter;
boolean success = false;
try {
- indexWriter = new FixedGapTermsIndexWriter(state);
+ indexWriter = new VariableGapTermsIndexWriter(state, new VariableGapTermsIndexWriter.EveryNTermSelector(state.termIndexInterval));
success = true;
} finally {
if (!success) {
@@ -116,12 +115,11 @@ public class PulsingCodec extends Codec
boolean success = false;
try {
- indexReader = new FixedGapTermsIndexReader(state.dir,
- state.fieldInfos,
- state.segmentInfo.name,
- state.termsIndexDivisor,
- BytesRef.getUTF8SortedAsUnicodeComparator(),
- state.codecId);
+ indexReader = new VariableGapTermsIndexReader(state.dir,
+ state.fieldInfos,
+ state.segmentInfo.name,
+ state.termsIndexDivisor,
+ state.codecId);
success = true;
} finally {
if (!success) {
@@ -155,7 +153,7 @@ public class PulsingCodec extends Codec
public void files(Directory dir, SegmentInfo segmentInfo, String id, Set<String> files) throws IOException {
StandardPostingsReader.files(dir, segmentInfo, id, files);
BlockTermsReader.files(dir, segmentInfo, id, files);
- FixedGapTermsIndexReader.files(dir, segmentInfo, id, files);
+ VariableGapTermsIndexReader.files(dir, segmentInfo, id, files);
}
@Override
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/messages/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/messages/package.html?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/messages/package.html (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/messages/package.html Wed Mar 30 09:17:25 2011
@@ -45,7 +45,7 @@ Features:
<p>
Lazy loading of Message Strings
-<pre>
+<pre class="prettyprint">
public class MessagesTestBundle extends NLS {
private static final String BUNDLE_NAME = MessagesTestBundle.class.getName();
@@ -85,7 +85,7 @@ Lazy loading of Message Strings
<p>
Normal loading of Message Strings
-<pre>
+<pre class="prettyprint">
String message1 = NLS.getLocalizedMessage(MessagesTestBundle.Q0004E_INVALID_SYNTAX_ESCAPE_UNICODE_TRUNCATION);
String message2 = NLS.getLocalizedMessage(MessagesTestBundle.Q0004E_INVALID_SYNTAX_ESCAPE_UNICODE_TRUNCATION, Locale.JAPANESE);
</pre>
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java Wed Mar 30 09:17:25 2011
@@ -78,9 +78,9 @@ public abstract class QueryParserBase {
// maps field names to date resolutions
Map<String,DateTools.Resolution> fieldToDateResolution = null;
- // The collator to use when determining range inclusion,
- // for use when constructing RangeQuerys.
- Collator rangeCollator = null;
+ //Whether or not to analyze range terms when constructing RangeQuerys
+ // (For example, analyzing terms into collation keys for locale-sensitive RangeQuery)
+ boolean analyzeRangeTerms = false;
boolean autoGeneratePhraseQueries;
@@ -391,27 +391,21 @@ public abstract class QueryParserBase {
}
/**
- * Sets the collator used to determine index term inclusion in ranges
- * for RangeQuerys.
- * <p/>
- * <strong>WARNING:</strong> Setting the rangeCollator to a non-null
- * collator using this method will cause every single index Term in the
- * Field referenced by lowerTerm and/or upperTerm to be examined.
- * Depending on the number of index Terms in this Field, the operation could
- * be very slow.
- *
- * @param rc the collator to use when constructing RangeQuerys
+ * Set whether or not to analyze range terms when constructing RangeQuerys.
+ * For example, setting this to true can enable analyzing terms into
+ * collation keys for locale-sensitive RangeQuery.
+ *
+ * @param analyzeRangeTerms whether or not terms should be analyzed for RangeQuerys
*/
- public void setRangeCollator(Collator rc) {
- rangeCollator = rc;
+ public void setAnalyzeRangeTerms(boolean analyzeRangeTerms) {
+ this.analyzeRangeTerms = analyzeRangeTerms;
}
/**
- * @return the collator used to determine index term inclusion in ranges
- * for RangeQuerys.
+ * @return whether or not to analyze range terms when constructing RangeQuerys.
*/
- public Collator getRangeCollator() {
- return rangeCollator;
+ public boolean getAnalyzeRangeTerms() {
+ return analyzeRangeTerms;
}
protected void addClause(List<BooleanClause> clauses, int conj, int mods, Query q) {
@@ -538,18 +532,19 @@ public abstract class QueryParserBase {
// ignore
}
+ BytesRef bytes = termAtt == null ? null : termAtt.getBytesRef();
+
if (numTokens == 0)
return null;
else if (numTokens == 1) {
- BytesRef term = new BytesRef();
try {
boolean hasNext = buffer.incrementToken();
assert hasNext == true;
- termAtt.toBytesRef(term);
+ termAtt.fillBytesRef();
} catch (IOException e) {
// safe to ignore, because we know the number of tokens
}
- return newTermQuery(new Term(field, term));
+ return newTermQuery(new Term(field, new BytesRef(bytes)));
} else {
if (severalTokensAtSamePosition || (!quoted && !autoGeneratePhraseQueries)) {
if (positionCount == 1 || (!quoted && !autoGeneratePhraseQueries)) {
@@ -560,17 +555,15 @@ public abstract class QueryParserBase {
BooleanClause.Occur.MUST : BooleanClause.Occur.SHOULD;
for (int i = 0; i < numTokens; i++) {
- BytesRef term = new BytesRef();
try {
boolean hasNext = buffer.incrementToken();
assert hasNext == true;
- termAtt.toBytesRef(term);
+ termAtt.fillBytesRef();
} catch (IOException e) {
// safe to ignore, because we know the number of tokens
}
-
Query currentQuery = newTermQuery(
- new Term(field, term));
+ new Term(field, new BytesRef(bytes)));
q.add(currentQuery, occur);
}
return q;
@@ -582,12 +575,11 @@ public abstract class QueryParserBase {
List<Term> multiTerms = new ArrayList<Term>();
int position = -1;
for (int i = 0; i < numTokens; i++) {
- BytesRef term = new BytesRef();
int positionIncrement = 1;
try {
boolean hasNext = buffer.incrementToken();
assert hasNext == true;
- termAtt.toBytesRef(term);
+ termAtt.fillBytesRef();
if (posIncrAtt != null) {
positionIncrement = posIncrAtt.getPositionIncrement();
}
@@ -604,7 +596,7 @@ public abstract class QueryParserBase {
multiTerms.clear();
}
position += positionIncrement;
- multiTerms.add(new Term(field, term));
+ multiTerms.add(new Term(field, new BytesRef(bytes)));
}
if (enablePositionIncrements) {
mpq.add(multiTerms.toArray(new Term[0]),position);
@@ -619,15 +611,13 @@ public abstract class QueryParserBase {
pq.setSlop(phraseSlop);
int position = -1;
-
for (int i = 0; i < numTokens; i++) {
- BytesRef term = new BytesRef();
int positionIncrement = 1;
try {
boolean hasNext = buffer.incrementToken();
assert hasNext == true;
- termAtt.toBytesRef(term);
+ termAtt.fillBytesRef();
if (posIncrAtt != null) {
positionIncrement = posIncrAtt.getPositionIncrement();
}
@@ -637,9 +627,9 @@ public abstract class QueryParserBase {
if (enablePositionIncrements) {
position += positionIncrement;
- pq.add(new Term(field, term),position);
+ pq.add(new Term(field, new BytesRef(bytes)),position);
} else {
- pq.add(new Term(field, term));
+ pq.add(new Term(field, new BytesRef(bytes)));
}
}
return pq;
@@ -792,6 +782,36 @@ public abstract class QueryParserBase {
return new FuzzyQuery(term,minimumSimilarity,prefixLength);
}
+ private BytesRef analyzeRangePart(String field, String part) {
+ TokenStream source;
+
+ try {
+ source = analyzer.reusableTokenStream(field, new StringReader(part));
+ source.reset();
+ } catch (IOException e) {
+ source = analyzer.tokenStream(field, new StringReader(part));
+ }
+
+ TermToBytesRefAttribute termAtt = source.getAttribute(TermToBytesRefAttribute.class);
+ BytesRef bytes = termAtt.getBytesRef();
+
+ try {
+ if (!source.incrementToken())
+ throw new IllegalArgumentException("analyzer returned no terms for range part: " + part);
+ termAtt.fillBytesRef();
+ if (source.incrementToken())
+ throw new IllegalArgumentException("analyzer returned too many terms for range part: " + part);
+ } catch (IOException e) {
+ throw new RuntimeException("error analyzing range part: " + part, e);
+ }
+
+ try {
+ source.close();
+ } catch (IOException ignored) {}
+
+ return new BytesRef(bytes);
+ }
+
/**
* Builds a new TermRangeQuery instance
* @param field Field
@@ -802,7 +822,23 @@ public abstract class QueryParserBase {
* @return new TermRangeQuery instance
*/
protected Query newRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) {
- final TermRangeQuery query = new TermRangeQuery(field, part1, part2, startInclusive, endInclusive, rangeCollator);
+ final BytesRef start;
+ final BytesRef end;
+
+ if (part1 == null) {
+ start = null;
+ } else {
+ start = analyzeRangeTerms ? analyzeRangePart(field, part1) : new BytesRef(part1);
+ }
+
+ if (part2 == null) {
+ end = null;
+ } else {
+ end = analyzeRangeTerms ? analyzeRangePart(field, part2) : new BytesRef(part2);
+ }
+
+ final TermRangeQuery query = new TermRangeQuery(field, start, end, startInclusive, endInclusive);
+
query.setRewriteMethod(multiTermRewriteMethod);
return query;
}
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java Wed Mar 30 09:17:25 2011
@@ -20,7 +20,7 @@ import org.apache.lucene.index.FieldInve
*/
/** Expert: Default scoring implementation. */
-public class DefaultSimilarity extends Similarity implements SimilarityProvider {
+public class DefaultSimilarity extends Similarity {
/** Implemented as
* <code>state.getBoost()*lengthNorm(numTerms)</code>, where
@@ -40,11 +40,6 @@ public class DefaultSimilarity extends S
return state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms)));
}
- /** Implemented as <code>1/sqrt(sumOfSquaredWeights)</code>. */
- public float queryNorm(float sumOfSquaredWeights) {
- return (float)(1.0 / Math.sqrt(sumOfSquaredWeights));
- }
-
/** Implemented as <code>sqrt(freq)</code>. */
@Override
public float tf(float freq) {
@@ -63,11 +58,6 @@ public class DefaultSimilarity extends S
return (float)(Math.log(numDocs/(double)(docFreq+1)) + 1.0);
}
- /** Implemented as <code>overlap / maxOverlap</code>. */
- public float coord(int overlap, int maxOverlap) {
- return overlap / (float)maxOverlap;
- }
-
// Default true
protected boolean discountOverlaps = true;
@@ -88,12 +78,4 @@ public class DefaultSimilarity extends S
public boolean getDiscountOverlaps() {
return discountOverlaps;
}
-
- /**
- * Returns this default implementation for all fields.
- * Override this method to customize scoring on a per-field basis.
- */
- public Similarity get(String field) {
- return this;
- }
}