You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2011/05/05 11:09:43 UTC
svn commit: r1099728 [1/2] - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/ lucene/backwards/src/test/org/apache/lucene/index/
lucene/contrib/queries/src/test/org/apache/lucene/search/
lucene/src/java/org/apache/lucene/index/ lucene/sr...
Author: mikemccand
Date: Thu May 5 09:09:41 2011
New Revision: 1099728
URL: http://svn.apache.org/viewvc?rev=1099728&view=rev
Log:
LUCENE-1076: allow non-contiguous merges; improve handling of buffered deletes (merge from trunk, part 1)
Added:
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java
- copied, changed from r1065095, lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java
lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java
- copied unchanged from r1065095, lucene/dev/trunk/lucene/src/test/org/apache/lucene/index/MockRandomMergePolicy.java
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java
lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearch.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestNorms.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPayloads.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/BaseTestRangeFilter.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestDocBoost.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestFilteredQuery.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestFilteredSearch.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestSubScorerFreqs.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/TestTermScorer.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/function/FunctionTestSetup.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/search/spans/TestSpansAdvanced2.java
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java (original)
+++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java Thu May 5 09:09:41 2011
@@ -70,12 +70,12 @@ public class TestPerSegmentDeletes exten
// flushing without applying deletes means
// there will still be deletes in the segment infos
writer.flush(false, false);
- assertTrue(writer.bufferedDeletes.any());
+ //assertTrue(writer.bufferedDeletes.any());
// get reader flushes pending deletes
// so there should not be anymore
IndexReader r1 = writer.getReader();
- assertFalse(writer.bufferedDeletes.any());
+ //assertFalse(writer.bufferedDeletes.any());
r1.close();
// delete id:2 from the first segment
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queries/src/test/org/apache/lucene/search/FuzzyLikeThisQueryTest.java Thu May 5 09:09:41 2011
@@ -42,7 +42,6 @@ public class FuzzyLikeThisQueryTest exte
super.setUp();
directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random, directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
-
//Add series of docs with misspelt names
addDoc(writer, "jonathon smythe","1");
addDoc(writer, "jonathan smith","2");
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java Thu May 5 09:09:41 2011
@@ -17,410 +17,165 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import java.io.IOException;
-import java.io.PrintStream;
+import java.util.ArrayList;
import java.util.HashMap;
-import java.util.Date;
-import java.util.Map.Entry;
+import java.util.List;
import java.util.Map;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.SortedMap;
+import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Weight;
-
-/** Holds a {@link SegmentDeletes} for each segment in the
- * index. */
+import org.apache.lucene.util.RamUsageEstimator;
+/** Holds buffered deletes, by docID, term or query for a
+ * single segment. This is used to hold buffered pending
+ * deletes against the to-be-flushed segment as well as
+ * per-segment deletes for each segment in the index. */
+
+// NOTE: we are sync'd by BufferedDeletes, ie, all access to
+// instances of this class is via sync'd methods on
+// BufferedDeletes
class BufferedDeletes {
- // Deletes for all flushed/merged segments:
- private final Map<SegmentInfo,SegmentDeletes> deletesMap = new HashMap<SegmentInfo,SegmentDeletes>();
-
- // used only by assert
- private Term lastDeleteTerm;
-
- private PrintStream infoStream;
- private final AtomicLong bytesUsed = new AtomicLong();
- private final AtomicInteger numTerms = new AtomicInteger();
- private final int messageID;
-
- public BufferedDeletes(int messageID) {
- this.messageID = messageID;
- }
-
- private synchronized void message(String message) {
- if (infoStream != null) {
- infoStream.println("BD " + messageID + " [" + new Date() + "; " + Thread.currentThread().getName() + "]: BD " + message);
- }
- }
-
- public synchronized void setInfoStream(PrintStream infoStream) {
- this.infoStream = infoStream;
- }
-
- public synchronized void pushDeletes(SegmentDeletes newDeletes, SegmentInfo info) {
- pushDeletes(newDeletes, info, false);
- }
-
- // Moves all pending deletes onto the provided segment,
- // then clears the pending deletes
- public synchronized void pushDeletes(SegmentDeletes newDeletes, SegmentInfo info, boolean noLimit) {
- assert newDeletes.any();
- numTerms.addAndGet(newDeletes.numTermDeletes.get());
-
- if (!noLimit) {
- assert !deletesMap.containsKey(info);
- assert info != null;
- deletesMap.put(info, newDeletes);
- bytesUsed.addAndGet(newDeletes.bytesUsed.get());
+ /* Rough logic: HashMap has an array[Entry] w/ varying
+ load factor (say 2 * POINTER). Entry is object w/ Term
+ key, Integer val, int hash, Entry next
+ (OBJ_HEADER + 3*POINTER + INT). Term is object w/
+ String field and String text (OBJ_HEADER + 2*POINTER).
+ We don't count Term's field since it's interned.
+ Term's text is String (OBJ_HEADER + 4*INT + POINTER +
+ OBJ_HEADER + string.length*CHAR). Integer is
+ OBJ_HEADER + INT. */
+ final static int BYTES_PER_DEL_TERM = 8*RamUsageEstimator.NUM_BYTES_OBJECT_REF + 5*RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 6*RamUsageEstimator.NUM_BYTES_INT;
+
+ /* Rough logic: del docIDs are List<Integer>. Say list
+ allocates ~2X size (2*POINTER). Integer is OBJ_HEADER
+ + int */
+ final static int BYTES_PER_DEL_DOCID = 2*RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + RamUsageEstimator.NUM_BYTES_INT;
+
+ /* Rough logic: HashMap has an array[Entry] w/ varying
+ load factor (say 2 * POINTER). Entry is object w/
+ Query key, Integer val, int hash, Entry next
+ (OBJ_HEADER + 3*POINTER + INT). Query we often
+ undercount (say 24 bytes). Integer is OBJ_HEADER + INT. */
+ final static int BYTES_PER_DEL_QUERY = 5*RamUsageEstimator.NUM_BYTES_OBJECT_REF + 2*RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 2*RamUsageEstimator.NUM_BYTES_INT + 24;
+
+ // TODO: many of the deletes stored here will map to
+ // Integer.MAX_VALUE; we could be more efficient for this
+ // case ie use a SortedSet not a SortedMap. But: Java's
+ // SortedSet impls are simply backed by a Map so we won't
+ // save anything unless we do something custom...
+ final AtomicInteger numTermDeletes = new AtomicInteger();
+ final SortedMap<Term,Integer> terms = new TreeMap<Term,Integer>();
+ final Map<Query,Integer> queries = new HashMap<Query,Integer>();
+ final List<Integer> docIDs = new ArrayList<Integer>();
+
+ public static final Integer MAX_INT = Integer.valueOf(Integer.MAX_VALUE);
+
+ final AtomicLong bytesUsed = new AtomicLong();
+
+ private final static boolean VERBOSE_DELETES = false;
+
+ long gen;
+
+ @Override
+ public String toString() {
+ if (VERBOSE_DELETES) {
+ return "gen=" + gen + " numTerms=" + numTermDeletes + ", terms=" + terms
+ + ", queries=" + queries + ", docIDs=" + docIDs + ", bytesUsed="
+ + bytesUsed;
} else {
- final SegmentDeletes deletes = getDeletes(info);
- bytesUsed.addAndGet(-deletes.bytesUsed.get());
- deletes.update(newDeletes, noLimit);
- bytesUsed.addAndGet(deletes.bytesUsed.get());
- }
- if (infoStream != null) {
- message("push deletes seg=" + info + " dels=" + getDeletes(info));
- }
- assert checkDeleteStats();
- }
-
- public synchronized void clear() {
- deletesMap.clear();
- numTerms.set(0);
- bytesUsed.set(0);
- }
-
- synchronized boolean any() {
- return bytesUsed.get() != 0;
- }
-
- public int numTerms() {
- return numTerms.get();
- }
-
- public long bytesUsed() {
- return bytesUsed.get();
- }
-
- // IW calls this on finishing a merge. While the merge
- // was running, it's possible new deletes were pushed onto
- // our last (and only our last) segment. In this case we
- // must carry forward those deletes onto the merged
- // segment.
- synchronized void commitMerge(MergePolicy.OneMerge merge) {
- assert checkDeleteStats();
- if (infoStream != null) {
- message("commitMerge merge.info=" + merge.info + " merge.segments=" + merge.segments);
- }
- final SegmentInfo lastInfo = merge.segments.lastElement();
- final SegmentDeletes lastDeletes = deletesMap.get(lastInfo);
- if (lastDeletes != null) {
- deletesMap.remove(lastInfo);
- assert !deletesMap.containsKey(merge.info);
- deletesMap.put(merge.info, lastDeletes);
- // don't need to update numTerms/bytesUsed since we
- // are just moving the deletes from one info to
- // another
- if (infoStream != null) {
- message("commitMerge done: new deletions=" + lastDeletes);
+ String s = "gen=" + gen;
+ if (numTermDeletes.get() != 0) {
+ s += " " + numTermDeletes.get() + " deleted terms (unique count=" + terms.size() + ")";
}
- } else if (infoStream != null) {
- message("commitMerge done: no new deletions");
- }
- assert !anyDeletes(merge.segments.range(0, merge.segments.size()-1));
- assert checkDeleteStats();
- }
-
- synchronized void clear(SegmentDeletes deletes) {
- deletes.clear();
- }
-
- public synchronized boolean applyDeletes(IndexWriter.ReaderPool readerPool, SegmentInfos segmentInfos, SegmentInfos applyInfos) throws IOException {
- if (!any()) {
- return false;
- }
- final long t0 = System.currentTimeMillis();
-
- if (infoStream != null) {
- message("applyDeletes: applyInfos=" + applyInfos + "; index=" + segmentInfos);
- }
-
- assert checkDeleteStats();
-
- assert applyInfos.size() > 0;
-
- boolean any = false;
-
- final SegmentInfo lastApplyInfo = applyInfos.lastElement();
- final int lastIdx = segmentInfos.indexOf(lastApplyInfo);
-
- final SegmentInfo firstInfo = applyInfos.firstElement();
- final int firstIdx = segmentInfos.indexOf(firstInfo);
-
- // applyInfos must be a slice of segmentInfos
- assert lastIdx - firstIdx + 1 == applyInfos.size();
-
- // iterate over all segment infos backwards
- // coalesceing deletes along the way
- // when we're at or below the last of the
- // segments to apply to, start applying the deletes
- // we traverse up to the first apply infos
- SegmentDeletes coalescedDeletes = null;
- boolean hasDeletes = false;
- for (int segIdx=segmentInfos.size()-1; segIdx >= firstIdx; segIdx--) {
- final SegmentInfo info = segmentInfos.info(segIdx);
- final SegmentDeletes deletes = deletesMap.get(info);
- assert deletes == null || deletes.any();
-
- if (deletes == null && coalescedDeletes == null) {
- continue;
+ if (queries.size() != 0) {
+ s += " " + queries.size() + " deleted queries";
}
-
- if (infoStream != null) {
- message("applyDeletes: seg=" + info + " segment's deletes=[" + (deletes == null ? "null" : deletes) + "]; coalesced deletes=[" + (coalescedDeletes == null ? "null" : coalescedDeletes) + "]");
+ if (docIDs.size() != 0) {
+ s += " " + docIDs.size() + " deleted docIDs";
}
-
- hasDeletes |= deletes != null;
-
- if (segIdx <= lastIdx && hasDeletes) {
-
- final long delCountInc = applyDeletes(readerPool, info, coalescedDeletes, deletes);
-
- if (delCountInc != 0) {
- any = true;
- }
-
- if (infoStream != null) {
- message("deletes touched " + delCountInc + " docIDs");
- }
-
- if (deletes != null) {
- // we've applied doc ids, and they're only applied
- // on the current segment
- bytesUsed.addAndGet(-deletes.docIDs.size() * SegmentDeletes.BYTES_PER_DEL_DOCID);
- deletes.clearDocIDs();
- }
- }
-
- // now coalesce at the max limit
- if (deletes != null) {
- if (coalescedDeletes == null) {
- coalescedDeletes = new SegmentDeletes();
- }
- // TODO: we could make this single pass (coalesce as
- // we apply the deletes
- coalescedDeletes.update(deletes, true);
- }
- }
-
- // move all deletes to segment just before our merge.
- if (firstIdx > 0) {
-
- SegmentDeletes mergedDeletes = null;
- // TODO: we could also make this single pass
- for (SegmentInfo info : applyInfos) {
- final SegmentDeletes deletes = deletesMap.get(info);
- if (deletes != null) {
- assert deletes.any();
- if (mergedDeletes == null) {
- mergedDeletes = getDeletes(segmentInfos.info(firstIdx-1));
- numTerms.addAndGet(-mergedDeletes.numTermDeletes.get());
- assert numTerms.get() >= 0;
- bytesUsed.addAndGet(-mergedDeletes.bytesUsed.get());
- assert bytesUsed.get() >= 0;
- }
-
- mergedDeletes.update(deletes, true);
- }
+ if (bytesUsed.get() != 0) {
+ s += " bytesUsed=" + bytesUsed.get();
}
- if (mergedDeletes != null) {
- numTerms.addAndGet(mergedDeletes.numTermDeletes.get());
- bytesUsed.addAndGet(mergedDeletes.bytesUsed.get());
- }
-
- if (infoStream != null) {
- if (mergedDeletes != null) {
- message("applyDeletes: merge all deletes into seg=" + segmentInfos.info(firstIdx-1) + ": " + mergedDeletes);
- } else {
- message("applyDeletes: no deletes to merge");
- }
- }
- } else {
- // We drop the deletes in this case, because we've
- // applied them to segment infos starting w/ the first
- // segment. There are no prior segments so there's no
- // reason to keep them around. When the applyInfos ==
- // segmentInfos this means all deletes have been
- // removed:
- }
- remove(applyInfos);
-
- assert checkDeleteStats();
- assert applyInfos != segmentInfos || !any();
-
- if (infoStream != null) {
- message("applyDeletes took " + (System.currentTimeMillis()-t0) + " msec");
+ return s;
}
- return any;
}
- private synchronized long applyDeletes(IndexWriter.ReaderPool readerPool,
- SegmentInfo info,
- SegmentDeletes coalescedDeletes,
- SegmentDeletes segmentDeletes) throws IOException {
- assert readerPool.infoIsLive(info);
-
- assert coalescedDeletes == null || coalescedDeletes.docIDs.size() == 0;
-
- long delCount = 0;
-
- // Lock order: IW -> BD -> RP
- SegmentReader reader = readerPool.get(info, false);
- try {
- if (coalescedDeletes != null) {
- delCount += applyDeletes(coalescedDeletes, reader);
- }
- if (segmentDeletes != null) {
- delCount += applyDeletes(segmentDeletes, reader);
+ void update(BufferedDeletes in) {
+ numTermDeletes.addAndGet(in.numTermDeletes.get());
+ for (Map.Entry<Term,Integer> ent : in.terms.entrySet()) {
+ final Term term = ent.getKey();
+ if (!terms.containsKey(term)) {
+ // only incr bytesUsed if this term wasn't already buffered:
+ bytesUsed.addAndGet(BYTES_PER_DEL_TERM);
}
- } finally {
- readerPool.release(reader);
+ terms.put(term, MAX_INT);
}
- return delCount;
- }
-
- private synchronized long applyDeletes(SegmentDeletes deletes, SegmentReader reader) throws IOException {
-
- long delCount = 0;
- assert checkDeleteTerm(null);
-
- if (deletes.terms.size() > 0) {
- final TermDocs docs = reader.termDocs();
-
- for (Entry<Term,Integer> entry: deletes.terms.entrySet()) {
- Term term = entry.getKey();
- // Since we visit terms sorted, we gain performance
- // by re-using the same TermsEnum and seeking only
- // forwards
- assert checkDeleteTerm(term);
- docs.seek(term);
-
- final int limit = entry.getValue();
- while (docs.next()) {
- final int docID = docs.doc();
- if (docID >= limit) {
- break;
- }
- // TODO: we could/should change
- // reader.deleteDocument to return boolean
- // true if it did in fact delete, because here
- // we could be deleting an already-deleted doc
- // which makes this an upper bound:
- delCount++;
-
- reader.deleteDocument(docID);
- }
+ for (Map.Entry<Query,Integer> ent : in.queries.entrySet()) {
+ final Query query = ent.getKey();
+ if (!queries.containsKey(query)) {
+ // only incr bytesUsed if this query wasn't already buffered:
+ bytesUsed.addAndGet(BYTES_PER_DEL_QUERY);
}
+ queries.put(query, MAX_INT);
}
- // Delete by docID
- for (Integer docIdInt : deletes.docIDs) {
- int docID = docIdInt.intValue();
- reader.deleteDocument(docID);
- delCount++;
- }
-
- // Delete by query
- if (deletes.queries.size() > 0) {
- IndexSearcher searcher = new IndexSearcher(reader);
- try {
- for (Entry<Query, Integer> entry : deletes.queries.entrySet()) {
- Query query = entry.getKey();
- int limit = entry.getValue().intValue();
- Weight weight = query.weight(searcher);
- Scorer scorer = weight.scorer(reader, true, false);
- if (scorer != null) {
- while(true) {
- int doc = scorer.nextDoc();
- if (doc >= limit)
- break;
-
- reader.deleteDocument(doc);
- // TODO: we could/should change
- // reader.deleteDocument to return boolean
- // true if it did in fact delete, because here
- // we could be deleting an already-deleted doc
- // which makes this an upper bound:
- delCount++;
- }
- }
- }
- } finally {
- searcher.close();
- }
- }
-
- return delCount;
+ // docIDs never move across segments and the docIDs
+ // should already be cleared
}
-
- public synchronized SegmentDeletes getDeletes(SegmentInfo info) {
- SegmentDeletes deletes = deletesMap.get(info);
- if (deletes == null) {
- deletes = new SegmentDeletes();
- deletesMap.put(info, deletes);
+
+ public void addQuery(Query query, int docIDUpto) {
+ Integer current = queries.put(query, docIDUpto);
+ // increment bytes used only if the query wasn't added so far.
+ if (current == null) {
+ bytesUsed.addAndGet(BYTES_PER_DEL_QUERY);
}
- return deletes;
}
-
- public synchronized void remove(SegmentInfos infos) {
- assert infos.size() > 0;
- for (SegmentInfo info : infos) {
- SegmentDeletes deletes = deletesMap.get(info);
- if (deletes != null) {
- bytesUsed.addAndGet(-deletes.bytesUsed.get());
- assert bytesUsed.get() >= 0: "bytesUsed=" + bytesUsed;
- numTerms.addAndGet(-deletes.numTermDeletes.get());
- assert numTerms.get() >= 0: "numTerms=" + numTerms;
- deletesMap.remove(info);
- }
- }
+
+ public void addDocID(int docID) {
+ docIDs.add(Integer.valueOf(docID));
+ bytesUsed.addAndGet(BYTES_PER_DEL_DOCID);
}
- // used only by assert
- private boolean anyDeletes(SegmentInfos infos) {
- for(SegmentInfo info : infos) {
- if (deletesMap.containsKey(info)) {
- return true;
- }
+ public void addTerm(Term term, int docIDUpto) {
+ Integer current = terms.get(term);
+ if (current != null && docIDUpto < current) {
+ // Only record the new number if it's greater than the
+ // current one. This is important because if multiple
+ // threads are replacing the same doc at nearly the
+ // same time, it's possible that one thread that got a
+ // higher docID is scheduled before the other
+ // threads. If we blindly replace then we can
+ // incorrectly get both docs indexed.
+ return;
}
- return false;
- }
- // used only by assert
- private boolean checkDeleteTerm(Term term) {
- if (term != null) {
- assert lastDeleteTerm == null || term.compareTo(lastDeleteTerm) > 0: "lastTerm=" + lastDeleteTerm + " vs term=" + term;
+ terms.put(term, Integer.valueOf(docIDUpto));
+ numTermDeletes.incrementAndGet();
+ if (current == null) {
+ bytesUsed.addAndGet(BYTES_PER_DEL_TERM + term.text.length() * RamUsageEstimator.NUM_BYTES_CHAR);
}
- lastDeleteTerm = term;
- return true;
+ }
+
+ void clear() {
+ terms.clear();
+ queries.clear();
+ docIDs.clear();
+ numTermDeletes.set(0);
+ bytesUsed.set(0);
}
- // only for assert
- private boolean checkDeleteStats() {
- int numTerms2 = 0;
- long bytesUsed2 = 0;
- for(SegmentDeletes deletes : deletesMap.values()) {
- numTerms2 += deletes.numTermDeletes.get();
- bytesUsed2 += deletes.bytesUsed.get();
- }
- assert numTerms2 == numTerms.get(): "numTerms2=" + numTerms2 + " vs " + numTerms.get();
- assert bytesUsed2 == bytesUsed.get(): "bytesUsed2=" + bytesUsed2 + " vs " + bytesUsed;
- return true;
+ void clearDocIDs() {
+ bytesUsed.addAndGet(-docIDs.size()*BYTES_PER_DEL_DOCID);
+ docIDs.clear();
+ }
+
+ boolean any() {
+ return terms.size() > 0 || docIDs.size() > 0 || queries.size() > 0;
}
}
Copied: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java (from r1065095, lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java?p2=lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java&r1=1065095&r2=1099728&rev=1099728&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java Thu May 5 09:09:41 2011
@@ -28,7 +28,6 @@ import java.util.Collections;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
-import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
@@ -130,7 +129,7 @@ class BufferedDeletesStream {
// Sorts SegmentInfos from smallest to biggest bufferedDelGen:
private static final Comparator<SegmentInfo> sortByDelGen = new Comparator<SegmentInfo>() {
- @Override
+ // @Override -- not until Java 1.6
public int compare(SegmentInfo si1, SegmentInfo si2) {
final long cmp = si1.getBufferedDeletesGen() - si2.getBufferedDeletesGen();
if (cmp > 0) {
@@ -318,58 +317,30 @@ class BufferedDeletesStream {
assert checkDeleteTerm(null);
if (deletes.terms.size() > 0) {
- Fields fields = reader.fields();
- if (fields == null) {
- // This reader has no postings
- return 0;
- }
-
- TermsEnum termsEnum = null;
-
- String currentField = null;
- DocsEnum docs = null;
+ final TermDocs docs = reader.termDocs();
for (Entry<Term,Integer> entry: deletes.terms.entrySet()) {
Term term = entry.getKey();
// Since we visit terms sorted, we gain performance
// by re-using the same TermsEnum and seeking only
// forwards
- if (term.field() != currentField) {
- assert currentField == null || currentField.compareTo(term.field()) < 0;
- currentField = term.field();
- Terms terms = fields.terms(currentField);
- if (terms != null) {
- termsEnum = terms.iterator();
- } else {
- termsEnum = null;
- }
- }
-
- if (termsEnum == null) {
- continue;
- }
assert checkDeleteTerm(term);
+ docs.seek(term);
- if (termsEnum.seek(term.bytes(), false) == TermsEnum.SeekStatus.FOUND) {
- DocsEnum docsEnum = termsEnum.docs(reader.getDeletedDocs(), docs);
-
- if (docsEnum != null) {
- docs = docsEnum;
- final int limit = entry.getValue();
- while (true) {
- final int docID = docs.nextDoc();
- if (docID == DocsEnum.NO_MORE_DOCS || docID >= limit) {
- break;
- }
- reader.deleteDocument(docID);
- // TODO: we could/should change
- // reader.deleteDocument to return boolean
- // true if it did in fact delete, because here
- // we could be deleting an already-deleted doc
- // which makes this an upper bound:
- delCount++;
- }
+ final int limit = entry.getValue();
+ while (docs.next()) {
+ final int docID = docs.doc();
+ if (docID >= limit) {
+ break;
}
+ // TODO: we could/should change
+ // reader.deleteDocument to return boolean
+ // true if it did in fact delete, because here
+ // we could be deleting an already-deleted doc
+ // which makes this an upper bound:
+ delCount++;
+
+ reader.deleteDocument(docID);
}
}
}
@@ -384,14 +355,12 @@ class BufferedDeletesStream {
// Delete by query
if (deletes.queries.size() > 0) {
IndexSearcher searcher = new IndexSearcher(reader);
- assert searcher.getTopReaderContext().isAtomic;
- final AtomicReaderContext readerContext = (AtomicReaderContext) searcher.getTopReaderContext();
try {
for (Entry<Query, Integer> entry : deletes.queries.entrySet()) {
Query query = entry.getKey();
int limit = entry.getValue().intValue();
Weight weight = query.weight(searcher);
- Scorer scorer = weight.scorer(readerContext, Weight.ScorerContext.def());
+ Scorer scorer = weight.scorer(reader, true, false);
if (scorer != null) {
while(true) {
int doc = scorer.nextDoc();
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Thu May 5 09:09:41 2011
@@ -132,7 +132,7 @@ final class DocumentsWriter {
private final int maxThreadStates;
// Deletes for our still-in-RAM (to be flushed next) segment
- private SegmentDeletes pendingDeletes = new SegmentDeletes();
+ private BufferedDeletes pendingDeletes = new BufferedDeletes();
static class DocState {
DocumentsWriter docWriter;
@@ -264,16 +264,16 @@ final class DocumentsWriter {
private boolean closed;
private final FieldInfos fieldInfos;
- private final BufferedDeletes bufferedDeletes;
+ private final BufferedDeletesStream bufferedDeletesStream;
private final IndexWriter.FlushControl flushControl;
- DocumentsWriter(IndexWriterConfig config, Directory directory, IndexWriter writer, FieldInfos fieldInfos, BufferedDeletes bufferedDeletes) throws IOException {
+ DocumentsWriter(IndexWriterConfig config, Directory directory, IndexWriter writer, FieldInfos fieldInfos, BufferedDeletesStream bufferedDeletesStream) throws IOException {
this.directory = directory;
this.writer = writer;
this.similarity = config.getSimilarity();
this.maxThreadStates = config.getMaxThreadStates();
this.fieldInfos = fieldInfos;
- this.bufferedDeletes = bufferedDeletes;
+ this.bufferedDeletesStream = bufferedDeletesStream;
flushControl = writer.flushControl;
consumer = config.getIndexingChain().getChain(this);
@@ -463,23 +463,24 @@ final class DocumentsWriter {
}
// for testing
- public SegmentDeletes getPendingDeletes() {
+ public BufferedDeletes getPendingDeletes() {
return pendingDeletes;
}
private void pushDeletes(SegmentInfo newSegment, SegmentInfos segmentInfos) {
// Lock order: DW -> BD
if (pendingDeletes.any()) {
- if (newSegment != null) {
+ if (segmentInfos.size() > 0 || newSegment != null) {
if (infoStream != null) {
- message("flush: push buffered deletes to newSegment");
+ message("flush: push buffered deletes");
}
- bufferedDeletes.pushDeletes(pendingDeletes, newSegment);
- } else if (segmentInfos.size() > 0) {
+ bufferedDeletesStream.push(pendingDeletes);
if (infoStream != null) {
- message("flush: push buffered deletes to previously flushed segment " + segmentInfos.lastElement());
+ message("flush: delGen=" + pendingDeletes.gen);
+ }
+ if (newSegment != null) {
+ newSegment.setBufferedDeletesGen(pendingDeletes.gen);
}
- bufferedDeletes.pushDeletes(pendingDeletes, segmentInfos.lastElement(), true);
} else {
if (infoStream != null) {
message("flush: drop buffered deletes: no segments");
@@ -488,7 +489,9 @@ final class DocumentsWriter {
// there are no segments, the deletions cannot
// affect anything.
}
- pendingDeletes = new SegmentDeletes();
+ pendingDeletes = new BufferedDeletes();
+ } else if (newSegment != null) {
+ newSegment.setBufferedDeletesGen(bufferedDeletesStream.getNextGen());
}
}
@@ -599,7 +602,6 @@ final class DocumentsWriter {
// Lock order: IW -> DW -> BD
pushDeletes(newSegment, segmentInfos);
-
if (infoStream != null) {
message("flush time " + (System.currentTimeMillis()-startTime) + " msec");
}
@@ -1013,7 +1015,7 @@ final class DocumentsWriter {
final boolean doBalance;
final long deletesRAMUsed;
- deletesRAMUsed = bufferedDeletes.bytesUsed();
+ deletesRAMUsed = bufferedDeletesStream.bytesUsed();
final long ramBufferSize;
final double mb = config.getRAMBufferSizeMB();
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java Thu May 5 09:09:41 2011
@@ -309,7 +309,7 @@ public class IndexWriter implements Clos
private final AtomicInteger flushDeletesCount = new AtomicInteger();
final ReaderPool readerPool = new ReaderPool();
- final BufferedDeletes bufferedDeletes;
+ final BufferedDeletesStream bufferedDeletesStream;
// This is a "write once" variable (like the organic dye
// on a DVD-R that may or may not be heated by a laser and
@@ -1077,8 +1077,8 @@ public class IndexWriter implements Clos
mergePolicy = conf.getMergePolicy();
mergePolicy.setIndexWriter(this);
mergeScheduler = conf.getMergeScheduler();
- bufferedDeletes = new BufferedDeletes(messageID);
- bufferedDeletes.setInfoStream(infoStream);
+ bufferedDeletesStream = new BufferedDeletesStream(messageID);
+ bufferedDeletesStream.setInfoStream(infoStream);
poolReaders = conf.getReaderPooling();
OpenMode mode = conf.getOpenMode();
@@ -1146,7 +1146,7 @@ public class IndexWriter implements Clos
setRollbackSegmentInfos(segmentInfos);
- docWriter = new DocumentsWriter(config, directory, this, getCurrentFieldInfos(), bufferedDeletes);
+ docWriter = new DocumentsWriter(config, directory, this, getCurrentFieldInfos(), bufferedDeletesStream);
docWriter.setInfoStream(infoStream);
docWriter.setMaxFieldLength(maxFieldLength);
@@ -1646,7 +1646,7 @@ public class IndexWriter implements Clos
this.infoStream = infoStream;
docWriter.setInfoStream(infoStream);
deleter.setInfoStream(infoStream);
- bufferedDeletes.setInfoStream(infoStream);
+ bufferedDeletesStream.setInfoStream(infoStream);
if (infoStream != null)
messageState();
}
@@ -1938,7 +1938,7 @@ public class IndexWriter implements Clos
public synchronized boolean hasDeletions() throws IOException {
ensureOpen();
- if (bufferedDeletes.any()) {
+ if (bufferedDeletesStream.any()) {
return true;
}
if (docWriter.anyDeletions()) {
@@ -2678,7 +2678,7 @@ public class IndexWriter implements Clos
mergePolicy.close();
mergeScheduler.close();
- bufferedDeletes.clear();
+ bufferedDeletesStream.clear();
synchronized(this) {
@@ -3349,10 +3349,10 @@ public class IndexWriter implements Clos
// tiny segments:
if (flushControl.getFlushDeletes() ||
(config.getRAMBufferSizeMB() != IndexWriterConfig.DISABLE_AUTO_FLUSH &&
- bufferedDeletes.bytesUsed() > (1024*1024*config.getRAMBufferSizeMB()/2))) {
+ bufferedDeletesStream.bytesUsed() > (1024*1024*config.getRAMBufferSizeMB()/2))) {
applyAllDeletes = true;
if (infoStream != null) {
- message("force apply deletes bytesUsed=" + bufferedDeletes.bytesUsed() + " vs ramBuffer=" + (1024*1024*config.getRAMBufferSizeMB()));
+ message("force apply deletes bytesUsed=" + bufferedDeletesStream.bytesUsed() + " vs ramBuffer=" + (1024*1024*config.getRAMBufferSizeMB()));
}
}
}
@@ -3362,12 +3362,15 @@ public class IndexWriter implements Clos
message("apply all deletes during flush");
}
flushDeletesCount.incrementAndGet();
- if (bufferedDeletes.applyDeletes(readerPool, segmentInfos, segmentInfos)) {
+ final BufferedDeletesStream.ApplyDeletesResult result = bufferedDeletesStream.applyDeletes(readerPool, segmentInfos);
+ if (result.anyDeletes) {
checkpoint();
}
+ bufferedDeletesStream.prune(segmentInfos);
+ assert !bufferedDeletesStream.any();
flushControl.clearDeletes();
} else if (infoStream != null) {
- message("don't apply deletes now delTermCount=" + bufferedDeletes.numTerms() + " bytesUsed=" + bufferedDeletes.bytesUsed());
+ message("don't apply deletes now delTermCount=" + bufferedDeletesStream.numTerms() + " bytesUsed=" + bufferedDeletesStream.bytesUsed());
}
doAfterFlush();
@@ -3393,7 +3396,7 @@ public class IndexWriter implements Clos
*/
public final long ramSizeInBytes() {
ensureOpen();
- return docWriter.bytesUsed() + bufferedDeletes.bytesUsed();
+ return docWriter.bytesUsed() + bufferedDeletesStream.bytesUsed();
}
/** Expert: Return the number of documents currently
@@ -3403,28 +3406,12 @@ public class IndexWriter implements Clos
return docWriter.getNumDocs();
}
- private int ensureContiguousMerge(MergePolicy.OneMerge merge) {
-
- int first = segmentInfos.indexOf(merge.segments.info(0));
- if (first == -1)
- throw new MergePolicy.MergeException("could not find segment " + merge.segments.info(0).name + " in current index " + segString(), directory);
-
- final int numSegments = segmentInfos.size();
-
- final int numSegmentsToMerge = merge.segments.size();
- for(int i=0;i<numSegmentsToMerge;i++) {
- final SegmentInfo info = merge.segments.info(i);
-
- if (first + i >= numSegments || !segmentInfos.info(first+i).equals(info)) {
- if (segmentInfos.indexOf(info) == -1)
- throw new MergePolicy.MergeException("MergePolicy selected a segment (" + info.name + ") that is not in the current index " + segString(), directory);
- else
- throw new MergePolicy.MergeException("MergePolicy selected non-contiguous segments to merge (" + merge.segString(directory) + " vs " + segString() + "), which IndexWriter (currently) cannot handle",
- directory);
+ private void ensureValidMerge(MergePolicy.OneMerge merge) {
+ for(SegmentInfo info : merge.segments) {
+ if (segmentInfos.indexOf(info) == -1) {
+ throw new MergePolicy.MergeException("MergePolicy selected a segment (" + info.name + ") that is not in the current index " + segString(), directory);
}
}
-
- return first;
}
/** Carefully merges deletes for the segments we just
@@ -3449,9 +3436,11 @@ public class IndexWriter implements Clos
// started merging:
int docUpto = 0;
int delCount = 0;
+ long minGen = Long.MAX_VALUE;
for(int i=0; i < sourceSegments.size(); i++) {
SegmentInfo info = sourceSegments.info(i);
+ minGen = Math.min(info.getBufferedDeletesGen(), minGen);
int docCount = info.docCount;
SegmentReader previousReader = merge.readersClone[i];
SegmentReader currentReader = merge.readers[i];
@@ -3500,9 +3489,17 @@ public class IndexWriter implements Clos
assert mergedReader.numDeletedDocs() == delCount;
mergedReader.hasChanges = delCount > 0;
+
+ // If new deletes were applied while we were merging
+ // (which happens if eg commit() or getReader() is
+ // called during our merge), then it better be the case
+ // that the delGen has increased for all our merged
+ // segments:
+ assert !mergedReader.hasChanges || minGen > mergedReader.getSegmentInfo().getBufferedDeletesGen();
+
+ mergedReader.getSegmentInfo().setBufferedDeletesGen(minGen);
}
- /* FIXME if we want to support non-contiguous segment merges */
synchronized private boolean commitMerge(MergePolicy.OneMerge merge, SegmentReader mergedReader) throws IOException {
assert testPoint("startCommitMerge");
@@ -3528,7 +3525,7 @@ public class IndexWriter implements Clos
return false;
}
- final int start = ensureContiguousMerge(merge);
+ ensureValidMerge(merge);
commitMergedDeletes(merge, mergedReader);
@@ -3538,10 +3535,32 @@ public class IndexWriter implements Clos
// format as well:
setMergeDocStoreIsCompoundFile(merge);
- segmentInfos.subList(start, start + merge.segments.size()).clear();
assert !segmentInfos.contains(merge.info);
- segmentInfos.add(start, merge.info);
-
+
+ final Set mergedAway = new HashSet<SegmentInfo>(merge.segments);
+ int segIdx = 0;
+ int newSegIdx = 0;
+ boolean inserted = false;
+ final int curSegCount = segmentInfos.size();
+ while(segIdx < curSegCount) {
+ final SegmentInfo info = segmentInfos.info(segIdx++);
+ if (mergedAway.contains(info)) {
+ if (!inserted) {
+ segmentInfos.set(segIdx-1, merge.info);
+ inserted = true;
+ newSegIdx++;
+ }
+ } else {
+ segmentInfos.set(newSegIdx++, info);
+ }
+ }
+ assert newSegIdx == curSegCount - merge.segments.size() + 1;
+ segmentInfos.subList(newSegIdx, segmentInfos.size()).clear();
+
+ if (infoStream != null) {
+ message("after commit: " + segString());
+ }
+
closeMergeReaders(merge, false);
// Must note the change to segmentInfos so any commits
@@ -3553,16 +3572,12 @@ public class IndexWriter implements Clos
// disk, updating SegmentInfo, etc.:
readerPool.clear(merge.segments);
- // remove pending deletes of the segments
- // that were merged, moving them onto the segment just
- // before the merged segment
- // Lock order: IW -> BD
- bufferedDeletes.commitMerge(merge);
-
if (merge.optimize) {
// cascade the optimize:
segmentsToOptimize.add(merge.info);
}
+
+
return true;
}
@@ -3691,7 +3706,7 @@ public class IndexWriter implements Clos
}
}
- ensureContiguousMerge(merge);
+ ensureValidMerge(merge);
pendingMerges.add(merge);
@@ -3741,6 +3756,9 @@ public class IndexWriter implements Clos
throw new IllegalStateException("this writer hit an OutOfMemoryError; cannot merge");
}
+ // TODO: is there any perf benefit to sorting
+ // merged segments? eg biggest to smallest?
+
if (merge.info != null)
// mergeInit already done
return;
@@ -3748,11 +3766,6 @@ public class IndexWriter implements Clos
if (merge.isAborted())
return;
- // Lock order: IW -> BD
- if (bufferedDeletes.applyDeletes(readerPool, segmentInfos, merge.segments)) {
- checkpoint();
- }
-
boolean hasVectors = false;
for (SegmentInfo sourceSegment : merge.segments) {
if (sourceSegment.getHasVectors()) {
@@ -3765,6 +3778,17 @@ public class IndexWriter implements Clos
// names.
merge.info = new SegmentInfo(newSegmentName(), 0, directory, false, true, false, hasVectors);
+ // Lock order: IW -> BD
+ final BufferedDeletesStream.ApplyDeletesResult result = bufferedDeletesStream.applyDeletes(readerPool, merge.segments);
+ if (result.anyDeletes) {
+ checkpoint();
+ }
+
+ merge.info.setBufferedDeletesGen(result.gen);
+
+ // Lock order: IW -> BD
+ bufferedDeletesStream.prune(segmentInfos);
+
Map<String,String> details = new HashMap<String,String>();
details.put("optimize", Boolean.toString(merge.optimize));
details.put("mergeFactor", Integer.toString(merge.segments.size()));
@@ -4402,7 +4426,7 @@ public class IndexWriter implements Clos
}
synchronized boolean nrtIsCurrent(SegmentInfos infos) {
- return infos.version == segmentInfos.version && !docWriter.anyChanges() && !bufferedDeletes.any();
+ return infos.version == segmentInfos.version && !docWriter.anyChanges() && !bufferedDeletesStream.any();
}
synchronized boolean isClosed() {
@@ -4569,7 +4593,7 @@ public class IndexWriter implements Clos
final double ramBufferSizeMB = config.getRAMBufferSizeMB();
if (ramBufferSizeMB != IndexWriterConfig.DISABLE_AUTO_FLUSH) {
final long limit = (long) (ramBufferSizeMB*1024*1024);
- long used = bufferedDeletes.bytesUsed() + docWriter.bytesUsed();
+ long used = bufferedDeletesStream.bytesUsed() + docWriter.bytesUsed();
if (used >= limit) {
// DocumentsWriter may be able to free up some
@@ -4577,7 +4601,7 @@ public class IndexWriter implements Clos
// Lock order: FC -> DW
docWriter.balanceRAM();
- used = bufferedDeletes.bytesUsed() + docWriter.bytesUsed();
+ used = bufferedDeletesStream.bytesUsed() + docWriter.bytesUsed();
if (used >= limit) {
return setFlushPending("ram full: " + reason, false);
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/LogMergePolicy.java Thu May 5 09:09:41 2011
@@ -19,6 +19,8 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Set;
+import java.util.Arrays;
+import java.util.Comparator;
/** <p>This class implements a {@link MergePolicy} that tries
* to merge segments into levels of exponentially
@@ -67,6 +69,7 @@ public abstract class LogMergePolicy ext
// out there wrote his own LMP ...
protected long maxMergeSizeForOptimize = Long.MAX_VALUE;
protected int maxMergeDocs = DEFAULT_MAX_MERGE_DOCS;
+ protected boolean requireContiguousMerge = true;
protected double noCFSRatio = DEFAULT_NO_CFS_RATIO;
@@ -105,6 +108,21 @@ public abstract class LogMergePolicy ext
writer.get().message("LMP: " + message);
}
+ /** If true, merges must be in-order slice of the
+ * segments. If false, then the merge policy is free to
+ * pick any segments. The default is false, which is
+ * in general more efficient than true since it gives the
+ * merge policy more freedom to pick closely sized
+ * segments. */
+ public void setRequireContiguousMerge(boolean v) {
+ requireContiguousMerge = v;
+ }
+
+ /** See {@link #setRequireContiguousMerge}. */
+ public boolean getRequireContiguousMerge() {
+ return requireContiguousMerge;
+ }
+
/** <p>Returns the number of segments that are merged at
* once and also controls the total number of segments
* allowed to accumulate in the index.</p> */
@@ -348,6 +366,8 @@ public abstract class LogMergePolicy ext
// If the segments are already optimized (e.g. there's only 1 segment), or
// there are <maxNumSegements, all optimized, nothing to do.
if (isOptimized(infos, maxNumSegments, segmentsToOptimize)) return null;
+
+ // TODO: handle non-contiguous merge case differently?
// Find the newest (rightmost) segment that needs to
// be optimized (other segments may have been flushed
@@ -436,6 +456,37 @@ public abstract class LogMergePolicy ext
return spec;
}
+ private static class SegmentInfoAndLevel implements Comparable {
+ SegmentInfo info;
+ float level;
+ int index;
+
+ public SegmentInfoAndLevel(SegmentInfo info, float level, int index) {
+ this.info = info;
+ this.level = level;
+ this.index = index;
+ }
+
+ // Sorts largest to smallest
+ public int compareTo(Object o) {
+ SegmentInfoAndLevel other = (SegmentInfoAndLevel) o;
+ if (level < other.level)
+ return 1;
+ else if (level > other.level)
+ return -1;
+ else
+ return 0;
+ }
+ }
+
+ private static class SortByIndex implements Comparator<SegmentInfoAndLevel> {
+ public int compare(SegmentInfoAndLevel o1, SegmentInfoAndLevel o2) {
+ return o1.index - o2.index;
+ }
+ }
+
+ private static final SortByIndex sortByIndex = new SortByIndex();
+
/** Checks if any merges are now necessary and returns a
* {@link MergePolicy.MergeSpecification} if so. A merge
* is necessary when there are more than {@link
@@ -452,7 +503,7 @@ public abstract class LogMergePolicy ext
// Compute levels, which is just log (base mergeFactor)
// of the size of each segment
- float[] levels = new float[numSegments];
+ SegmentInfoAndLevel[] levels = new SegmentInfoAndLevel[numSegments];
final float norm = (float) Math.log(mergeFactor);
for(int i=0;i<numSegments;i++) {
@@ -462,7 +513,12 @@ public abstract class LogMergePolicy ext
// Floor tiny segments
if (size < 1)
size = 1;
- levels[i] = (float) Math.log(size)/norm;
+ levels[i] = new SegmentInfoAndLevel(info, (float) Math.log(size)/norm, i);
+ message("seg " + info.name + " level=" + levels[i].level + " size=" + size);
+ }
+
+ if (!requireContiguousMerge) {
+ Arrays.sort(levels);
}
final float levelFloor;
@@ -485,9 +541,9 @@ public abstract class LogMergePolicy ext
// Find max level of all segments not already
// quantized.
- float maxLevel = levels[start];
+ float maxLevel = levels[start].level;
for(int i=1+start;i<numSegments;i++) {
- final float level = levels[i];
+ final float level = levels[i].level;
if (level > maxLevel)
maxLevel = level;
}
@@ -508,7 +564,7 @@ public abstract class LogMergePolicy ext
int upto = numSegments-1;
while(upto >= start) {
- if (levels[upto] >= levelBottom) {
+ if (levels[upto].level >= levelBottom) {
break;
}
upto--;
@@ -521,18 +577,26 @@ public abstract class LogMergePolicy ext
while(end <= 1+upto) {
boolean anyTooLarge = false;
for(int i=start;i<end;i++) {
- final SegmentInfo info = infos.info(i);
+ final SegmentInfo info = levels[i].info;
anyTooLarge |= (size(info) >= maxMergeSize || sizeDocs(info) >= maxMergeDocs);
}
if (!anyTooLarge) {
if (spec == null)
spec = new MergeSpecification();
- if (verbose())
+ if (verbose()) {
message(" " + start + " to " + end + ": add this merge");
- spec.add(new OneMerge(infos.range(start, end)));
- } else if (verbose())
+ }
+ Arrays.sort(levels, start, end, sortByIndex);
+ final SegmentInfos mergeInfos = new SegmentInfos();
+ for(int i=start;i<end;i++) {
+ mergeInfos.add(levels[i].info);
+ assert infos.contains(levels[i].info);
+ }
+ spec.add(new OneMerge(mergeInfos));
+ } else if (verbose()) {
message(" " + start + " to " + end + ": contains segment over maxMergeSize or maxMergeDocs; skipping");
+ }
start = end;
end = start + mergeFactor;
@@ -580,6 +644,7 @@ public abstract class LogMergePolicy ext
sb.append("calibrateSizeByDeletes=").append(calibrateSizeByDeletes).append(", ");
sb.append("maxMergeDocs=").append(maxMergeDocs).append(", ");
sb.append("useCompoundFile=").append(useCompoundFile);
+ sb.append("requireContiguousMerge=").append(requireContiguousMerge);
sb.append("]");
return sb.toString();
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Thu May 5 09:09:41 2011
@@ -102,6 +102,10 @@ public final class SegmentInfo {
// afterwards ("3.0", "3.1" etc.).
// see Constants.LUCENE_MAIN_VERSION.
private String version;
+
+ // NOTE: only used in-RAM by IW to track buffered deletes;
+ // this is never written to/read from the Directory
+ private long bufferedDeletesGen;
public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, boolean hasSingleNormFile,
boolean hasProx, boolean hasVectors) {
@@ -856,5 +860,12 @@ public final class SegmentInfo {
public String getVersion() {
return version;
}
-
+
+ long getBufferedDeletesGen() {
+ return bufferedDeletesGen;
+ }
+
+ void setBufferedDeletesGen(long v) {
+ bufferedDeletesGen = v;
+ }
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java Thu May 5 09:09:41 2011
@@ -31,8 +31,6 @@ import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Field.Index;
import org.apache.lucene.document.Field.Store;
@@ -43,6 +41,7 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.index.LogByteSizeMergePolicy;
import org.apache.lucene.index.LogDocMergePolicy;
import org.apache.lucene.index.LogMergePolicy;
+import org.apache.lucene.index.MockRandomMergePolicy;
import org.apache.lucene.index.SerialMergeScheduler;
import org.apache.lucene.index.SlowMultiReaderWrapper;
import org.apache.lucene.search.BooleanQuery;
@@ -652,15 +651,11 @@ public abstract class LuceneTestCase ext
c.setMaxThreadStates(_TestUtil.nextInt(r, 1, 20));
}
- if (c.getMergePolicy() instanceof LogMergePolicy) {
- LogMergePolicy logmp = (LogMergePolicy) c.getMergePolicy();
- logmp.setUseCompoundFile(r.nextBoolean());
- logmp.setCalibrateSizeByDeletes(r.nextBoolean());
- if (r.nextInt(3) == 2) {
- logmp.setMergeFactor(2);
- } else {
- logmp.setMergeFactor(_TestUtil.nextInt(r, 2, 20));
- }
+ // TODO: turn this back on after all backports
+ if (false && r.nextBoolean()) {
+ c.setMergePolicy(new MockRandomMergePolicy(r));
+ } else {
+ c.setMergePolicy(newLogMergePolicy());
}
c.setReaderPooling(r.nextBoolean());
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestMergeSchedulerExternal.java Thu May 5 09:09:41 2011
@@ -95,8 +95,8 @@ public class TestMergeSchedulerExternal
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergeScheduler(new MyMergeScheduler())
- .setMaxBufferedDocs(2).setRAMBufferSizeMB(
- IndexWriterConfig.DISABLE_AUTO_FLUSH));
+ .setMaxBufferedDocs(2).setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH)
+ .setMergePolicy(newLogMergePolicy()));
LogMergePolicy logMP = (LogMergePolicy) writer.getConfig().getMergePolicy();
logMP.setMergeFactor(10);
for(int i=0;i<20;i++)
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearch.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearch.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearch.java Thu May 5 09:09:41 2011
@@ -75,8 +75,11 @@ public class TestSearch extends LuceneTe
Directory directory = newDirectory();
Analyzer analyzer = new MockAnalyzer(random);
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
- LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setUseCompoundFile(useCompoundFile);
+ MergePolicy mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+ }
+
IndexWriter writer = new IndexWriter(directory, conf);
String[] docs = {
@@ -91,6 +94,7 @@ public class TestSearch extends LuceneTe
for (int j = 0; j < docs.length; j++) {
Document d = new Document();
d.add(newField("contents", docs[j], Field.Store.YES, Field.Index.ANALYZED));
+ d.add(newField("id", ""+j, Field.Index.NOT_ANALYZED_NO_NORMS));
writer.addDocument(d);
}
writer.close();
@@ -107,18 +111,17 @@ public class TestSearch extends LuceneTe
};
ScoreDoc[] hits = null;
+ Sort sort = new Sort(new SortField[] {
+ SortField.FIELD_SCORE,
+ new SortField("id", SortField.INT)});
+
QueryParser parser = new QueryParser(TEST_VERSION_CURRENT, "contents", analyzer);
parser.setPhraseSlop(4);
for (int j = 0; j < queries.length; j++) {
Query query = parser.parse(queries[j]);
out.println("Query: " + query.toString("contents"));
- //DateFilter filter =
- // new DateFilter("modified", Time(1997,0,1), Time(1998,0,1));
- //DateFilter filter = DateFilter.Before("modified", Time(1997,00,01));
- //System.out.println(filter);
-
- hits = searcher.search(query, null, 1000).scoreDocs;
+ hits = searcher.search(query, null, 1000, sort).scoreDocs;
out.println(hits.length + " total results");
for (int i = 0 ; i < hits.length && i < 10; i++) {
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java Thu May 5 09:09:41 2011
@@ -80,8 +80,10 @@ public class TestSearchForDuplicates ext
Directory directory = newDirectory();
Analyzer analyzer = new MockAnalyzer(random);
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, analyzer);
- LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setUseCompoundFile(useCompoundFiles);
+ final MergePolicy mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFiles);
+ }
IndexWriter writer = new IndexWriter(directory, conf);
if (VERBOSE) {
System.out.println("TEST: now build index");
@@ -106,7 +108,11 @@ public class TestSearchForDuplicates ext
Query query = parser.parse(HIGH_PRIORITY);
out.println("Query: " + query.toString(PRIORITY_FIELD));
- ScoreDoc[] hits = searcher.search(query, null, MAX_DOCS).scoreDocs;
+ final Sort sort = new Sort(new SortField[] {
+ SortField.FIELD_SCORE,
+ new SortField(ID_FIELD, SortField.INT)});
+
+ ScoreDoc[] hits = searcher.search(query, null, MAX_DOCS, sort).scoreDocs;
printHits(out, hits, searcher);
checkHits(hits, MAX_DOCS, searcher);
@@ -121,7 +127,7 @@ public class TestSearchForDuplicates ext
query = parser.parse(HIGH_PRIORITY + " OR " + MED_PRIORITY);
out.println("Query: " + query.toString(PRIORITY_FIELD));
- hits = searcher.search(query, null, MAX_DOCS).scoreDocs;
+ hits = searcher.search(query, null, MAX_DOCS, sort).scoreDocs;
printHits(out, hits, searcher);
checkHits(hits, MAX_DOCS, searcher);
@@ -143,7 +149,7 @@ public class TestSearchForDuplicates ext
private void checkHits(ScoreDoc[] hits, int expectedCount, Searcher searcher) throws IOException {
assertEquals("total results", expectedCount, hits.length);
for (int i = 0 ; i < hits.length; i++) {
- if ( i < 10 || (i > 94 && i < 105) ) {
+ if (i < 10 || (i > 94 && i < 105) ) {
Document d = searcher.doc(hits[i].doc);
assertEquals("check " + i, String.valueOf(i), d.get(ID_FIELD));
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAtomicUpdate.java Thu May 5 09:09:41 2011
@@ -20,7 +20,6 @@ import org.apache.lucene.util.*;
import org.apache.lucene.store.*;
import org.apache.lucene.document.*;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.SimpleAnalyzer;
import java.util.Random;
import java.io.File;
@@ -132,6 +131,7 @@ public class TestAtomicUpdate extends Lu
.setMaxBufferedDocs(7);
((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(3);
IndexWriter writer = new MockIndexWriter(directory, conf);
+ writer.setInfoStream(VERBOSE ? System.out : null);
// Establish a base index of 100 docs:
for(int i=0;i<100;i++) {
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java Thu May 5 09:09:41 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.index;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@@ -133,11 +132,15 @@ public class TestConcurrentMergeSchedule
IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random))
.setMergePolicy(mp));
+ writer.setInfoStream(VERBOSE ? System.out : null);
Document doc = new Document();
Field idField = newField("id", "", Field.Store.YES, Field.Index.NOT_ANALYZED);
doc.add(idField);
for(int i=0;i<10;i++) {
+ if (VERBOSE) {
+ System.out.println("\nTEST: cycle");
+ }
for(int j=0;j<100;j++) {
idField.setValue(Integer.toString(i*100+j));
writer.addDocument(doc);
@@ -145,6 +148,9 @@ public class TestConcurrentMergeSchedule
int delID = i;
while(delID < 100*(1+i)) {
+ if (VERBOSE) {
+ System.out.println("TEST: del " + delID);
+ }
writer.deleteDocuments(new Term("id", ""+delID));
delID += 10;
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java Thu May 5 09:09:41 2011
@@ -120,6 +120,9 @@ public class TestDeletionPolicy extends
}
public void onInit(List<? extends IndexCommit> commits) throws IOException {
+ if (VERBOSE) {
+ System.out.println("TEST: onInit");
+ }
verifyCommitOrder(commits);
numOnInit++;
// do no deletions on init
@@ -127,6 +130,9 @@ public class TestDeletionPolicy extends
}
public void onCommit(List<? extends IndexCommit> commits) throws IOException {
+ if (VERBOSE) {
+ System.out.println("TEST: onCommit");
+ }
verifyCommitOrder(commits);
doDeletes(commits, true);
}
@@ -201,8 +207,10 @@ public class TestDeletionPolicy extends
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random))
.setIndexDeletionPolicy(policy);
- LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setUseCompoundFile(true);
+ MergePolicy mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ ((LogMergePolicy) mp).setUseCompoundFile(true);
+ }
IndexWriter writer = new IndexWriter(dir, conf);
writer.close();
@@ -216,8 +224,10 @@ public class TestDeletionPolicy extends
conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
new MockAnalyzer(random)).setOpenMode(
OpenMode.APPEND).setIndexDeletionPolicy(policy);
- lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setUseCompoundFile(true);
+ mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ ((LogMergePolicy) mp).setUseCompoundFile(true);
+ }
writer = new IndexWriter(dir, conf);
for(int j=0;j<17;j++) {
addDoc(writer);
@@ -281,6 +291,10 @@ public class TestDeletionPolicy extends
public void testKeepAllDeletionPolicy() throws IOException {
for(int pass=0;pass<2;pass++) {
+ if (VERBOSE) {
+ System.out.println("TEST: cycle pass=" + pass);
+ }
+
boolean useCompoundFile = (pass % 2) != 0;
// Never deletes a commit
@@ -293,34 +307,49 @@ public class TestDeletionPolicy extends
TEST_VERSION_CURRENT, new MockAnalyzer(random))
.setIndexDeletionPolicy(policy).setMaxBufferedDocs(10)
.setMergeScheduler(new SerialMergeScheduler());
- LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setUseCompoundFile(useCompoundFile);
- lmp.setMergeFactor(10);
+ MergePolicy mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+ }
IndexWriter writer = new IndexWriter(dir, conf);
for(int i=0;i<107;i++) {
addDoc(writer);
}
writer.close();
- conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
- new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(
- OpenMode.APPEND).setIndexDeletionPolicy(policy);
- lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setUseCompoundFile(useCompoundFile);
- writer = new IndexWriter(dir, conf);
- writer.optimize();
- writer.close();
-
- assertEquals(1, policy.numOnInit);
+ final boolean isOptimized;
+ {
+ IndexReader r = IndexReader.open(dir);
+ isOptimized = r.isOptimized();
+ r.close();
+ }
+ if (!isOptimized) {
+ conf = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(
+ OpenMode.APPEND).setIndexDeletionPolicy(policy)
+ .setMergePolicy(newLogMergePolicy());
+ mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+ }
+ if (VERBOSE) {
+ System.out.println("TEST: open writer for optimize");
+ }
+ writer = new IndexWriter(dir, conf);
+ writer.setInfoStream(VERBOSE ? System.out : null);
+ writer.optimize();
+ writer.close();
+ }
+ assertEquals(isOptimized ? 0:1, policy.numOnInit);
// If we are not auto committing then there should
// be exactly 2 commits (one per close above):
- assertEquals(2, policy.numOnCommit);
+ assertEquals(1 + (isOptimized ? 0:1), policy.numOnCommit);
// Test listCommits
Collection<IndexCommit> commits = IndexReader.listCommits(dir);
// 2 from closing writer
- assertEquals(2, commits.size());
+ assertEquals(1 + (isOptimized ? 0:1), commits.size());
// Make sure we can open a reader on each commit:
for (final IndexCommit commit : commits) {
@@ -481,8 +510,10 @@ public class TestDeletionPolicy extends
TEST_VERSION_CURRENT, new MockAnalyzer(random))
.setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
.setMaxBufferedDocs(10);
- LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setUseCompoundFile(useCompoundFile);
+ MergePolicy mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+ }
IndexWriter writer = new IndexWriter(dir, conf);
for(int i=0;i<107;i++) {
addDoc(writer);
@@ -491,8 +522,10 @@ public class TestDeletionPolicy extends
conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
.setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
- lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setUseCompoundFile(useCompoundFile);
+ mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ ((LogMergePolicy) mp).setUseCompoundFile(true);
+ }
writer = new IndexWriter(dir, conf);
writer.optimize();
writer.close();
@@ -530,8 +563,10 @@ public class TestDeletionPolicy extends
TEST_VERSION_CURRENT, new MockAnalyzer(random))
.setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
.setMaxBufferedDocs(10);
- LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setUseCompoundFile(useCompoundFile);
+ MergePolicy mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+ }
IndexWriter writer = new IndexWriter(dir, conf);
for(int i=0;i<17;i++) {
addDoc(writer);
@@ -587,24 +622,34 @@ public class TestDeletionPolicy extends
IndexWriterConfig conf = newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random))
.setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy);
- LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setUseCompoundFile(useCompoundFile);
+ MergePolicy mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+ }
IndexWriter writer = new IndexWriter(dir, conf);
writer.close();
Term searchTerm = new Term("content", "aaa");
Query query = new TermQuery(searchTerm);
for(int i=0;i<N+1;i++) {
+ if (VERBOSE) {
+ System.out.println("\nTEST: cycle i=" + i);
+ }
conf = newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random))
.setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
- lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setUseCompoundFile(useCompoundFile);
+ mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+ }
writer = new IndexWriter(dir, conf);
for(int j=0;j<17;j++) {
addDoc(writer);
}
// this is a commit
+ if (VERBOSE) {
+ System.out.println("TEST: close writer");
+ }
writer.close();
IndexReader reader = IndexReader.open(dir, policy, false);
reader.deleteDocument(3*i+1);
@@ -613,20 +658,28 @@ public class TestDeletionPolicy extends
ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
assertEquals(16*(1+i), hits.length);
// this is a commit
+ if (VERBOSE) {
+ System.out.println("TEST: close reader numOnCommit=" + policy.numOnCommit);
+ }
reader.close();
searcher.close();
}
conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))
.setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy);
- lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setUseCompoundFile(useCompoundFile);
+ mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+ }
+ IndexReader r = IndexReader.open(dir);
+ final boolean wasOptimized = r.isOptimized();
+ r.close();
writer = new IndexWriter(dir, conf);
writer.optimize();
// this is a commit
writer.close();
assertEquals(2*(N+1)+1, policy.numOnInit);
- assertEquals(2*(N+2), policy.numOnCommit);
+ assertEquals(2*(N+2) - (wasOptimized ? 1:0), policy.numOnCommit);
IndexSearcher searcher = new IndexSearcher(dir, false);
ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
@@ -693,8 +746,10 @@ public class TestDeletionPolicy extends
TEST_VERSION_CURRENT, new MockAnalyzer(random))
.setOpenMode(OpenMode.CREATE).setIndexDeletionPolicy(policy)
.setMaxBufferedDocs(10);
- LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setUseCompoundFile(useCompoundFile);
+ MergePolicy mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+ }
IndexWriter writer = new IndexWriter(dir, conf);
writer.close();
Term searchTerm = new Term("content", "aaa");
@@ -706,8 +761,10 @@ public class TestDeletionPolicy extends
TEST_VERSION_CURRENT, new MockAnalyzer(random))
.setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(policy)
.setMaxBufferedDocs(10);
- lmp = (LogMergePolicy) conf.getMergePolicy();
- lmp.setUseCompoundFile(useCompoundFile);
+ mp = conf.getMergePolicy();
+ if (mp instanceof LogMergePolicy) {
+ ((LogMergePolicy) mp).setUseCompoundFile(useCompoundFile);
+ }
writer = new IndexWriter(dir, conf);
for(int j=0;j<17;j++) {
addDoc(writer);
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java Thu May 5 09:09:41 2011
@@ -107,8 +107,7 @@ public class TestIndexReaderCloneNorms e
createIndex(random, dir3);
IndexWriter iw = new IndexWriter(dir3, newIndexWriterConfig(
TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.APPEND)
- .setMaxBufferedDocs(5));
- ((LogMergePolicy) iw.getConfig().getMergePolicy()).setMergeFactor(3);
+ .setMaxBufferedDocs(5).setMergePolicy(newLogMergePolicy(3)));
iw.addIndexes(new Directory[] { dir1, dir2 });
iw.optimize();
iw.close();
@@ -125,8 +124,7 @@ public class TestIndexReaderCloneNorms e
// now with optimize
iw = new IndexWriter(dir3, newIndexWriterConfig( TEST_VERSION_CURRENT,
- anlzr).setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(5));
- ((LogMergePolicy) iw.getConfig().getMergePolicy()).setMergeFactor(3);
+ anlzr).setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(5).setMergePolicy(newLogMergePolicy(3)));
iw.optimize();
iw.close();
verifyIndex(dir3);
@@ -230,7 +228,7 @@ public class TestIndexReaderCloneNorms e
private void createIndex(Random random, Directory dir) throws IOException {
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.CREATE)
- .setMaxBufferedDocs(5).setSimilarity(similarityOne));
+ .setMaxBufferedDocs(5).setSimilarity(similarityOne).setMergePolicy(newLogMergePolicy()));
LogMergePolicy lmp = (LogMergePolicy) iw.getConfig().getMergePolicy();
lmp.setMergeFactor(3);
lmp.setUseCompoundFile(true);
@@ -282,7 +280,7 @@ public class TestIndexReaderCloneNorms e
throws IOException {
IndexWriterConfig conf = newIndexWriterConfig(
TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.APPEND)
- .setMaxBufferedDocs(5).setSimilarity(similarityOne);
+ .setMaxBufferedDocs(5).setSimilarity(similarityOne).setMergePolicy(newLogMergePolicy());
LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
lmp.setMergeFactor(3);
lmp.setUseCompoundFile(compound);
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Thu May 5 09:09:41 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.index;
*/
import java.io.ByteArrayOutputStream;
-import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.io.Reader;
@@ -35,7 +34,6 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.SimpleAnalyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockFixedLengthPayloadFilter;
import org.apache.lucene.analysis.TokenStream;
@@ -1345,9 +1343,9 @@ public class TestIndexWriter extends Luc
}
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy()));
writer.setInfoStream(VERBOSE ? System.out : null);
- LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
- lmp.setMergeFactor(2);
- lmp.setUseCompoundFile(false);
+ //LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
+ //lmp.setMergeFactor(2);
+ //lmp.setUseCompoundFile(false);
Document doc = new Document();
String contents = "aa bb cc dd ee ff gg hh ii jj kk";
@@ -1404,7 +1402,7 @@ public class TestIndexWriter extends Luc
IndexWriterConfig conf = newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE)
- .setMaxBufferedDocs(2);
+ .setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy());
if (pass == 2) {
conf.setMergeScheduler(new SerialMergeScheduler());
}
@@ -1580,8 +1578,8 @@ public class TestIndexWriter extends Luc
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random))
.setMaxBufferedDocs(2).setRAMBufferSizeMB(
- IndexWriterConfig.DISABLE_AUTO_FLUSH));
-
+ IndexWriterConfig.DISABLE_AUTO_FLUSH));
+ writer.setInfoStream(VERBOSE ? System.out : null);
Document document = new Document();
document = new Document();
@@ -2634,11 +2632,6 @@ public class TestIndexWriter extends Luc
dir.close();
}
- // both start & end are inclusive
- private final int getInt(Random r, int start, int end) {
- return start + r.nextInt(1+end-start);
- }
-
public void testDeleteUnusedFiles() throws Exception {
for(int iter=0;iter<2;iter++) {
@@ -2827,6 +2820,8 @@ public class TestIndexWriter extends Luc
.setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy()));
String[] files = dir.listAll();
+ writer.setInfoStream(VERBOSE ? System.out : null);
+
// Creating over empty dir should not create any files,
// or, at most the write.lock file
final int extraFileCount;
@@ -2848,9 +2843,10 @@ public class TestIndexWriter extends Luc
doc = new Document();
doc.add(newField("c", "val", Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));
writer.addDocument(doc);
+
// The second document should cause a flush.
- assertTrue("flush should have occurred and files created", dir.listAll().length > 5 + extraFileCount);
-
+ assertTrue("flush should have occurred and files should have been created", dir.listAll().length > 5 + extraFileCount);
+
// After rollback, IW should remove all files
writer.rollback();
assertEquals("no files should exist in the directory after rollback", 0, dir.listAll().length);
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1099728&r1=1099727&r2=1099728&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Thu May 5 09:09:41 2011
@@ -286,6 +286,7 @@ public class TestIndexWriterExceptions e
public void testExceptionDocumentsWriterInit() throws IOException {
Directory dir = newDirectory();
MockIndexWriter2 w = new MockIndexWriter2(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)));
+ w.setInfoStream(VERBOSE ? System.out : null);
Document doc = new Document();
doc.add(newField("field", "a field", Field.Store.YES,
Field.Index.ANALYZED));
@@ -357,7 +358,7 @@ public class TestIndexWriterExceptions e
public void testExceptionOnMergeInit() throws IOException {
Directory dir = newDirectory();
IndexWriterConfig conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))
- .setMaxBufferedDocs(2).setMergeScheduler(new ConcurrentMergeScheduler());
+ .setMaxBufferedDocs(2).setMergeScheduler(new ConcurrentMergeScheduler()).setMergePolicy(newLogMergePolicy());
((LogMergePolicy) conf.getMergePolicy()).setMergeFactor(2);
MockIndexWriter3 w = new MockIndexWriter3(dir, conf);
w.doFail = true;
@@ -521,7 +522,7 @@ public class TestIndexWriterExceptions e
System.out.println("TEST: cycle i=" + i);
}
MockDirectoryWrapper dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer));
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setMergePolicy(newLogMergePolicy()));
writer.setInfoStream(VERBOSE ? System.out : null);
// don't allow a sudden merge to clean up the deleted