You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by bu...@apache.org on 2011/01/16 02:46:00 UTC
svn commit: r1059431 - in
/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index:
DocumentsWriter.java IndexWriter.java SegmentInfo.java
TermVectorsTermsWriter.java TermVectorsWriter.java
Author: buschmi
Date: Sun Jan 16 01:46:00 2011
New Revision: 1059431
URL: http://svn.apache.org/viewvc?rev=1059431&view=rev
Log:
LUCENE-2324: several bug fixes
Modified:
lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1059431&r1=1059430&r2=1059431&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Sun Jan 16 01:46:00 2011
@@ -370,13 +370,11 @@ final class DocumentsWriter {
long perThreadRAMUsedBeforeAdd = dwpt.bytesUsed();
dwpt.addDocument(doc, analyzer);
- synchronized(DocumentsWriter.this) {
- if (delTerm != null) {
- deleteTerm(delTerm);
- }
- dwpt.commitDocument();
- numDocsInRAM.incrementAndGet();
+ if (delTerm != null) {
+ deleteTerm(delTerm);
}
+ dwpt.commitDocument();
+ numDocsInRAM.incrementAndGet();
newSegment = finishAddDocument(dwpt, perThreadRAMUsedBeforeAdd);
if (newSegment != null) {
@@ -502,19 +500,11 @@ final class DocumentsWriter {
"reating compound file for newly flushed segment " + newSegment.name);
}
- indexWriter.deleter.deleteFile(IndexFileNames.segmentFileName(newSegment.name, "",
- IndexFileNames.COMPOUND_FILE_EXTENSION));
- for (String file : newSegment.files()) {
- indexWriter.deleter.deleteFile(file);
- }
-
+ indexWriter.deleter.refresh(newSegment.name);
}
}
- for (String file : newSegment.files()) {
- indexWriter.deleter.deleteFile(file);
- }
-
+ indexWriter.deleter.deleteNewFiles(newSegment.files());
newSegment.setUseCompoundFile(true);
}
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1059431&r1=1059430&r2=1059431&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/IndexWriter.java Sun Jan 16 01:46:00 2011
@@ -1396,9 +1396,9 @@ public class IndexWriter implements Clos
public void updateDocument(Term term, Document doc, Analyzer analyzer)
throws CorruptIndexException, IOException {
ensureOpen();
- boolean maybeMerge = false;
try {
boolean success = false;
+ boolean maybeMerge = false;
try {
maybeMerge = docWriter.updateDocument(doc, analyzer, term);
success = true;
@@ -1406,13 +1406,13 @@ public class IndexWriter implements Clos
if (!success && infoStream != null)
message("hit exception updating document");
}
+
+ if (maybeMerge) {
+ maybeMerge();
+ }
} catch (OutOfMemoryError oom) {
handleOOM(oom, "updateDocument");
}
-
- if (maybeMerge) {
- maybeMerge();
- }
}
// for test purpose
@@ -2186,6 +2186,9 @@ public class IndexWriter implements Clos
}
// Update SI appropriately
+ // if this call is removed in the future we need to make
+ // sure that info.clearFiles() is called here
+ info.setDocStore(info.getDocStoreOffset(), newDsName, info.getDocStoreIsCompoundFile());
info.dir = directory;
info.name = newSegName;
@@ -2501,7 +2504,6 @@ public class IndexWriter implements Clos
boolean maybeMerge = docWriter.flushAllThreads(applyAllDeletes);
synchronized(this) {
-
if (!applyAllDeletes) {
// If deletes alone are consuming > 1/2 our RAM
// buffer, force them all to apply now. This is to
@@ -2536,8 +2538,8 @@ public class IndexWriter implements Clos
success = true;
return maybeMerge;
- }
+ }
} catch (OutOfMemoryError oom) {
handleOOM(oom, "doFlush");
// never hit
@@ -2697,6 +2699,12 @@ public class IndexWriter implements Clos
commitMergedDeletes(merge, mergedReader);
+ // If the doc store we are using has been closed and
+ // is in now compound format (but wasn't when we
+ // started), then we will switch to the compound
+ // format as well:
+ setMergeDocStoreIsCompoundFile(merge);
+
segmentInfos.subList(start, start + merge.segments.size()).clear();
assert !segmentInfos.contains(merge.info);
segmentInfos.add(start, merge.info);
@@ -2972,6 +2980,23 @@ public class IndexWriter implements Clos
runningMerges.remove(merge);
}
+ private synchronized void setMergeDocStoreIsCompoundFile(MergePolicy.OneMerge merge) {
+ final String mergeDocStoreSegment = merge.info.getDocStoreSegment();
+ if (mergeDocStoreSegment != null && !merge.info.getDocStoreIsCompoundFile()) {
+ final int size = segmentInfos.size();
+ for(int i=0;i<size;i++) {
+ final SegmentInfo info = segmentInfos.info(i);
+ final String docStoreSegment = info.getDocStoreSegment();
+ if (docStoreSegment != null &&
+ docStoreSegment.equals(mergeDocStoreSegment) &&
+ info.getDocStoreIsCompoundFile()) {
+ merge.info.setDocStoreIsCompoundFile(true);
+ break;
+ }
+ }
+ }
+ }
+
private synchronized void closeMergeReaders(MergePolicy.OneMerge merge, boolean suppressExceptions) throws IOException {
final int numSegments = merge.segments.size();
if (suppressExceptions) {
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1059431&r1=1059430&r2=1059431&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Sun Jan 16 01:46:00 2011
@@ -409,6 +409,20 @@ public final class SegmentInfo {
}
@Deprecated
+ public void setDocStoreIsCompoundFile(boolean docStoreIsCompoundFile) {
+ this.docStoreIsCompoundFile = docStoreIsCompoundFile;
+ clearFiles();
+ }
+
+ @Deprecated
+ void setDocStore(int offset, String segment, boolean isCompoundFile) {
+ docStoreOffset = offset;
+ docStoreSegment = segment;
+ docStoreIsCompoundFile = isCompoundFile;
+ clearFiles();
+ }
+
+ @Deprecated
public String getDocStoreSegment() {
return docStoreSegment;
}
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java?rev=1059431&r1=1059430&r2=1059431&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java Sun Jan 16 01:46:00 2011
@@ -112,6 +112,10 @@ final class TermVectorsTermsWriter exten
assert docWriter.writer.testPoint("TermVectorsTermsWriter.finishDocument start");
+ if (numVectorFields == 0) {
+ return;
+ }
+
initTermVectorsWriter();
fill(docState.docID);
Modified: lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java?rev=1059431&r1=1059430&r2=1059431&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java (original)
+++ lucene/dev/branches/realtime_search/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java Sun Jan 16 01:46:00 2011
@@ -20,12 +20,13 @@ package org.apache.lucene.index;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.StringHelper;
import java.io.IOException;
final class TermVectorsWriter {
-
+
private IndexOutput tvx = null, tvd = null, tvf = null;
private FieldInfos fieldInfos;
@@ -46,7 +47,7 @@ final class TermVectorsWriter {
/**
* Add a complete document specified by all its term vectors. If document has no
* term vectors, add value for tvx.
- *
+ *
* @param vectors
* @throws IOException
*/
@@ -99,7 +100,7 @@ final class TermVectorsWriter {
final int[] freqs = vectors[i].getTermFrequencies();
for (int j=0; j<numTerms; j++) {
-
+
int start = j == 0 ? 0 : StringHelper.bytesDifference(terms[j-1].bytes,
terms[j-1].length,
terms[j].bytes,
@@ -181,30 +182,11 @@ final class TermVectorsWriter {
assert tvd.getFilePointer() == tvdPosition;
assert tvf.getFilePointer() == tvfPosition;
}
-
+
/** Close all streams. */
final void close() throws IOException {
// make an effort to close all streams we can but remember and re-throw
// the first exception encountered in this process
- IOException keep = null;
- if (tvx != null)
- try {
- tvx.close();
- } catch (IOException e) {
- keep = e;
- }
- if (tvd != null)
- try {
- tvd.close();
- } catch (IOException e) {
- if (keep == null) keep = e;
- }
- if (tvf != null)
- try {
- tvf.close();
- } catch (IOException e) {
- if (keep == null) keep = e;
- }
- if (keep != null) throw (IOException) keep.fillInStackTrace();
+ IOUtils.closeSafely(tvx, tvd, tvf);
}
}