You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2010/12/17 17:16:00 UTC
svn commit: r1050439 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/src/test/org/apache/lucene/index/
lucene/src/java/org/apache/lucene/index/
lucene/src/test/org/apache/lucene/index/
lucene/src/test/org/apache/lucene/store/ solr/
Author: mikemccand
Date: Fri Dec 17 16:15:59 2010
New Revision: 1050439
URL: http://svn.apache.org/viewvc?rev=1050439&view=rev
Log:
LUCENE-2811: backport to 3.x
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentMerger.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDoc.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/store/MockIndexOutputWrapper.java
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1050439&r1=1050438&r2=1050439&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestSegmentMerger.java Fri Dec 17 16:15:59 2010
@@ -22,13 +22,9 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.document.Document;
-import java.io.IOException;
-import java.util.Collection;
-
public class TestSegmentMerger extends LuceneTestCase {
//The variables for the new merged segment
private Directory mergedDir = new RAMDirectory();
- private String mergedSegment = "test";
//First segment to be merged
private Directory merge1Dir = new RAMDirectory();
private Document doc1 = new Document();
@@ -61,53 +57,4 @@ public class TestSegmentMerger extends L
assertTrue(reader1 != null);
assertTrue(reader2 != null);
}
-
- public void testMerge() throws IOException {
- SegmentMerger merger = new SegmentMerger(mergedDir, mergedSegment);
- merger.add(reader1);
- merger.add(reader2);
- int docsMerged = merger.merge();
- assertTrue(docsMerged == 2);
- //Should be able to open a new SegmentReader against the new directory
- SegmentReader mergedReader = SegmentReader.get(true, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, true), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
- assertTrue(mergedReader != null);
- assertTrue(mergedReader.numDocs() == 2);
- Document newDoc1 = mergedReader.document(0);
- assertTrue(newDoc1 != null);
- //There are 2 unstored fields on the document
- assertTrue(DocHelper.numFields(newDoc1) == DocHelper.numFields(doc1) - DocHelper.unstored.size());
- Document newDoc2 = mergedReader.document(1);
- assertTrue(newDoc2 != null);
- assertTrue(DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - DocHelper.unstored.size());
-
- TermDocs termDocs = mergedReader.termDocs(new Term(DocHelper.TEXT_FIELD_2_KEY, "field"));
- assertTrue(termDocs != null);
- assertTrue(termDocs.next() == true);
-
- Collection stored = mergedReader.getFieldNames(IndexReader.FieldOption.INDEXED_WITH_TERMVECTOR);
- assertTrue(stored != null);
- //System.out.println("stored size: " + stored.size());
- assertTrue("We do not have 3 fields that were indexed with term vector",stored.size() == 3);
-
- TermFreqVector vector = mergedReader.getTermFreqVector(0, DocHelper.TEXT_FIELD_2_KEY);
- assertTrue(vector != null);
- String [] terms = vector.getTerms();
- assertTrue(terms != null);
- //System.out.println("Terms size: " + terms.length);
- assertTrue(terms.length == 3);
- int [] freqs = vector.getTermFrequencies();
- assertTrue(freqs != null);
- //System.out.println("Freqs size: " + freqs.length);
- assertTrue(vector instanceof TermPositionVector == true);
-
- for (int i = 0; i < terms.length; i++) {
- String term = terms[i];
- int freq = freqs[i];
- //System.out.println("Term: " + term + " Freq: " + freq);
- assertTrue(DocHelper.FIELD_2_TEXT.indexOf(term) != -1);
- assertTrue(DocHelper.FIELD_2_FREQS[i] == freq);
- }
-
- TestSegmentReader.checkNorms(mergedReader);
- }
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=1050439&r1=1050438&r2=1050439&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Fri Dec 17 16:15:59 2010
@@ -443,7 +443,7 @@ final class DocumentsWriter {
assert docStoreSegment != null;
if (infoStream != null) {
- message("closeDocStore: files=" + openFiles + "; segment=" + docStoreSegment + "; docStoreOffset=" + docStoreOffset + "; numDocsInStore=" + numDocsInStore + "; isSeparate=" + isSeparate);
+ message("closeDocStore: openFiles=" + openFiles + "; segment=" + docStoreSegment + "; docStoreOffset=" + docStoreOffset + "; numDocsInStore=" + numDocsInStore + "; isSeparate=" + isSeparate);
}
closedFiles.clear();
@@ -711,17 +711,19 @@ final class DocumentsWriter {
final SegmentWriteState flushState = new SegmentWriteState(this, directory, segment, docStoreSegment, numDocsInRAM, numDocsInStore, writer.getConfig().getTermIndexInterval());
- newSegment = new SegmentInfo(segment, numDocsInRAM, directory, false, true, -1, null, false, hasProx());
+ newSegment = new SegmentInfo(segment, numDocsInRAM, directory, false, true, -1, null, false, hasProx(), false);
if (!closeDocStore || docStoreOffset != 0) {
newSegment.setDocStoreSegment(docStoreSegment);
newSegment.setDocStoreOffset(docStoreOffset);
}
-
+
if (closeDocStore) {
closeDocStore(flushState, writer, deleter, newSegment, mergePolicy, segmentInfos);
}
+ boolean hasVectors = flushState.hasVectors;
+
if (numDocsInRAM > 0) {
assert nextDocID == numDocsInRAM;
@@ -740,6 +742,19 @@ final class DocumentsWriter {
final long startNumBytesUsed = bytesUsed();
consumer.flush(threads, flushState);
+ hasVectors |= flushState.hasVectors;
+
+ if (hasVectors) {
+ if (infoStream != null) {
+ message("new segment has vectors");
+ }
+ newSegment.setHasVectors(true);
+ } else {
+ if (infoStream != null) {
+ message("new segment has no vectors");
+ }
+ }
+
if (infoStream != null) {
message("flushedFiles=" + flushState.flushedFiles);
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1050439&r1=1050438&r2=1050439&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/IndexWriter.java Fri Dec 17 16:15:59 2010
@@ -1820,8 +1820,10 @@ public class IndexWriter implements Clos
mergePolicy.close();
- finishMerges(waitForMerges);
- stopMerges = true;
+ synchronized(this) {
+ finishMerges(waitForMerges);
+ stopMerges = true;
+ }
mergeScheduler.close();
@@ -2024,10 +2026,12 @@ public class IndexWriter implements Clos
synchronized (this) {
// If docWriter has some aborted files that were
// never incref'd, then we clean them up here
+ deleter.checkpoint(segmentInfos, false);
if (docWriter != null) {
final Collection<String> files = docWriter.abortedFiles();
- if (files != null)
+ if (files != null) {
deleter.deleteNewFiles(files);
+ }
}
}
}
@@ -2652,7 +2656,14 @@ public class IndexWriter implements Clos
}
try {
- finishMerges(false);
+ synchronized(this) {
+ finishMerges(false);
+ stopMerges = true;
+ }
+
+ if (infoStream != null ) {
+ message("rollback: done finish merges");
+ }
// Must pre-close these two, in case they increment
// changeCount so that we can then set it to false
@@ -2894,7 +2905,9 @@ public class IndexWriter implements Clos
int docCount = merger.merge(); // merge 'em
SegmentInfo info = new SegmentInfo(mergedName, docCount, directory,
- false, true, -1, null, false, merger.hasProx());
+ false, true, -1, null, false,
+ merger.fieldInfos().hasProx(),
+ merger.fieldInfos().hasVectors());
setDiagnostics(info, "addIndexes(IndexReader...)");
boolean useCompoundFile;
@@ -3333,8 +3346,16 @@ public class IndexWriter implements Clos
return false;
} finally {
flushControl.clearFlushPending();
- if (!success && infoStream != null) {
- message("hit exception during flush");
+ if (!success) {
+ if (infoStream != null) {
+ message("hit exception during flush");
+ }
+ if (docWriter != null) {
+ final Collection<String> files = docWriter.abortedFiles();
+ if (files != null) {
+ deleter.deleteNewFiles(files);
+ }
+ }
}
}
}
@@ -3489,8 +3510,6 @@ public class IndexWriter implements Clos
// format as well:
setMergeDocStoreIsCompoundFile(merge);
- merge.info.setHasProx(merger.hasProx());
-
segmentInfos.subList(start, start + merge.segments.size()).clear();
assert !segmentInfos.contains(merge.info);
segmentInfos.add(start, merge.info);
@@ -3718,6 +3737,7 @@ public class IndexWriter implements Clos
boolean mergeDocStores = false;
boolean doFlushDocStore = false;
+ boolean hasVectors = false;
final String currentDocStoreSegment = docWriter.getDocStoreSegment();
// Test each segment to be merged: check if we need to
@@ -3729,6 +3749,10 @@ public class IndexWriter implements Clos
if (si.hasDeletions())
mergeDocStores = true;
+ if (si.getHasVectors()) {
+ hasVectors = true;
+ }
+
// If it has its own (private) doc stores we must
// merge the doc stores
if (-1 == si.getDocStoreOffset())
@@ -3814,8 +3838,8 @@ public class IndexWriter implements Clos
docStoreOffset,
docStoreSegment,
docStoreIsCompoundFile,
- false);
-
+ false,
+ hasVectors);
Map<String,String> details = new HashMap<String,String>();
details.put("optimize", Boolean.toString(merge.optimize));
@@ -3823,6 +3847,10 @@ public class IndexWriter implements Clos
details.put("mergeDocStores", Boolean.toString(mergeDocStores));
setDiagnostics(merge.info, "merge", details);
+ if (infoStream != null) {
+ message("merge seg=" + merge.info.name + " mergeDocStores=" + mergeDocStores);
+ }
+
// Also enroll the merged segment into mergingSegments;
// this prevents it from getting selected for a merge
// after our merge is done but while we are building the
@@ -4042,6 +4070,11 @@ public class IndexWriter implements Clos
assert mergedDocCount == totDocCount;
+ // Very important to do this before opening the reader
+ // because codec must know if prox was written for
+ // this segment:
+ merge.info.setHasProx(merger.fieldInfos().hasProx());
+
boolean useCompoundFile;
synchronized (this) { // Guard segmentInfos
useCompoundFile = mergePolicy.useCompoundFile(segmentInfos, merge.info);
@@ -4224,7 +4257,7 @@ public class IndexWriter implements Clos
// are called, deleter should know about every
// file referenced by the current head
// segmentInfos:
- assert deleter.exists(fileName) : "IndexFileDeleter doesn't know about file " + fileName;
+ assert deleter.exists(fileName): "IndexFileDeleter doesn't know about file " + fileName;
}
return true;
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java?rev=1050439&r1=1050438&r2=1050439&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/MergePolicy.java Fri Dec 17 16:15:59 2010
@@ -156,6 +156,9 @@ public abstract class MergePolicy implem
if (mergeDocStores) {
b.append(" [mergeDocStores]");
}
+ if (aborted) {
+ b.append(" [ABORTED]");
+ }
return b.toString();
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1050439&r1=1050438&r2=1050439&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Fri Dec 17 16:15:59 2010
@@ -88,38 +88,26 @@ public final class SegmentInfo {
private boolean hasProx; // True if this segment has any fields with omitTermFreqAndPositions==false
+ private boolean hasVectors; // True if this segment wrote term vectors
+
private Map<String,String> diagnostics;
- public SegmentInfo(String name, int docCount, Directory dir) {
+ public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, boolean hasSingleNormFile,
+ int docStoreOffset, String docStoreSegment, boolean docStoreIsCompoundFile, boolean hasProx,
+ boolean hasVectors) {
this.name = name;
this.docCount = docCount;
this.dir = dir;
delGen = NO;
- isCompoundFile = CHECK_DIR;
- preLockless = true;
- hasSingleNormFile = false;
- docStoreOffset = -1;
- docStoreSegment = name;
- docStoreIsCompoundFile = false;
- delCount = 0;
- hasProx = true;
- }
-
- public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, boolean hasSingleNormFile) {
- this(name, docCount, dir, isCompoundFile, hasSingleNormFile, -1, null, false, true);
- }
-
- public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, boolean hasSingleNormFile,
- int docStoreOffset, String docStoreSegment, boolean docStoreIsCompoundFile, boolean hasProx) {
- this(name, docCount, dir);
this.isCompoundFile = (byte) (isCompoundFile ? YES : NO);
- this.hasSingleNormFile = hasSingleNormFile;
preLockless = false;
+ this.hasSingleNormFile = hasSingleNormFile;
this.docStoreOffset = docStoreOffset;
this.docStoreSegment = docStoreSegment;
this.docStoreIsCompoundFile = docStoreIsCompoundFile;
- this.hasProx = hasProx;
delCount = 0;
+ this.hasProx = hasProx;
+ this.hasVectors = hasVectors;
assert docStoreOffset == -1 || docStoreSegment != null: "dso=" + docStoreOffset + " dss=" + docStoreSegment + " docCount=" + docCount;
}
@@ -135,6 +123,8 @@ public final class SegmentInfo {
delGen = src.delGen;
docStoreOffset = src.docStoreOffset;
docStoreIsCompoundFile = src.docStoreIsCompoundFile;
+ hasVectors = src.hasVectors;
+ hasProx = src.hasProx;
if (src.normGen == null) {
normGen = null;
} else {
@@ -213,6 +203,36 @@ public final class SegmentInfo {
} else {
diagnostics = Collections.<String,String>emptyMap();
}
+
+ if (format <= SegmentInfos.FORMAT_HAS_VECTORS) {
+ hasVectors = input.readByte() == 1;
+ } else {
+ final String storesSegment;
+ final String ext;
+ final boolean isCompoundFile;
+ if (docStoreOffset != -1) {
+ storesSegment = docStoreSegment;
+ isCompoundFile = docStoreIsCompoundFile;
+ ext = IndexFileNames.COMPOUND_FILE_STORE_EXTENSION;
+ } else {
+ storesSegment = name;
+ isCompoundFile = getUseCompoundFile();
+ ext = IndexFileNames.COMPOUND_FILE_EXTENSION;
+ }
+ final Directory dirToTest;
+ if (isCompoundFile) {
+ dirToTest = new CompoundFileReader(dir, IndexFileNames.segmentFileName(storesSegment, ext));
+ } else {
+ dirToTest = dir;
+ }
+ try {
+ hasVectors = dirToTest.fileExists(IndexFileNames.segmentFileName(storesSegment, IndexFileNames.VECTORS_INDEX_EXTENSION));
+ } finally {
+ if (isCompoundFile) {
+ dirToTest.close();
+ }
+ }
+ }
} else {
delGen = CHECK_DIR;
normGen = null;
@@ -267,6 +287,15 @@ public final class SegmentInfo {
return sizeInBytes;
}
+ public boolean getHasVectors() throws IOException {
+ return hasVectors;
+ }
+
+ public void setHasVectors(boolean v) {
+ hasVectors = v;
+ clearFiles();
+ }
+
public boolean hasDeletions()
throws IOException {
// Cases:
@@ -308,21 +337,18 @@ public final class SegmentInfo {
}
@Override
- public Object clone () {
- SegmentInfo si = new SegmentInfo(name, docCount, dir);
- si.isCompoundFile = isCompoundFile;
+ public Object clone() {
+ SegmentInfo si = new SegmentInfo(name, docCount, dir, false, hasSingleNormFile,
+ docStoreOffset, docStoreSegment, docStoreIsCompoundFile,
+ hasProx, hasVectors);
si.delGen = delGen;
si.delCount = delCount;
- si.hasProx = hasProx;
si.preLockless = preLockless;
- si.hasSingleNormFile = hasSingleNormFile;
+ si.isCompoundFile = isCompoundFile;
si.diagnostics = new HashMap<String, String>(diagnostics);
if (normGen != null) {
si.normGen = normGen.clone();
}
- si.docStoreOffset = docStoreOffset;
- si.docStoreSegment = docStoreSegment;
- si.docStoreIsCompoundFile = docStoreIsCompoundFile;
return si;
}
@@ -556,6 +582,7 @@ public final class SegmentInfo {
output.writeInt(delCount);
output.writeByte((byte) (hasProx ? 1:0));
output.writeStringStringMap(diagnostics);
+ output.writeByte((byte) (hasVectors ? 1 : 0));
}
void setHasProx(boolean hasProx) {
@@ -603,12 +630,22 @@ public final class SegmentInfo {
if (docStoreIsCompoundFile) {
files.add(IndexFileNames.segmentFileName(docStoreSegment, IndexFileNames.COMPOUND_FILE_STORE_EXTENSION));
} else {
- for (String ext : IndexFileNames.STORE_INDEX_EXTENSIONS)
- addIfExists(files, IndexFileNames.segmentFileName(docStoreSegment, ext));
+ files.add(IndexFileNames.segmentFileName(docStoreSegment, IndexFileNames.FIELDS_INDEX_EXTENSION));
+ files.add(IndexFileNames.segmentFileName(docStoreSegment, IndexFileNames.FIELDS_EXTENSION));
+ if (hasVectors) {
+ files.add(IndexFileNames.segmentFileName(docStoreSegment, IndexFileNames.VECTORS_INDEX_EXTENSION));
+ files.add(IndexFileNames.segmentFileName(docStoreSegment, IndexFileNames.VECTORS_DOCUMENTS_EXTENSION));
+ files.add(IndexFileNames.segmentFileName(docStoreSegment, IndexFileNames.VECTORS_FIELDS_EXTENSION));
+ }
}
} else if (!useCompoundFile) {
- for (String ext : IndexFileNames.STORE_INDEX_EXTENSIONS)
- addIfExists(files, IndexFileNames.segmentFileName(name, ext));
+ files.add(IndexFileNames.segmentFileName(name, IndexFileNames.FIELDS_INDEX_EXTENSION));
+ files.add(IndexFileNames.segmentFileName(name, IndexFileNames.FIELDS_EXTENSION));
+ if (hasVectors) {
+ files.add(IndexFileNames.segmentFileName(name, IndexFileNames.VECTORS_INDEX_EXTENSION));
+ files.add(IndexFileNames.segmentFileName(name, IndexFileNames.VECTORS_DOCUMENTS_EXTENSION));
+ files.add(IndexFileNames.segmentFileName(name, IndexFileNames.VECTORS_FIELDS_EXTENSION));
+ }
}
String delFileName = IndexFileNames.fileNameFromGeneration(name, IndexFileNames.DELETES_EXTENSION, delGen);
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java?rev=1050439&r1=1050438&r2=1050439&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentInfos.java Fri Dec 17 16:15:59 2010
@@ -89,8 +89,11 @@ public final class SegmentInfos extends
* diagnostics storage, and switches userData to Map */
public static final int FORMAT_DIAGNOSTICS = -9;
+ /** Each segment records whether it has term vectors */
+ public static final int FORMAT_HAS_VECTORS = -10;
+
/* This must always point to the most recent file format. */
- static final int CURRENT_FORMAT = FORMAT_DIAGNOSTICS;
+ public static final int CURRENT_FORMAT = FORMAT_HAS_VECTORS;
public int counter = 0; // used to name new segments
/**
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1050439&r1=1050438&r2=1050439&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Fri Dec 17 16:15:59 2010
@@ -103,9 +103,9 @@ final class SegmentMerger {
}
termIndexInterval = writer.getConfig().getTermIndexInterval();
}
-
- boolean hasProx() {
- return fieldInfos.hasProx();
+
+ public FieldInfos fieldInfos() {
+ return fieldInfos;
}
/**
@@ -151,8 +151,9 @@ final class SegmentMerger {
mergeTerms();
mergeNorms();
- if (mergeDocStores && fieldInfos.hasVectors())
+ if (mergeDocStores && fieldInfos.hasVectors()) {
mergeVectors();
+ }
return mergedDocs;
}
@@ -162,7 +163,7 @@ final class SegmentMerger {
// Basic files
for (String ext : IndexFileNames.COMPOUND_EXTENSIONS) {
- if (ext.equals(IndexFileNames.PROX_EXTENSION) && !hasProx())
+ if (ext.equals(IndexFileNames.PROX_EXTENSION) && !fieldInfos.hasProx())
continue;
if (mergeDocStores || (!ext.equals(IndexFileNames.FIELDS_EXTENSION) &&
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1050439&r1=1050438&r2=1050439&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java Fri Dec 17 16:15:59 2010
@@ -300,7 +300,7 @@ public class SegmentReader extends Index
throw new CorruptIndexException("doc counts differ for segment " + segment + ": fieldsReader shows " + fieldsReaderOrig.size() + " but segmentInfo shows " + si.docCount);
}
- if (fieldInfos.hasVectors()) { // open term vector files only as needed
+ if (si.getHasVectors()) { // open term vector files only as needed
termVectorsReaderOrig = new TermVectorsReader(storeDir, storesSegment, fieldInfos, readBufferSize, si.getDocStoreOffset(), si.docCount);
}
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java?rev=1050439&r1=1050438&r2=1050439&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java Fri Dec 17 16:15:59 2010
@@ -31,6 +31,7 @@ class SegmentWriteState {
int termIndexInterval;
int numDocsInStore;
Collection<String> flushedFiles;
+ public boolean hasVectors;
public SegmentWriteState(DocumentsWriter docWriter, Directory directory, String segmentName, String docStoreSegmentName, int numDocs,
int numDocsInStore, int termIndexInterval) {
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java?rev=1050439&r1=1050438&r2=1050439&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java Fri Dec 17 16:15:59 2010
@@ -76,41 +76,33 @@ class TermVectorsReader implements Clone
try {
String idxName = IndexFileNames.segmentFileName(segment, IndexFileNames.VECTORS_INDEX_EXTENSION);
- if (d.fileExists(idxName)) {
- tvx = d.openInput(idxName, readBufferSize);
- format = checkValidFormat(tvx);
- tvd = d.openInput(IndexFileNames.segmentFileName(segment, IndexFileNames.VECTORS_DOCUMENTS_EXTENSION), readBufferSize);
- final int tvdFormat = checkValidFormat(tvd);
- tvf = d.openInput(IndexFileNames.segmentFileName(segment, IndexFileNames.VECTORS_FIELDS_EXTENSION), readBufferSize);
- final int tvfFormat = checkValidFormat(tvf);
+ tvx = d.openInput(idxName, readBufferSize);
+ format = checkValidFormat(tvx);
+ tvd = d.openInput(IndexFileNames.segmentFileName(segment, IndexFileNames.VECTORS_DOCUMENTS_EXTENSION), readBufferSize);
+ final int tvdFormat = checkValidFormat(tvd);
+ tvf = d.openInput(IndexFileNames.segmentFileName(segment, IndexFileNames.VECTORS_FIELDS_EXTENSION), readBufferSize);
+ final int tvfFormat = checkValidFormat(tvf);
- assert format == tvdFormat;
- assert format == tvfFormat;
+ assert format == tvdFormat;
+ assert format == tvfFormat;
- if (format >= FORMAT_VERSION2) {
- numTotalDocs = (int) (tvx.length() >> 4);
- } else {
- assert (tvx.length()-FORMAT_SIZE) % 8 == 0;
- numTotalDocs = (int) (tvx.length() >> 3);
- }
+ if (format >= FORMAT_VERSION2) {
+ numTotalDocs = (int) (tvx.length() >> 4);
+ } else {
+ assert (tvx.length()-FORMAT_SIZE) % 8 == 0;
+ numTotalDocs = (int) (tvx.length() >> 3);
+ }
- if (-1 == docStoreOffset) {
- this.docStoreOffset = 0;
- this.size = numTotalDocs;
- assert size == 0 || numTotalDocs == size;
- } else {
- this.docStoreOffset = docStoreOffset;
- this.size = size;
- // Verify the file is long enough to hold all of our
- // docs
- assert numTotalDocs >= size + docStoreOffset: "numTotalDocs=" + numTotalDocs + " size=" + size + " docStoreOffset=" + docStoreOffset;
- }
+ if (-1 == docStoreOffset) {
+ this.docStoreOffset = 0;
+ this.size = numTotalDocs;
+ assert size == 0 || numTotalDocs == size;
} else {
- // If all documents flushed in a segment had hit
- // non-aborting exceptions, it's possible that
- // FieldInfos.hasVectors returns true yet the term
- // vector files don't exist.
- format = 0;
+ this.docStoreOffset = docStoreOffset;
+ this.size = size;
+ // Verify the file is long enough to hold all of our
+ // docs
+ assert numTotalDocs >= size + docStoreOffset: "numTotalDocs=" + numTotalDocs + " size=" + size + " docStoreOffset=" + docStoreOffset;
}
this.fieldInfos = fieldInfos;
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java?rev=1050439&r1=1050438&r2=1050439&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java Fri Dec 17 16:15:59 2010
@@ -37,6 +37,7 @@ final class TermVectorsTermsWriter exten
IndexOutput tvd;
IndexOutput tvf;
int lastDocID;
+ boolean hasVectors;
public TermVectorsTermsWriter(DocumentsWriter docWriter) {
this.docWriter = docWriter;
@@ -57,6 +58,7 @@ final class TermVectorsTermsWriter exten
// because, although FieldInfos.hasVectors() will return
// true, the TermVectorsReader gracefully handles
// non-existence of the term vectors files.
+ state.hasVectors = hasVectors;
if (tvx != null) {
@@ -108,6 +110,8 @@ final class TermVectorsTermsWriter exten
docWriter.removeOpenFile(docName);
lastDocID = 0;
+ state.hasVectors = hasVectors;
+ hasVectors = false;
}
}
@@ -146,7 +150,7 @@ final class TermVectorsTermsWriter exten
synchronized void initTermVectorsWriter() throws IOException {
if (tvx == null) {
-
+
final String docStoreSegment = docWriter.getDocStoreSegment();
if (docStoreSegment == null)
@@ -159,6 +163,7 @@ final class TermVectorsTermsWriter exten
String idxName = IndexFileNames.segmentFileName(docStoreSegment, IndexFileNames.VECTORS_INDEX_EXTENSION);
String docName = IndexFileNames.segmentFileName(docStoreSegment, IndexFileNames.VECTORS_DOCUMENTS_EXTENSION);
String fldName = IndexFileNames.segmentFileName(docStoreSegment, IndexFileNames.VECTORS_FIELDS_EXTENSION);
+ hasVectors = true;
tvx = docWriter.directory.createOutput(idxName);
tvd = docWriter.directory.createOutput(docName);
tvf = docWriter.directory.createOutput(fldName);
@@ -218,6 +223,7 @@ final class TermVectorsTermsWriter exten
@Override
public void abort() {
+ hasVectors = false;
if (tvx != null) {
try {
tvx.close();
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1050439&r1=1050438&r2=1050439&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Fri Dec 17 16:15:59 2010
@@ -597,9 +597,11 @@ public class TestAddIndexes extends Luce
addDoc(writer);
writer.close();
- dir2 = new MockDirectoryWrapper(random, new RAMDirectory());
+ dir2 = newDirectory();
writer2 = new IndexWriter(dir2, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ writer2.setInfoStream(VERBOSE ? System.out : null);
writer2.commit();
+
readers = new IndexReader[NUM_COPY];
for(int i=0;i<NUM_COPY;i++)
@@ -869,14 +871,17 @@ public class TestAddIndexes extends Luce
// LUCENE-1335: test simultaneous addIndexes & close
public void testAddIndexesWithRollback() throws Throwable {
-
+
final int NUM_COPY = 50;
CommitAndAddIndexes3 c = new CommitAndAddIndexes3(NUM_COPY);
c.launchThreads(-1);
- Thread.sleep(500);
+ Thread.sleep(_TestUtil.nextInt(random, 100, 500));
// Close w/o first stopping/joining the threads
+ if (VERBOSE) {
+ System.out.println("TEST: now force rollback");
+ }
c.didClose = true;
c.writer2.rollback();
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1050439&r1=1050438&r2=1050439&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Fri Dec 17 16:15:59 2010
@@ -69,11 +69,25 @@ public class TestBackwardsCompatibility
/*
public void testCreatePreLocklessCFS() throws IOException {
- createIndex("index.cfs", true);
+ createIndex(random, "index.cfs", true);
}
public void testCreatePreLocklessNoCFS() throws IOException {
- createIndex("index.nocfs", false);
+ createIndex(random, "index.nocfs", false);
+ }
+ */
+
+ /*
+ public void testCreateCFS() throws IOException {
+ String dirName = "testindex.cfs";
+ createIndex(random, dirName, true);
+ //rmDir(dirName);
+ }
+
+ public void testCreateNoCFS() throws IOException {
+ String dirName = "testindex.nocfs";
+ createIndex(random, dirName, false);
+ //rmDir(dirName);
}
*/
@@ -111,18 +125,6 @@ public class TestBackwardsCompatibility
zipFile.close();
}
- public void testCreateCFS() throws IOException {
- String dirName = "testindex.cfs";
- createIndex(random, dirName, true);
- rmDir(dirName);
- }
-
- public void testCreateNoCFS() throws IOException {
- String dirName = "testindex.nocfs";
- createIndex(random, dirName, true);
- rmDir(dirName);
- }
-
final String[] oldNames = {"19.cfs",
"19.nocfs",
"20.cfs",
@@ -352,6 +354,10 @@ public class TestBackwardsCompatibility
f = d.getField("fie\u2C77ld");
assertEquals("field with non-ascii name", f.stringValue());
}
+
+ TermFreqVector tfv = reader.getTermFreqVector(i, "utf8");
+ assertNotNull("docID=" + i + " index=" + dirName, tfv);
+ assertTrue(tfv instanceof TermPositionVector);
}
} else
// Only ID 7 is deleted
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDoc.java?rev=1050439&r1=1050438&r2=1050439&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestDoc.java Fri Dec 17 16:15:59 2010
@@ -191,13 +191,17 @@ public class TestDoc extends LuceneTestC
r1.close();
r2.close();
+ final SegmentInfo info = new SegmentInfo(merged, si1.docCount + si2.docCount, si1.dir,
+ useCompoundFile, true, -1, null, false, merger.fieldInfos().hasProx(),
+ merger.fieldInfos().hasVectors());
+
if (useCompoundFile) {
Collection<String> filesToDelete = merger.createCompoundFile(merged + ".cfs");
for (final String fileToDelete : filesToDelete)
si1.dir.deleteFile(fileToDelete);
}
- return new SegmentInfo(merged, si1.docCount + si2.docCount, si1.dir, useCompoundFile, true);
+ return info;
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1050439&r1=1050438&r2=1050439&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Fri Dec 17 16:15:59 2010
@@ -1037,10 +1037,14 @@ public class TestIndexWriter extends Luc
public void testEmptyDocAfterFlushingRealDoc() throws IOException {
Directory dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
+ writer.setInfoStream(VERBOSE ? System.out : null);
Document doc = new Document();
doc.add(newField("field", "aaa", Field.Store.YES, Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
writer.addDocument(doc);
writer.commit();
+ if (VERBOSE) {
+ System.out.println("\nTEST: now add empty doc");
+ }
writer.addDocument(new Document());
writer.close();
_TestUtil.checkIndex(dir);
@@ -1179,7 +1183,11 @@ public class TestIndexWriter extends Luc
Directory dir = newDirectory();
int delID = 0;
for(int i=0;i<20;i++) {
+ if (VERBOSE) {
+ System.out.println("TEST: iter=" + i);
+ }
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setMaxBufferedDocs(2));
+ writer.setInfoStream(VERBOSE ? System.out : null);
LogMergePolicy lmp = (LogMergePolicy) writer.getConfig().getMergePolicy();
lmp.setMergeFactor(2);
lmp.setUseCompoundFile(false);
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java?rev=1050439&r1=1050438&r2=1050439&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java Fri Dec 17 16:15:59 2010
@@ -51,7 +51,7 @@ public class TestIndexWriterOnDiskFull e
System.out.println("TEST: pass=" + pass);
}
boolean doAbort = pass == 1;
- long diskFree = 200;
+ long diskFree = _TestUtil.nextInt(random, 100, 300);
while(true) {
if (VERBOSE) {
System.out.println("TEST: cycle: diskFree=" + diskFree);
@@ -120,7 +120,7 @@ public class TestIndexWriterOnDiskFull e
dir.close();
// Now try again w/ more space:
- diskFree += 500;
+ diskFree += _TestUtil.nextInt(random, 400, 600);
} else {
//_TestUtil.syncConcurrentMerges(writer);
dir.setMaxSizeInBytes(0);
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java?rev=1050439&r1=1050438&r2=1050439&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java Fri Dec 17 16:15:59 2010
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.document.Document;
import java.io.IOException;
@@ -76,7 +77,12 @@ public class TestSegmentMerger extends L
int docsMerged = merger.merge();
assertTrue(docsMerged == 2);
//Should be able to open a new SegmentReader against the new directory
- SegmentReader mergedReader = SegmentReader.get(true, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, true), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
+ SegmentReader mergedReader = SegmentReader.get(false, mergedDir,
+ new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, true, -1,
+ null, false, merger.fieldInfos().hasProx(),
+ merger.fieldInfos().hasVectors()),
+ BufferedIndexInput.BUFFER_SIZE, true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
+
assertTrue(mergedReader != null);
assertTrue(mergedReader.numDocs() == 2);
Document newDoc1 = mergedReader.document(0);
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/store/MockIndexOutputWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/store/MockIndexOutputWrapper.java?rev=1050439&r1=1050438&r2=1050439&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/store/MockIndexOutputWrapper.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/store/MockIndexOutputWrapper.java Fri Dec 17 16:15:59 2010
@@ -106,6 +106,7 @@ public class MockIndexOutputWrapper exte
message += ")";
if (LuceneTestCase.VERBOSE) {
System.out.println(Thread.currentThread().getName() + ": MDW: now throw fake disk full");
+ new Throwable().printStackTrace(System.out);
}
throw new IOException(message);
} else {