You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/05/23 02:53:53 UTC
svn commit: r1126285 - in /lucene/dev/branches/branch_3x/lucene: ./
backwards/ backwards/src/test/org/apache/lucene/index/TestDocumentWriter.java
src/java/org/apache/lucene/index/SegmentCoreReaders.java
src/java/org/apache/lucene/index/SegmentReader.java
Author: rmuir
Date: Mon May 23 00:53:52 2011
New Revision: 1126285
URL: http://svn.apache.org/viewvc?rev=1126285&view=rev
Log:
LUCENE-3127: pull CoreReaders out of SegmentReader.java
Added:
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
- copied, changed from r1126284, lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
Modified:
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestDocumentWriter.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java
Modified: lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=1126285&r1=1126284&r2=1126285&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/dev/branches/branch_3x/lucene/backwards/src/test/org/apache/lucene/index/TestDocumentWriter.java Mon May 23 00:53:52 2011
@@ -60,54 +60,6 @@ public class TestDocumentWriter extends
assertTrue(dir != null);
}
- public void testAddDocument() throws Exception {
- Document testDoc = new Document();
- DocHelper.setupDoc(testDoc);
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)));
- writer.addDocument(testDoc);
- writer.commit();
- SegmentInfo info = writer.newestSegment();
- writer.close();
- //After adding the document, we should be able to read it back in
- SegmentReader reader = SegmentReader.get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR);
- assertTrue(reader != null);
- Document doc = reader.document(0);
- assertTrue(doc != null);
-
- //System.out.println("Document: " + doc);
- Fieldable [] fields = doc.getFields("textField2");
- assertTrue(fields != null && fields.length == 1);
- assertTrue(fields[0].stringValue().equals(DocHelper.FIELD_2_TEXT));
- assertTrue(fields[0].isTermVectorStored());
-
- fields = doc.getFields("textField1");
- assertTrue(fields != null && fields.length == 1);
- assertTrue(fields[0].stringValue().equals(DocHelper.FIELD_1_TEXT));
- assertFalse(fields[0].isTermVectorStored());
-
- fields = doc.getFields("keyField");
- assertTrue(fields != null && fields.length == 1);
- assertTrue(fields[0].stringValue().equals(DocHelper.KEYWORD_TEXT));
-
- fields = doc.getFields(DocHelper.NO_NORMS_KEY);
- assertTrue(fields != null && fields.length == 1);
- assertTrue(fields[0].stringValue().equals(DocHelper.NO_NORMS_TEXT));
-
- fields = doc.getFields(DocHelper.TEXT_FIELD_3_KEY);
- assertTrue(fields != null && fields.length == 1);
- assertTrue(fields[0].stringValue().equals(DocHelper.FIELD_3_TEXT));
-
- // test that the norms are not present in the segment if
- // omitNorms is true
- for (int i = 0; i < reader.core.fieldInfos.size(); i++) {
- FieldInfo fi = reader.core.fieldInfos.fieldInfo(i);
- if (fi.isIndexed) {
- assertTrue(fi.omitNorms == !reader.hasNorms(fi.name));
- }
- }
- reader.close();
- }
-
public void testPositionIncrementGap() throws IOException {
Analyzer analyzer = new Analyzer() {
@Override
Copied: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (from r1126284, lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java?p2=lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java&p1=lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java&r1=1126284&r2=1126285&rev=1126285&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java Mon May 23 00:53:52 2011
@@ -20,13 +20,13 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.IndexInput;
/** Holds core readers that are shared (unchanged) when
* SegmentReader is cloned or reopened */
final class SegmentCoreReaders {
-
+
// Counts how many other reader share the core objects
// (freqStream, proxStream, tis, etc.) of this reader;
// when coreRef drops to 0, these core objects may be
@@ -34,126 +34,186 @@ final class SegmentCoreReaders {
// closed, even those it shares core objects with other
// SegmentReaders:
private final AtomicInteger ref = new AtomicInteger(1);
-
+
final String segment;
final FieldInfos fieldInfos;
-
- final FieldsProducer fields;
-
+ final IndexInput freqStream;
+ final IndexInput proxStream;
+ final TermInfosReader tisNoIndex;
+
final Directory dir;
final Directory cfsDir;
final int readBufferSize;
final int termsIndexDivisor;
-
+
private final SegmentReader owner;
-
+
+ TermInfosReader tis;
FieldsReader fieldsReaderOrig;
TermVectorsReader termVectorsReaderOrig;
CompoundFileReader cfsReader;
CompoundFileReader storeCFSReader;
-
+
SegmentCoreReaders(SegmentReader owner, Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor) throws IOException {
-
- if (termsIndexDivisor == 0) {
- throw new IllegalArgumentException("indexDivisor must be < 0 (don't load terms index) or greater than 0 (got 0)");
- }
-
segment = si.name;
- final SegmentCodecs segmentCodecs = si.getSegmentCodecs();
this.readBufferSize = readBufferSize;
this.dir = dir;
-
+
boolean success = false;
-
+
try {
Directory dir0 = dir;
if (si.getUseCompoundFile()) {
- cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
+ cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
dir0 = cfsReader;
}
cfsDir = dir0;
- si.loadFieldInfos(cfsDir, false); // prevent opening the CFS to load fieldInfos
- fieldInfos = si.getFieldInfos();
-
+
+ fieldInfos = new FieldInfos(cfsDir, IndexFileNames.segmentFileName(segment, IndexFileNames.FIELD_INFOS_EXTENSION));
+
this.termsIndexDivisor = termsIndexDivisor;
-
- // Ask codec for its Fields
- fields = segmentCodecs.codec().fieldsProducer(new SegmentReadState(cfsDir, si, fieldInfos, readBufferSize, termsIndexDivisor));
- assert fields != null;
-
+ TermInfosReader reader = new TermInfosReader(cfsDir, segment, fieldInfos, readBufferSize, termsIndexDivisor);
+ if (termsIndexDivisor == -1) {
+ tisNoIndex = reader;
+ } else {
+ tis = reader;
+ tisNoIndex = null;
+ }
+
+ // make sure that all index files have been read or are kept open
+ // so that if an index update removes them we'll still have them
+ freqStream = cfsDir.openInput(IndexFileNames.segmentFileName(segment, IndexFileNames.FREQ_EXTENSION), readBufferSize);
+
+ if (fieldInfos.hasProx()) {
+ proxStream = cfsDir.openInput(IndexFileNames.segmentFileName(segment, IndexFileNames.PROX_EXTENSION), readBufferSize);
+ } else {
+ proxStream = null;
+ }
success = true;
} finally {
if (!success) {
decRef();
}
}
-
+
// Must assign this at the end -- if we hit an
// exception above core, we don't want to attempt to
// purge the FieldCache (will hit NPE because core is
// not assigned yet).
this.owner = owner;
}
-
+
synchronized TermVectorsReader getTermVectorsReaderOrig() {
return termVectorsReaderOrig;
}
-
+
synchronized FieldsReader getFieldsReaderOrig() {
return fieldsReaderOrig;
}
-
+
synchronized void incRef() {
ref.incrementAndGet();
}
-
+
synchronized Directory getCFSReader() {
return cfsReader;
}
-
+
+ synchronized TermInfosReader getTermsReader() {
+ if (tis != null) {
+ return tis;
+ } else {
+ return tisNoIndex;
+ }
+ }
+
+ synchronized boolean termsIndexIsLoaded() {
+ return tis != null;
+ }
+
+ // NOTE: only called from IndexWriter when a near
+ // real-time reader is opened, or applyDeletes is run,
+ // sharing a segment that's still being merged. This
+ // method is not fully thread safe, and relies on the
+ // synchronization in IndexWriter
+ synchronized void loadTermsIndex(SegmentInfo si, int termsIndexDivisor) throws IOException {
+ if (tis == null) {
+ Directory dir0;
+ if (si.getUseCompoundFile()) {
+ // In some cases, we were originally opened when CFS
+ // was not used, but then we are asked to open the
+ // terms reader with index, the segment has switched
+ // to CFS
+ if (cfsReader == null) {
+ cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
+ }
+ dir0 = cfsReader;
+ } else {
+ dir0 = dir;
+ }
+
+ tis = new TermInfosReader(dir0, segment, fieldInfos, readBufferSize, termsIndexDivisor);
+ }
+ }
+
synchronized void decRef() throws IOException {
-
+
if (ref.decrementAndGet() == 0) {
+
+ // close everything, nothing is shared anymore with other readers
+ if (tis != null) {
+ tis.close();
+ // null so if an app hangs on to us we still free most ram
+ tis = null;
+ }
- if (fields != null) {
- fields.close();
+ if (tisNoIndex != null) {
+ tisNoIndex.close();
}
+ if (freqStream != null) {
+ freqStream.close();
+ }
+
+ if (proxStream != null) {
+ proxStream.close();
+ }
+
if (termVectorsReaderOrig != null) {
termVectorsReaderOrig.close();
}
-
+
if (fieldsReaderOrig != null) {
fieldsReaderOrig.close();
}
-
+
if (cfsReader != null) {
cfsReader.close();
}
-
+
if (storeCFSReader != null) {
storeCFSReader.close();
}
-
+
// Now, notify any ReaderFinished listeners:
if (owner != null) {
owner.notifyReaderFinishedListeners();
}
}
}
-
+
synchronized void openDocStores(SegmentInfo si) throws IOException {
-
+
assert si.name.equals(segment);
-
+
if (fieldsReaderOrig == null) {
final Directory storeDir;
if (si.getDocStoreOffset() != -1) {
if (si.getDocStoreIsCompoundFile()) {
assert storeCFSReader == null;
storeCFSReader = new CompoundFileReader(dir,
- IndexFileNames.segmentFileName(si.getDocStoreSegment(), "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION),
- readBufferSize);
+ IndexFileNames.segmentFileName(si.getDocStoreSegment(), IndexFileNames.COMPOUND_FILE_STORE_EXTENSION),
+ readBufferSize);
storeDir = storeCFSReader;
assert storeDir != null;
} else {
@@ -165,7 +225,7 @@ final class SegmentCoreReaders {
// was not used, but then we are asked to open doc
// stores after the segment has switched to CFS
if (cfsReader == null) {
- cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
+ cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
}
storeDir = cfsReader;
assert storeDir != null;
@@ -173,16 +233,22 @@ final class SegmentCoreReaders {
storeDir = dir;
assert storeDir != null;
}
-
- final String storesSegment = si.getDocStoreSegment();
+
+ final String storesSegment;
+ if (si.getDocStoreOffset() != -1) {
+ storesSegment = si.getDocStoreSegment();
+ } else {
+ storesSegment = segment;
+ }
+
fieldsReaderOrig = new FieldsReader(storeDir, storesSegment, fieldInfos, readBufferSize,
- si.getDocStoreOffset(), si.docCount);
-
+ si.getDocStoreOffset(), si.docCount);
+
// Verify two sources of "maxDoc" agree:
if (si.getDocStoreOffset() == -1 && fieldsReaderOrig.size() != si.docCount) {
throw new CorruptIndexException("doc counts differ for segment " + segment + ": fieldsReader shows " + fieldsReaderOrig.size() + " but segmentInfo shows " + si.docCount);
}
-
+
if (si.getHasVectors()) { // open term vector files only as needed
termVectorsReaderOrig = new TermVectorsReader(storeDir, storesSegment, fieldInfos, readBufferSize, si.getDocStoreOffset(), si.docCount);
}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java?rev=1126285&r1=1126284&r2=1126285&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/index/SegmentReader.java Mon May 23 00:53:52 2011
@@ -70,240 +70,7 @@ public class SegmentReader extends Index
IndexInput singleNormStream;
AtomicInteger singleNormRef;
- CoreReaders core;
-
- // Holds core readers that are shared (unchanged) when
- // SegmentReader is cloned or reopened
- static final class CoreReaders {
-
- // Counts how many other reader share the core objects
- // (freqStream, proxStream, tis, etc.) of this reader;
- // when coreRef drops to 0, these core objects may be
- // closed. A given instance of SegmentReader may be
- // closed, even those it shares core objects with other
- // SegmentReaders:
- private final AtomicInteger ref = new AtomicInteger(1);
-
- final String segment;
- final FieldInfos fieldInfos;
- final IndexInput freqStream;
- final IndexInput proxStream;
- final TermInfosReader tisNoIndex;
-
- final Directory dir;
- final Directory cfsDir;
- final int readBufferSize;
- final int termsIndexDivisor;
-
- private final SegmentReader origInstance;
-
- TermInfosReader tis;
- FieldsReader fieldsReaderOrig;
- TermVectorsReader termVectorsReaderOrig;
- CompoundFileReader cfsReader;
- CompoundFileReader storeCFSReader;
-
- CoreReaders(SegmentReader origInstance, Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor) throws IOException {
- segment = si.name;
- this.readBufferSize = readBufferSize;
- this.dir = dir;
-
- boolean success = false;
-
- try {
- Directory dir0 = dir;
- if (si.getUseCompoundFile()) {
- cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
- dir0 = cfsReader;
- }
- cfsDir = dir0;
-
- fieldInfos = new FieldInfos(cfsDir, IndexFileNames.segmentFileName(segment, IndexFileNames.FIELD_INFOS_EXTENSION));
-
- this.termsIndexDivisor = termsIndexDivisor;
- TermInfosReader reader = new TermInfosReader(cfsDir, segment, fieldInfos, readBufferSize, termsIndexDivisor);
- if (termsIndexDivisor == -1) {
- tisNoIndex = reader;
- } else {
- tis = reader;
- tisNoIndex = null;
- }
-
- // make sure that all index files have been read or are kept open
- // so that if an index update removes them we'll still have them
- freqStream = cfsDir.openInput(IndexFileNames.segmentFileName(segment, IndexFileNames.FREQ_EXTENSION), readBufferSize);
-
- if (fieldInfos.hasProx()) {
- proxStream = cfsDir.openInput(IndexFileNames.segmentFileName(segment, IndexFileNames.PROX_EXTENSION), readBufferSize);
- } else {
- proxStream = null;
- }
- success = true;
- } finally {
- if (!success) {
- decRef();
- }
- }
-
- // Must assign this at the end -- if we hit an
- // exception above core, we don't want to attempt to
- // purge the FieldCache (will hit NPE because core is
- // not assigned yet).
- this.origInstance = origInstance;
- }
-
- synchronized TermVectorsReader getTermVectorsReaderOrig() {
- return termVectorsReaderOrig;
- }
-
- synchronized FieldsReader getFieldsReaderOrig() {
- return fieldsReaderOrig;
- }
-
- synchronized void incRef() {
- ref.incrementAndGet();
- }
-
- synchronized Directory getCFSReader() {
- return cfsReader;
- }
-
- synchronized TermInfosReader getTermsReader() {
- if (tis != null) {
- return tis;
- } else {
- return tisNoIndex;
- }
- }
-
- synchronized boolean termsIndexIsLoaded() {
- return tis != null;
- }
-
- // NOTE: only called from IndexWriter when a near
- // real-time reader is opened, or applyDeletes is run,
- // sharing a segment that's still being merged. This
- // method is not fully thread safe, and relies on the
- // synchronization in IndexWriter
- synchronized void loadTermsIndex(SegmentInfo si, int termsIndexDivisor) throws IOException {
- if (tis == null) {
- Directory dir0;
- if (si.getUseCompoundFile()) {
- // In some cases, we were originally opened when CFS
- // was not used, but then we are asked to open the
- // terms reader with index, the segment has switched
- // to CFS
- if (cfsReader == null) {
- cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
- }
- dir0 = cfsReader;
- } else {
- dir0 = dir;
- }
-
- tis = new TermInfosReader(dir0, segment, fieldInfos, readBufferSize, termsIndexDivisor);
- }
- }
-
- synchronized void decRef() throws IOException {
-
- if (ref.decrementAndGet() == 0) {
-
- // close everything, nothing is shared anymore with other readers
- if (tis != null) {
- tis.close();
- // null so if an app hangs on to us we still free most ram
- tis = null;
- }
-
- if (tisNoIndex != null) {
- tisNoIndex.close();
- }
-
- if (freqStream != null) {
- freqStream.close();
- }
-
- if (proxStream != null) {
- proxStream.close();
- }
-
- if (termVectorsReaderOrig != null) {
- termVectorsReaderOrig.close();
- }
-
- if (fieldsReaderOrig != null) {
- fieldsReaderOrig.close();
- }
-
- if (cfsReader != null) {
- cfsReader.close();
- }
-
- if (storeCFSReader != null) {
- storeCFSReader.close();
- }
-
- // Now, notify any ReaderFinished listeners:
- if (origInstance != null) {
- origInstance.notifyReaderFinishedListeners();
- }
- }
- }
-
- synchronized void openDocStores(SegmentInfo si) throws IOException {
-
- assert si.name.equals(segment);
-
- if (fieldsReaderOrig == null) {
- final Directory storeDir;
- if (si.getDocStoreOffset() != -1) {
- if (si.getDocStoreIsCompoundFile()) {
- assert storeCFSReader == null;
- storeCFSReader = new CompoundFileReader(dir,
- IndexFileNames.segmentFileName(si.getDocStoreSegment(), IndexFileNames.COMPOUND_FILE_STORE_EXTENSION),
- readBufferSize);
- storeDir = storeCFSReader;
- assert storeDir != null;
- } else {
- storeDir = dir;
- assert storeDir != null;
- }
- } else if (si.getUseCompoundFile()) {
- // In some cases, we were originally opened when CFS
- // was not used, but then we are asked to open doc
- // stores after the segment has switched to CFS
- if (cfsReader == null) {
- cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
- }
- storeDir = cfsReader;
- assert storeDir != null;
- } else {
- storeDir = dir;
- assert storeDir != null;
- }
-
- final String storesSegment;
- if (si.getDocStoreOffset() != -1) {
- storesSegment = si.getDocStoreSegment();
- } else {
- storesSegment = segment;
- }
-
- fieldsReaderOrig = new FieldsReader(storeDir, storesSegment, fieldInfos, readBufferSize,
- si.getDocStoreOffset(), si.docCount);
-
- // Verify two sources of "maxDoc" agree:
- if (si.getDocStoreOffset() == -1 && fieldsReaderOrig.size() != si.docCount) {
- throw new CorruptIndexException("doc counts differ for segment " + segment + ": fieldsReader shows " + fieldsReaderOrig.size() + " but segmentInfo shows " + si.docCount);
- }
-
- if (si.getHasVectors()) { // open term vector files only as needed
- termVectorsReaderOrig = new TermVectorsReader(storeDir, storesSegment, fieldInfos, readBufferSize, si.getDocStoreOffset(), si.docCount);
- }
- }
- }
- }
+ SegmentCoreReaders core;
/**
* Sets the initial value
@@ -344,7 +111,7 @@ public class SegmentReader extends Index
boolean success = false;
try {
- instance.core = new CoreReaders(instance, dir, si, readBufferSize, termInfosIndexDivisor);
+ instance.core = new SegmentCoreReaders(instance, dir, si, readBufferSize, termInfosIndexDivisor);
if (doOpenStores) {
instance.core.openDocStores(si);
}