You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by mi...@apache.org on 2010/03/10 23:53:38 UTC
svn commit: r921608 [1/2] - in /lucene/java/branches/flex_1458/src:
java/org/apache/lucene/index/ java/org/apache/lucene/index/codecs/
java/org/apache/lucene/index/codecs/intblock/
java/org/apache/lucene/index/codecs/preflex/ java/org/apache/lucene/ind...
Author: mikemccand
Date: Wed Mar 10 22:53:37 2010
New Revision: 921608
URL: http://svn.apache.org/viewvc?rev=921608&view=rev
Log:
remove more nocommits; small renaming/refactoring; optimize termsenum seeking for flex API on pre-flex index; new bulk read api
Added:
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReadState.java (with props)
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/CodecProvider.java
- copied, changed from r920448, lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codecs.java
Removed:
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codecs.java
Modified:
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CheckIndex.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocumentsWriter.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileNameFilter.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfo.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfos.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentMerger.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentWriteState.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/StoredFieldsWriter.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsEnum.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codec.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepCodec.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermScorer.java
lucene/java/branches/flex_1458/src/java/org/apache/lucene/util/BytesRef.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/TestExternalCodecs.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/FlexTestUtil.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestCodecs.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestDoc.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestIndexReader.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestSegmentMerger.java
lucene/java/branches/flex_1458/src/test/org/apache/lucene/index/TestStressIndexing2.java
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java Wed Mar 10 22:53:37 2010
@@ -48,10 +48,12 @@ class AllDocsEnum extends DocsEnum {
}
@Override
- public int read(int[] docs, int[] freqs) throws IOException {
- final int length = docs.length;
+ public BulkReadResult read() throws IOException {
+ initBulkResult();
+ final int[] docs = bulkResult.docs.ints;
+ final int[] freqs = bulkResult.freqs.ints;
int i = 0;
- while (i < length && doc < maxDoc) {
+ while (i < docs.length && doc < maxDoc) {
if (skipDocs == null || !skipDocs.get(doc)) {
docs[i] = doc;
freqs[i] = 1;
@@ -59,7 +61,8 @@ class AllDocsEnum extends DocsEnum {
}
doc++;
}
- return i;
+ bulkResult.count = i;
+ return bulkResult;
}
@Override
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CheckIndex.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CheckIndex.java Wed Mar 10 22:53:37 2010
@@ -22,7 +22,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.document.AbstractField; // for javadocs
import org.apache.lucene.document.Document;
-import org.apache.lucene.index.codecs.Codecs;
+import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -280,7 +280,7 @@ public class CheckIndex {
}
protected Status checkIndex(List<String> onlySegments) throws IOException {
- return checkIndex(onlySegments, Codecs.getDefault());
+ return checkIndex(onlySegments, CodecProvider.getDefault());
}
/** Returns a {@link Status} instance detailing
@@ -295,7 +295,7 @@ public class CheckIndex {
* <p><b>WARNING</b>: make sure
* you only call this when the index is not opened by any
* writer. */
- protected Status checkIndex(List<String> onlySegments, Codecs codecs) throws IOException {
+ protected Status checkIndex(List<String> onlySegments, CodecProvider codecs) throws IOException {
NumberFormat nf = NumberFormat.getInstance();
SegmentInfos sis = new SegmentInfos();
Status result = new Status();
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java Wed Mar 10 22:53:37 2010
@@ -35,7 +35,7 @@ import org.apache.lucene.search.Similari
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.index.codecs.Codecs;
+import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.BytesRef;
@@ -49,7 +49,7 @@ class DirectoryReader extends IndexReade
protected Directory directory;
protected boolean readOnly;
- protected Codecs codecs;
+ protected CodecProvider codecs;
IndexWriter writer;
@@ -77,10 +77,10 @@ class DirectoryReader extends IndexReade
// }
static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly,
- final int termInfosIndexDivisor, Codecs codecs) throws CorruptIndexException, IOException {
- final Codecs codecs2;
+ final int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException {
+ final CodecProvider codecs2;
if (codecs == null) {
- codecs2 = Codecs.getDefault();
+ codecs2 = CodecProvider.getDefault();
} else {
codecs2 = codecs;
}
@@ -103,7 +103,7 @@ class DirectoryReader extends IndexReade
// }
/** Construct reading the named set of readers. */
- DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, Codecs codecs) throws IOException {
+ DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws IOException {
this.directory = directory;
this.readOnly = readOnly;
this.segmentInfos = sis;
@@ -111,7 +111,7 @@ class DirectoryReader extends IndexReade
this.termInfosIndexDivisor = termInfosIndexDivisor;
if (codecs == null) {
- this.codecs = Codecs.getDefault();
+ this.codecs = CodecProvider.getDefault();
} else {
this.codecs = codecs;
}
@@ -151,13 +151,13 @@ class DirectoryReader extends IndexReade
}
// Used by near real-time search
- DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor, Codecs codecs) throws IOException {
+ DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor, CodecProvider codecs) throws IOException {
this.directory = writer.getDirectory();
this.readOnly = true;
segmentInfos = infos;
this.termInfosIndexDivisor = termInfosIndexDivisor;
if (codecs == null) {
- this.codecs = Codecs.getDefault();
+ this.codecs = CodecProvider.getDefault();
} else {
this.codecs = codecs;
}
@@ -213,13 +213,13 @@ class DirectoryReader extends IndexReade
/** This constructor is only used for {@link #reopen()} */
DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts,
- Map<String,byte[]> oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor, Codecs codecs) throws IOException {
+ Map<String,byte[]> oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor, CodecProvider codecs) throws IOException {
this.directory = directory;
this.readOnly = readOnly;
this.segmentInfos = infos;
this.termInfosIndexDivisor = termInfosIndexDivisor;
if (codecs == null) {
- this.codecs = Codecs.getDefault();
+ this.codecs = CodecProvider.getDefault();
} else {
this.codecs = codecs;
}
@@ -1039,11 +1039,11 @@ class DirectoryReader extends IndexReade
/** @see org.apache.lucene.index.IndexReader#listCommits */
public static Collection<IndexCommit> listCommits(Directory dir) throws IOException {
- return listCommits(dir, Codecs.getDefault());
+ return listCommits(dir, CodecProvider.getDefault());
}
/** @see org.apache.lucene.index.IndexReader#listCommits */
- public static Collection<IndexCommit> listCommits(Directory dir, Codecs codecs) throws IOException {
+ public static Collection<IndexCommit> listCommits(Directory dir, CodecProvider codecs) throws IOException {
final String[] files = dir.listAll();
Collection<IndexCommit> commits = new ArrayList<IndexCommit>();
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java Wed Mar 10 22:53:37 2010
@@ -113,8 +113,9 @@ final class DocFieldProcessorPerThread e
else
lastPerField.next = perField.next;
- if (state.docWriter.infoStream != null)
- state.docWriter.infoStream.println(" purge field=" + perField.fieldInfo.name);
+ if (state.infoStream != null) {
+ state.infoStream.println(" purge field=" + perField.fieldInfo.name);
+ }
totalFieldCount--;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java Wed Mar 10 22:53:37 2010
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.IntsRef;
/** Iterates through the documents, term freq and positions.
* NOTE: you must first call {@link #next}.
@@ -44,16 +45,42 @@ public abstract class DocsEnum extends D
return atts;
}
- // nocommit -- fix this API so that intblock codecs are
- // able to return their own int arrays, to save a copy... IntsRef?
- /** Bulk read: returns number of docs read. After this is
- * called, {@link #doc} and {@link #freq} are undefined.
+ // TODO: maybe add bulk read only docIDs (for eventual
+ // match-only scoring)
+
+ public static class BulkReadResult {
+ public final IntsRef docs = new IntsRef();
+ public final IntsRef freqs = new IntsRef();
+ public int count;
+ }
+
+ protected BulkReadResult bulkResult;
+
+ protected final void initBulkResult() {
+ if (bulkResult == null) {
+ bulkResult = new BulkReadResult();
+ bulkResult.docs.ints = new int[64];
+ bulkResult.freqs.ints = new int[64];
+ }
+ }
+
+ /** Bulk read (docs and freqs). After this is called,
+ * {@link #doc} and {@link #freq} are undefined. You must
+ * refer to the count member of BulkResult to determine
+ * how many docs were loaded (the IntsRef for docs and
+ * freqs will not have their length set). This method
+ * will not return null. The end has been reached when
+ * .count is 0.
*
* <p>NOTE: the default impl simply delegates to {@link
* #nextDoc}, but subclasses may do this more
* efficiently. */
- public int read(int[] docs, int[] freqs) throws IOException {
+ // nocommit -- maybe pre-share the BulkReadResult.... hmm
+ public BulkReadResult read() throws IOException {
+ initBulkResult();
int count = 0;
+ final int[] docs = bulkResult.docs.ints;
+ final int[] freqs = bulkResult.freqs.ints;
while(count < docs.length) {
final int doc = nextDoc();
if (doc != NO_MORE_DOCS) {
@@ -64,6 +91,7 @@ public abstract class DocsEnum extends D
break;
}
}
- return count;
+ bulkResult.count = count;
+ return bulkResult;
}
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocumentsWriter.java Wed Mar 10 22:53:37 2010
@@ -589,7 +589,7 @@ final class DocumentsWriter {
synchronized private void initFlushState(boolean onlyDocStore) {
initSegmentName(onlyDocStore);
- flushState = new SegmentWriteState(this, directory, segment, docFieldProcessor.fieldInfos,
+ flushState = new SegmentWriteState(infoStream, directory, segment, docFieldProcessor.fieldInfos,
docStoreSegment, numDocsInRAM, numDocsInStore, writer.getTermIndexInterval(),
writer.codecs);
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java Wed Mar 10 22:53:37 2010
@@ -17,7 +17,6 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import java.io.File;
import java.io.FileNotFoundException;
import java.io.FilenameFilter;
import java.io.IOException;
@@ -30,7 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.lucene.index.codecs.Codecs;
+import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.store.Directory;
/*
@@ -128,7 +127,7 @@ final class IndexFileDeleter {
* @throws IOException if there is a low-level IO error
*/
public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, DocumentsWriter docWriter,
- Codecs codecs)
+ CodecProvider codecs)
throws CorruptIndexException, IOException {
this.docWriter = docWriter;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileNameFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileNameFilter.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileNameFilter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileNameFilter.java Wed Mar 10 22:53:37 2010
@@ -20,7 +20,7 @@ package org.apache.lucene.index;
import java.io.File;
import java.io.FilenameFilter;
import java.util.HashSet;
-import org.apache.lucene.index.codecs.Codecs;
+import org.apache.lucene.index.codecs.CodecProvider;
/**
* Filename filter that accept filenames and extensions only
@@ -33,7 +33,7 @@ public class IndexFileNameFilter impleme
private final HashSet<String> extensions;
- public IndexFileNameFilter(Codecs codecs) {
+ public IndexFileNameFilter(CodecProvider codecs) {
extensions = new HashSet<String>();
for (String ext : IndexFileNames.INDEX_EXTENSIONS) {
extensions.add(ext);
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexReader.java Wed Mar 10 22:53:37 2010
@@ -20,7 +20,7 @@ package org.apache.lucene.index;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.search.Similarity;
-import org.apache.lucene.index.codecs.Codecs;
+import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.store.*;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
@@ -345,7 +345,7 @@ public abstract class IndexReader implem
/** Expert: returns an IndexReader reading the index in
* the given Directory, with a custom {@link
- * IndexDeletionPolicy}, and specified {@link Codecs}.
+ * IndexDeletionPolicy}, and specified {@link CodecProvider}.
* You should pass readOnly=true, since it gives much
* better concurrent performance, unless you intend to do
* write operations (delete documents or change norms)
@@ -365,18 +365,18 @@ public abstract class IndexReader implem
* memory usage, at the expense of higher latency when
* loading a TermInfo. The default value is 1. Set this
* to -1 to skip loading the terms index entirely.
- * @param codecs Codecs to use when opening index
+ * @param codecs CodecProvider to use when opening index
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, Codecs codecs) throws CorruptIndexException, IOException {
+ public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException {
return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor, codecs);
}
/** Expert: returns an IndexReader reading the index in
* the given Directory, using a specific commit and with
* a custom {@link IndexDeletionPolicy} and specified
- * {@link Codecs}. You should pass readOnly=true, since
+ * {@link CodecProvider}. You should pass readOnly=true, since
* it gives much better concurrent performance, unless
* you intend to do write operations (delete documents or
* change norms) with the reader.
@@ -398,18 +398,18 @@ public abstract class IndexReader implem
* memory usage, at the expense of higher latency when
* loading a TermInfo. The default value is 1. Set this
* to -1 to skip loading the terms index entirely.
- * @param codecs Codecs to use when opening index
+ * @param codecs CodecProvider to use when opening index
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, Codecs codecs) throws CorruptIndexException, IOException {
+ public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException {
return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor, codecs);
}
private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor,
- Codecs codecs) throws CorruptIndexException, IOException {
+ CodecProvider codecs) throws CorruptIndexException, IOException {
if (codecs == null) {
- codecs = Codecs.getDefault();
+ codecs = CodecProvider.getDefault();
}
return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, codecs);
}
@@ -553,7 +553,7 @@ public abstract class IndexReader implem
* @throws IOException if there is a low-level IO error
*/
public static long getCurrentVersion(Directory directory) throws CorruptIndexException, IOException {
- return SegmentInfos.readCurrentVersion(directory, Codecs.getDefault());
+ return SegmentInfos.readCurrentVersion(directory, CodecProvider.getDefault());
}
/**
@@ -571,7 +571,7 @@ public abstract class IndexReader implem
* @see #getCommitUserData()
*/
public static Map<String,String> getCommitUserData(Directory directory) throws CorruptIndexException, IOException {
- return SegmentInfos.readCurrentUserData(directory, Codecs.getDefault());
+ return SegmentInfos.readCurrentUserData(directory, CodecProvider.getDefault());
}
/**
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexWriter.java Wed Mar 10 22:53:37 2010
@@ -28,7 +28,7 @@ import org.apache.lucene.store.LockObtai
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.util.Constants;
-import org.apache.lucene.index.codecs.Codecs;
+import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.util.ThreadInterruptedException;
import java.io.IOException;
@@ -983,7 +983,7 @@ public class IndexWriter implements Clos
* <code>false</code> or if there is any other low-level
* IO error
*/
- public IndexWriter(Directory d, Analyzer a, boolean create, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl, IndexingChain indexingChain, IndexCommit commit, Codecs codecs)
+ public IndexWriter(Directory d, Analyzer a, boolean create, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl, IndexingChain indexingChain, IndexCommit commit, CodecProvider codecs)
throws CorruptIndexException, LockObtainFailedException, IOException {
init(d, a, create, deletionPolicy, mfl.getLimit(), indexingChain, commit, codecs);
}
@@ -1025,7 +1025,7 @@ public class IndexWriter implements Clos
init(d, a, false, deletionPolicy, mfl.getLimit(), null, commit, null);
}
- Codecs codecs;
+ CodecProvider codecs;
private void init(Directory d, Analyzer a, IndexDeletionPolicy deletionPolicy,
int maxFieldLength, IndexingChain indexingChain, IndexCommit commit)
@@ -1039,11 +1039,11 @@ public class IndexWriter implements Clos
private void init(Directory d, Analyzer a, final boolean create,
IndexDeletionPolicy deletionPolicy, int maxFieldLength,
- IndexingChain indexingChain, IndexCommit commit, Codecs codecsIn)
+ IndexingChain indexingChain, IndexCommit commit, CodecProvider codecsIn)
throws CorruptIndexException, LockObtainFailedException, IOException {
if (codecsIn == null) {
- codecs = Codecs.getDefault();
+ codecs = CodecProvider.getDefault();
} else {
codecs = codecsIn;
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/ReadOnlyDirectoryReader.java Wed Mar 10 22:53:37 2010
@@ -18,22 +18,22 @@ package org.apache.lucene.index;
*/
import org.apache.lucene.store.Directory;
-import org.apache.lucene.index.codecs.Codecs;
+import org.apache.lucene.index.codecs.CodecProvider;
import java.io.IOException;
import java.util.Map;
class ReadOnlyDirectoryReader extends DirectoryReader {
- ReadOnlyDirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, int termInfosIndexDivisor, Codecs codecs) throws IOException {
+ ReadOnlyDirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, int termInfosIndexDivisor, CodecProvider codecs) throws IOException {
super(directory, sis, deletionPolicy, true, termInfosIndexDivisor, codecs);
}
ReadOnlyDirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts, Map<String,byte[]> oldNormsCache, boolean doClone,
- int termInfosIndexDivisor, Codecs codecs) throws IOException {
+ int termInfosIndexDivisor, CodecProvider codecs) throws IOException {
super(directory, infos, oldReaders, oldStarts, oldNormsCache, true, doClone, termInfosIndexDivisor, codecs);
}
- ReadOnlyDirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor, Codecs codecs) throws IOException {
+ ReadOnlyDirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor, CodecProvider codecs) throws IOException {
super(writer, infos, termInfosIndexDivisor, codecs);
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfo.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfo.java Wed Mar 10 22:53:37 2010
@@ -22,7 +22,7 @@ import org.apache.lucene.store.IndexOutp
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.BitVector;
import org.apache.lucene.index.codecs.Codec;
-import org.apache.lucene.index.codecs.Codecs;
+import org.apache.lucene.index.codecs.CodecProvider;
import java.io.IOException;
import java.util.List;
import java.util.Map;
@@ -169,7 +169,7 @@ public final class SegmentInfo {
* @param format format of the segments info file
* @param input input handle to read segment info from
*/
- SegmentInfo(Directory dir, int format, IndexInput input, Codecs codecs) throws IOException {
+ SegmentInfo(Directory dir, int format, IndexInput input, CodecProvider codecs) throws IOException {
this.dir = dir;
name = input.readString();
docCount = input.readInt();
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfos.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfos.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfos.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentInfos.java Wed Mar 10 22:53:37 2010
@@ -23,7 +23,7 @@ import org.apache.lucene.store.IndexOutp
import org.apache.lucene.store.ChecksumIndexOutput;
import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.NoSuchDirectoryException;
-import org.apache.lucene.index.codecs.Codecs;
+import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.util.ThreadInterruptedException;
import java.io.FileNotFoundException;
@@ -234,7 +234,7 @@ public final class SegmentInfos extends
* @throws IOException if there is a low-level IO error
*/
public final void read(Directory directory, String segmentFileName,
- Codecs codecs) throws CorruptIndexException, IOException {
+ CodecProvider codecs) throws CorruptIndexException, IOException {
boolean success = false;
// Clear any previous segments:
@@ -307,10 +307,10 @@ public final class SegmentInfos extends
* @throws IOException if there is a low-level IO error
*/
public final void read(Directory directory) throws CorruptIndexException, IOException {
- read(directory, Codecs.getDefault());
+ read(directory, CodecProvider.getDefault());
}
- public final void read(Directory directory, final Codecs codecs) throws CorruptIndexException, IOException {
+ public final void read(Directory directory, final CodecProvider codecs) throws CorruptIndexException, IOException {
generation = lastGeneration = -1;
new FindSegmentsFile(directory) {
@@ -410,7 +410,7 @@ public final class SegmentInfos extends
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public static long readCurrentVersion(Directory directory, final Codecs codecs)
+ public static long readCurrentVersion(Directory directory, final CodecProvider codecs)
throws CorruptIndexException, IOException {
// Fully read the segments file: this ensures that it's
@@ -428,7 +428,7 @@ public final class SegmentInfos extends
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public static Map<String,String> readCurrentUserData(Directory directory, Codecs codecs)
+ public static Map<String,String> readCurrentUserData(Directory directory, CodecProvider codecs)
throws CorruptIndexException, IOException {
SegmentInfos sis = new SegmentInfos();
sis.read(directory, codecs);
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentMerger.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentMerger.java Wed Mar 10 22:53:37 2010
@@ -27,7 +27,7 @@ import java.util.List;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader.FieldOption;
import org.apache.lucene.index.MergePolicy.MergeAbortedException;
-import org.apache.lucene.index.codecs.Codecs;
+import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.MergeState;
import org.apache.lucene.index.codecs.FieldsConsumer;
@@ -75,11 +75,11 @@ final class SegmentMerger {
when merging stored fields */
private final static int MAX_RAW_MERGE_DOCS = 4192;
- private final Codecs codecs;
+ private final CodecProvider codecs;
private Codec codec;
private SegmentWriteState segmentWriteState;
- SegmentMerger(Directory dir, int termIndexInterval, String name, MergePolicy.OneMerge merge, Codecs codecs) {
+ SegmentMerger(Directory dir, int termIndexInterval, String name, MergePolicy.OneMerge merge, CodecProvider codecs) {
directory = dir;
this.codecs = codecs;
segment = name;
@@ -555,7 +555,7 @@ final class SegmentMerger {
private final void mergeTerms() throws CorruptIndexException, IOException {
- // Let Codecs decide which codec will be used to write
+ // Let CodecProvider decide which codec will be used to write
// the new segment:
codec = codecs.getWriter(segmentWriteState);
@@ -631,9 +631,13 @@ final class SegmentMerger {
final FieldsConsumer consumer = codec.fieldsConsumer(segmentWriteState);
- // nocommit: somewhat stupid that we create this only to
- // have it broken apart when we step through the docs
- // enums in MultidDcsEnum.... is there a cleaner way?
+ // NOTE: this is silly, yet, necessary -- we create a
+ // MultiBits as our skip docs only to have it broken
+ // apart when we step through the docs enums in
+ // MultidDcsEnum.... this only matters when we are
+ // interacting with a non-core IR subclass, because
+ // LegacyFieldsEnum.LegacyDocs[AndPositions]Enum checks
+ // that the skipDocs matches the delDocs for the reader
mergeState.multiDeletedDocs = new MultiBits(bits, bitsStarts);
try {
Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReadState.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReadState.java?rev=921608&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReadState.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReadState.java Wed Mar 10 22:53:37 2010
@@ -0,0 +1,43 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.Directory;
+
+/**
+ * @lucene.experimental
+ */
+public class SegmentReadState {
+ public final Directory dir;
+ public final SegmentInfo segmentInfo;
+ public final FieldInfos fieldInfos;
+ public final int readBufferSize;
+ public final int termsIndexDivisor;
+
+ public SegmentReadState(Directory dir,
+ SegmentInfo info,
+ FieldInfos fieldInfos,
+ int readBufferSize,
+ int termsIndexDivisor) {
+ this.dir = dir;
+ this.segmentInfo = info;
+ this.fieldInfos = fieldInfos;
+ this.readBufferSize = readBufferSize;
+ this.termsIndexDivisor = termsIndexDivisor;
+ }
+}
\ No newline at end of file
Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReadState.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentReader.java Wed Mar 10 22:53:37 2010
@@ -40,7 +40,7 @@ import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.CloseableThreadLocal;
import org.apache.lucene.util.UnicodeUtil;
-import org.apache.lucene.index.codecs.Codecs;
+import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.preflex.PreFlexFields;
import org.apache.lucene.index.codecs.preflex.SegmentTermDocs;
@@ -95,7 +95,7 @@ public class SegmentReader extends Index
final FieldsProducer fields;
final boolean isPreFlex;
- final Codecs codecs;
+ final CodecProvider codecs;
final Directory dir;
final Directory cfsDir;
@@ -109,7 +109,7 @@ public class SegmentReader extends Index
CompoundFileReader cfsReader;
CompoundFileReader storeCFSReader;
- CoreReaders(SegmentReader origInstance, Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor, Codecs codecs) throws IOException {
+ CoreReaders(SegmentReader origInstance, Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor, CodecProvider codecs) throws IOException {
if (termsIndexDivisor < 1 && termsIndexDivisor != -1) {
throw new IllegalArgumentException("indexDivisor must be -1 (don't load terms index) or greater than 0: got " + termsIndexDivisor);
@@ -120,7 +120,7 @@ public class SegmentReader extends Index
System.out.println("sr: init core for segment=" + segment);
}
if (codecs == null) {
- codecs = Codecs.getDefault();
+ codecs = CodecProvider.getDefault();
}
this.codecs = codecs;
this.readBufferSize = readBufferSize;
@@ -144,7 +144,7 @@ public class SegmentReader extends Index
if (Codec.DEBUG) {
System.out.println("sr.core.init: seg=" + si.name + " codec=" + si.getCodec());
}
- fields = si.getCodec().fieldsProducer(cfsDir, fieldInfos, si, readBufferSize, termsIndexDivisor);
+ fields = si.getCodec().fieldsProducer(new SegmentReadState(cfsDir, si, fieldInfos, readBufferSize, termsIndexDivisor));
assert fields != null;
isPreFlex = fields instanceof PreFlexFields;
@@ -517,10 +517,10 @@ public class SegmentReader extends Index
int readBufferSize,
boolean doOpenStores,
int termInfosIndexDivisor,
- Codecs codecs)
+ CodecProvider codecs)
throws CorruptIndexException, IOException {
if (codecs == null) {
- codecs = Codecs.getDefault();
+ codecs = CodecProvider.getDefault();
}
SegmentReader instance = readOnly ? new ReadOnlySegmentReader() : new SegmentReader();
@@ -1533,11 +1533,40 @@ public class SegmentReader extends Index
}
}
+ private DocsEnum.BulkReadResult pendingBulkResult;
+ private int pendingBulk;
+
public int read(int[] docs, int[] freqs) throws IOException {
if (!any) {
return 0;
+ } else if (pendingBulk > 0) {
+ final int left = pendingBulkResult.count - pendingBulk;
+ if (docs.length >= left) {
+ // read all pending
+ System.arraycopy(pendingBulkResult.docs.ints, pendingBulk, docs, 0, left);
+ System.arraycopy(pendingBulkResult.freqs.ints, pendingBulk, freqs, 0, left);
+ pendingBulk = 0;
+ return left;
+ } else {
+ // read only part of pending
+ System.arraycopy(pendingBulkResult.docs.ints, pendingBulk, docs, 0, docs.length);
+ System.arraycopy(pendingBulkResult.freqs.ints, pendingBulk, freqs, 0, docs.length);
+ pendingBulk += docs.length;
+ return docs.length;
+ }
} else {
- return docsEnum.read(docs, freqs);
+ // nothing pending
+ pendingBulkResult = docsEnum.read();
+ if (docs.length >= pendingBulkResult.count) {
+ System.arraycopy(pendingBulkResult.docs.ints, 0, docs, 0, pendingBulkResult.count);
+ System.arraycopy(pendingBulkResult.freqs.ints, 0, freqs, 0, pendingBulkResult.count);
+ return pendingBulkResult.count;
+ } else {
+ System.arraycopy(pendingBulkResult.docs.ints, 0, docs, 0, docs.length);
+ System.arraycopy(pendingBulkResult.freqs.ints, 0, freqs, 0, docs.length);
+ pendingBulk = docs.length;
+ return docs.length;
+ }
}
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentWriteState.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentWriteState.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentWriteState.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/SegmentWriteState.java Wed Mar 10 22:53:37 2010
@@ -19,10 +19,11 @@ package org.apache.lucene.index;
import java.util.HashSet;
import java.util.Collection;
+import java.io.PrintStream;
import org.apache.lucene.store.Directory;
import org.apache.lucene.index.codecs.Codec;
-import org.apache.lucene.index.codecs.Codecs;
+import org.apache.lucene.index.codecs.CodecProvider;
/**
* This class is not meant for public usage; it's only
@@ -31,45 +32,42 @@ import org.apache.lucene.index.codecs.Co
* @lucene.experimental
*/
public class SegmentWriteState {
- // nocommit -- not clean that this is here; sometimes we
- // write a newly flushed segment; other times a merged
- // segment (and this is null):
- DocumentsWriter docWriter;
- public Directory directory;
- public String segmentName;
- public FieldInfos fieldInfos;
- String docStoreSegmentName;
- public int numDocs;
- int numDocsInStore;
- public Collection<String> flushedFiles;
+ public final PrintStream infoStream;
+ public final Directory directory;
+ public final String segmentName;
+ public final FieldInfos fieldInfos;
+ public final String docStoreSegmentName;
+ public final int numDocs;
+ public int numDocsInStore;
+ public final Collection<String> flushedFiles;
// Actual codec used
- Codec codec;
+ final Codec codec;
/** Expert: The fraction of terms in the "dictionary" which should be stored
* in RAM. Smaller values use more memory, but make searching slightly
* faster, while larger values use less memory and make searching slightly
* slower. Searching is typically not dominated by dictionary lookup, so
* tweaking this is rarely useful.*/
- public int termIndexInterval;
+ public final int termIndexInterval;
/** Expert: The fraction of {@link TermDocs} entries stored in skip tables,
* used to accelerate {@link TermDocs#skipTo(int)}. Larger values result in
* smaller indexes, greater acceleration, but fewer accelerable cases, while
* smaller values result in bigger indexes, less acceleration and more
* accelerable cases. More detailed experiments would be useful here. */
- public int skipInterval = 16;
+ public final int skipInterval = 16;
/** Expert: The maximum number of skip levels. Smaller values result in
* slightly smaller indexes, but slower skipping in big posting lists.
*/
- public int maxSkipLevels = 10;
+ public final int maxSkipLevels = 10;
- public SegmentWriteState(DocumentsWriter docWriter, Directory directory, String segmentName, FieldInfos fieldInfos,
+ public SegmentWriteState(PrintStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos,
String docStoreSegmentName, int numDocs,
int numDocsInStore, int termIndexInterval,
- Codecs codecs) {
- this.docWriter = docWriter;
+ CodecProvider codecs) {
+ this.infoStream = infoStream;
this.directory = directory;
this.segmentName = segmentName;
this.fieldInfos = fieldInfos;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/StoredFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/StoredFieldsWriter.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/StoredFieldsWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/StoredFieldsWriter.java Wed Mar 10 22:53:37 2010
@@ -90,8 +90,8 @@ final class StoredFieldsWriter {
state.flushedFiles.add(fieldsName);
state.flushedFiles.add(fieldsIdxName);
- state.docWriter.removeOpenFile(fieldsName);
- state.docWriter.removeOpenFile(fieldsIdxName);
+ docWriter.removeOpenFile(fieldsName);
+ docWriter.removeOpenFile(fieldsIdxName);
if (4+((long) state.numDocsInStore)*8 != state.directory.fileLength(fieldsIdxName))
throw new RuntimeException("after flush: fdx size mismatch: " + state.numDocsInStore + " docs vs " + state.directory.fileLength(fieldsIdxName) + " length in bytes of " + fieldsIdxName + " file exists?=" + state.directory.fileExists(fieldsIdxName));
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsEnum.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/TermsEnum.java Wed Mar 10 22:53:37 2010
@@ -58,7 +58,6 @@ public abstract class TermsEnum {
* indicate whether exact term was found, a different
* term was found, or EOF was hit. The target term may
* be befor or after the current term. */
- // nocommit -- add boolean doCache?
public abstract SeekStatus seek(BytesRef text) throws IOException;
/** Seeks to the specified term by ordinal (position) as
@@ -81,11 +80,9 @@ public abstract class TermsEnum {
/** Returns ordinal position for current term. This is an
* optional method (the codec may throw {@link
* UnsupportedOperationException}). Do not call this
- * before calling next() for the first time, after next()
- * returns null or seek returns {@link
- * SeekStatus#END}. */
- // nocommit -- should we allow calling this after next
- // returns null? and it returns 1+ max ord?
+ * before calling {@link #next} for the first time or after
+ * {@link #next} returns null or {@link #seek} returns
+ * END; */
public abstract long ord() throws IOException;
/** Returns the number of documents containing the current
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codec.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codec.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codec.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codec.java Wed Mar 10 22:53:37 2010
@@ -21,9 +21,9 @@ import java.io.IOException;
import java.util.Set;
import org.apache.lucene.index.CorruptIndexException;
-import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
@@ -56,8 +56,7 @@ public abstract class Codec {
/** Reads a segment. NOTE: by the time this call
* returns, it must hold open any files it will need to
* use; else, those files may be deleted. */
- // nocommit -- make a SegmentReadState (symmetric)?
- public abstract FieldsProducer fieldsProducer(Directory dir, FieldInfos fieldInfos, SegmentInfo si, int readBufferSize, int indexDivisor) throws IOException;
+ public abstract FieldsProducer fieldsProducer(SegmentReadState state) throws IOException;
/** Gathers files associated with this segment */
public abstract void files(Directory dir, SegmentInfo segmentInfo, Set<String> files) throws IOException;
Copied: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/CodecProvider.java (from r920448, lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codecs.java)
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/CodecProvider.java?p2=lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/CodecProvider.java&p1=lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codecs.java&r1=920448&r2=921608&rev=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/Codecs.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/CodecProvider.java Wed Mar 10 22:53:37 2010
@@ -34,9 +34,10 @@ import org.apache.lucene.index.codecs.st
* pass this instance to IndexReader/IndexWriter (via
* package private APIs) to use different codecs when
* reading & writing segments.
+ *
* @lucene.experimental */
-public abstract class Codecs {
+public abstract class CodecProvider {
private final HashMap<String, Codec> codecs = new HashMap<String, Codec>();
@@ -68,15 +69,15 @@ public abstract class Codecs {
public abstract Codec getWriter(SegmentWriteState state);
- static private final Codecs defaultCodecs = new DefaultCodecs();
+ static private final CodecProvider defaultCodecs = new DefaultCodecProvider();
- public static Codecs getDefault() {
+ public static CodecProvider getDefault() {
return defaultCodecs;
}
}
-class DefaultCodecs extends Codecs {
- DefaultCodecs() {
+class DefaultCodecProvider extends CodecProvider {
+ DefaultCodecProvider() {
register(new StandardCodec());
register(new IntBlockCodec());
register(new PreFlexCodec());
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/intblock/IntBlockCodec.java Wed Mar 10 22:53:37 2010
@@ -20,9 +20,9 @@ package org.apache.lucene.index.codecs.i
import java.io.IOException;
import java.util.Set;
-import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
@@ -82,16 +82,19 @@ public class IntBlockCodec extends Codec
}
@Override
- public FieldsProducer fieldsProducer(Directory dir, FieldInfos fieldInfos, SegmentInfo si, int readBufferSize, int indexDivisor) throws IOException {
- StandardPostingsReader postingsReader = new SepPostingsReaderImpl(dir, si, readBufferSize, new SimpleIntBlockFactory(1024));
+ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
+ StandardPostingsReader postingsReader = new SepPostingsReaderImpl(state.dir,
+ state.segmentInfo,
+ state.readBufferSize,
+ new SimpleIntBlockFactory(1024));
StandardTermsIndexReader indexReader;
boolean success = false;
try {
- indexReader = new SimpleStandardTermsIndexReader(dir,
- fieldInfos,
- si.name,
- indexDivisor,
+ indexReader = new SimpleStandardTermsIndexReader(state.dir,
+ state.fieldInfos,
+ state.segmentInfo.name,
+ state.termsIndexDivisor,
BytesRef.getUTF8SortedAsUTF16Comparator());
success = true;
} finally {
@@ -103,9 +106,11 @@ public class IntBlockCodec extends Codec
success = false;
try {
FieldsProducer ret = new StandardTermsDictReader(indexReader,
- dir, fieldInfos, si.name,
+ state.dir,
+ state.fieldInfos,
+ state.segmentInfo.name,
postingsReader,
- readBufferSize,
+ state.readBufferSize,
BytesRef.getUTF8SortedAsUTF16Comparator(),
StandardCodec.TERMS_CACHE_SIZE);
success = true;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java Wed Mar 10 22:53:37 2010
@@ -24,7 +24,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
@@ -61,8 +61,8 @@ public class PreFlexCodec extends Codec
}
@Override
- public FieldsProducer fieldsProducer(Directory dir, FieldInfos fieldInfos, SegmentInfo info, int readBufferSize, int indexDivisor) throws IOException {
- return new PreFlexFields(dir, fieldInfos, info, readBufferSize, indexDivisor);
+ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
+ return new PreFlexFields(state.dir, state.fieldInfos, state.segmentInfo, state.readBufferSize, state.termsIndexDivisor);
}
@Override
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java Wed Mar 10 22:53:37 2010
@@ -57,7 +57,6 @@ public class PreFlexFields extends Field
private final int readBufferSize;
private Directory cfsReader;
- // nocommit -- we need the legacy terms cache back in here
PreFlexFields(Directory dir, FieldInfos fieldInfos, SegmentInfo info, int readBufferSize, int indexDivisor)
throws IOException {
@@ -147,8 +146,6 @@ public class PreFlexFields extends Field
// terms reader with index, the segment has switched
// to CFS
- // nocommit -- not clean that I open my own CFS
- // reader here; caller should pass it in?
if (!(dir instanceof CompoundFileReader)) {
dir0 = cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(si.name, IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
} else {
@@ -247,10 +244,13 @@ public class PreFlexFields extends Field
final Term t = termEnum.term();
if (t != null && t.field() == fieldInfo.name) {
// No need to seek -- we have already advanced onto
- // this field
+ // this field. We must be @ first term because
+ // flex API will not advance this enum further, on
+ // seeing a different field.
} else {
assert t == null || !t.field().equals(fieldInfo.name); // make sure field name is interned
- termEnum = getTermsDict().terms(new Term(fieldInfo.name, ""));
+ final TermInfosReader tis = getTermsDict();
+ tis.seekEnum(termEnum, new Term(fieldInfo.name, ""));
}
skipNext = true;
}
@@ -278,7 +278,13 @@ public class PreFlexFields extends Field
System.out.println("pff.seek term=" + term);
}
skipNext = false;
- termEnum = getTermsDict().terms(new Term(fieldInfo.name, term.utf8ToString()));
+ final TermInfosReader tis = getTermsDict();
+ final Term t0 = new Term(fieldInfo.name, term.utf8ToString());
+ if (termEnum == null) {
+ termEnum = tis.terms(t0);
+ } else {
+ tis.seekEnum(termEnum, t0);
+ }
final Term t = termEnum.term();
final BytesRef tr;
@@ -411,8 +417,14 @@ public class PreFlexFields extends Field
}
@Override
- public int read(int[] docs, int[] freqs) throws IOException {
- return this.docs.read(docs, freqs);
+ public BulkReadResult read() throws IOException {
+ if (bulkResult == null) {
+ initBulkResult();
+ bulkResult.docs.ints = new int[32];
+ bulkResult.freqs.ints = new int[32];
+ }
+ bulkResult.count = this.docs.read(bulkResult.docs.ints, bulkResult.freqs.ints);
+ return bulkResult;
}
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java Wed Mar 10 22:53:37 2010
@@ -129,7 +129,7 @@ public final class SegmentTermEnum exten
return clone;
}
- final void seek(long pointer, int p, Term t, TermInfo ti)
+ final void seek(long pointer, long p, Term t, TermInfo ti)
throws IOException {
input.seek(pointer);
position = p;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java Wed Mar 10 22:53:37 2010
@@ -23,7 +23,7 @@ package org.apache.lucene.index.codecs.p
* indexing. */
@Deprecated
-final class TermInfo {
+class TermInfo {
/** The number of documents which contain the term. */
int docFreq = 0;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java Wed Mar 10 22:53:37 2010
@@ -25,8 +25,8 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.CloseableThreadLocal;
+import org.apache.lucene.util.cache.DoubleBarrelLRUCache;
import org.apache.lucene.util.cache.Cache;
-import org.apache.lucene.util.cache.SimpleLRUCache;
/** This stores a monotonically increasing set of <Term, TermInfo> pairs in a
* Directory. Pairs are accessed either by Term or by ordinal position the
@@ -53,14 +53,22 @@ public final class TermInfosReader {
private final static int DEFAULT_CACHE_SIZE = 1024;
+ // Just adds term's ord to TermInfo
+ private final static class TermInfoAndOrd extends TermInfo {
+ final int termOrd;
+ public TermInfoAndOrd(TermInfo ti, int termOrd) {
+ super(ti);
+ this.termOrd = termOrd;
+ }
+ }
+
+ private final Cache<Term,TermInfoAndOrd> termsCache = new DoubleBarrelLRUCache<Term,TermInfoAndOrd>(DEFAULT_CACHE_SIZE);
+
/**
* Per-thread resources managed by ThreadLocal
*/
private static final class ThreadResources {
SegmentTermEnum termEnum;
-
- // Used for caching the least recently looked-up Terms
- Cache<Term,TermInfo> termInfoCache;
}
TermInfosReader(Directory dir, String seg, FieldInfos fis, int readBufferSize, int indexDivisor)
@@ -138,6 +146,7 @@ public final class TermInfosReader {
if (origEnum != null)
origEnum.close();
threadResources.close();
+ termsCache.close();
}
/** Returns the number of term/value pairs in the set. */
@@ -146,12 +155,10 @@ public final class TermInfosReader {
}
private ThreadResources getThreadResources() {
- ThreadResources resources = (ThreadResources)threadResources.get();
+ ThreadResources resources = threadResources.get();
if (resources == null) {
resources = new ThreadResources();
resources.termEnum = terms();
- // Cache does not have to be thread-safe, it is only used by one thread at the same time
- resources.termInfoCache = new SimpleLRUCache<Term,TermInfo>(DEFAULT_CACHE_SIZE);
threadResources.set(resources);
}
return resources;
@@ -178,36 +185,38 @@ public final class TermInfosReader {
private final void seekEnum(SegmentTermEnum enumerator, int indexOffset) throws IOException {
enumerator.seek(indexPointers[indexOffset],
- (indexOffset * totalIndexInterval) - 1,
- indexTerms[indexOffset], indexInfos[indexOffset]);
+ ((long) indexOffset * totalIndexInterval) - 1,
+ indexTerms[indexOffset], indexInfos[indexOffset]);
}
/** Returns the TermInfo for a Term in the set, or null. */
TermInfo get(Term term) throws IOException {
- return get(term, true);
+ return get(term, false);
}
/** Returns the TermInfo for a Term in the set, or null. */
- private TermInfo get(Term term, boolean useCache) throws IOException {
+ private TermInfo get(Term term, boolean mustSeekEnum) throws IOException {
if (size == 0) return null;
ensureIndexIsRead();
- TermInfo ti;
+ TermInfoAndOrd tiOrd = termsCache.get(term);
ThreadResources resources = getThreadResources();
- Cache<Term,TermInfo> cache = null;
-
- if (useCache) {
- cache = resources.termInfoCache;
- // check the cache first if the term was recently looked up
- ti = (TermInfo) cache.get(term);
- if (ti != null) {
- return ti;
- }
+
+ if (!mustSeekEnum && tiOrd != null) {
+ return tiOrd;
}
+ return seekEnum(resources.termEnum, term, tiOrd);
+ }
+
+ TermInfo seekEnum(SegmentTermEnum enumerator, Term term) throws IOException {
+ return seekEnum(enumerator, term, termsCache.get(term));
+ }
+
+ TermInfo seekEnum(SegmentTermEnum enumerator, Term term, TermInfoAndOrd tiOrd) throws IOException {
+
// optimize sequential access: first try scanning cached enum w/o seeking
- SegmentTermEnum enumerator = resources.termEnum;
if (enumerator.term() != null // term is at or past current
&& ((enumerator.prev() != null && term.compareTo(enumerator.prev())> 0)
|| term.compareTo(enumerator.term()) >= 0)) {
@@ -216,16 +225,23 @@ public final class TermInfosReader {
|| term.compareTo(indexTerms[enumOffset]) < 0) {
// no need to seek
+ final TermInfo ti;
+
int numScans = enumerator.scanTo(term);
if (enumerator.term() != null && term.compareTo(enumerator.term()) == 0) {
ti = enumerator.termInfo();
- if (cache != null && numScans > 1) {
+ if (numScans > 1) {
// we only want to put this TermInfo into the cache if
// scanEnum skipped more than one dictionary entry.
// This prevents RangeQueries or WildcardQueries to
// wipe out the cache when they iterate over a large numbers
// of terms in order
- cache.put(term, ti);
+ if (tiOrd == null) {
+ termsCache.put(term, new TermInfoAndOrd(ti, (int) enumerator.position));
+ } else {
+ assert sameTermInfo(ti, tiOrd, enumerator);
+ assert (int) enumerator.position == tiOrd.termOrd;
+ }
}
} else {
ti = null;
@@ -236,12 +252,24 @@ public final class TermInfosReader {
}
// random-access: must seek
- seekEnum(enumerator, getIndexOffset(term));
+ final int indexPos;
+ if (tiOrd != null) {
+ indexPos = tiOrd.termOrd / totalIndexInterval;
+ } else {
+ // Must do binary search:
+ indexPos = getIndexOffset(term);
+ }
+
+ seekEnum(enumerator, indexPos);
enumerator.scanTo(term);
+ final TermInfo ti;
if (enumerator.term() != null && term.compareTo(enumerator.term()) == 0) {
ti = enumerator.termInfo();
- if (cache != null) {
- cache.put(term, ti);
+ if (tiOrd == null) {
+ termsCache.put(term, new TermInfoAndOrd(ti, (int) enumerator.position));
+ } else {
+ assert sameTermInfo(ti, tiOrd, enumerator);
+ assert (int) enumerator.position == tiOrd.termOrd;
}
} else {
ti = null;
@@ -249,26 +277,23 @@ public final class TermInfosReader {
return ti;
}
- /** Returns the nth term in the set. */
- final Term get(int position) throws IOException {
- if (size == 0) return null;
-
- SegmentTermEnum enumerator = getThreadResources().termEnum;
- if (enumerator.term() != null &&
- position >= enumerator.position &&
- position < (enumerator.position + totalIndexInterval))
- return scanEnum(enumerator, position); // can avoid seek
-
- seekEnum(enumerator, position/totalIndexInterval); // must seek
- return scanEnum(enumerator, position);
- }
-
- private final Term scanEnum(SegmentTermEnum enumerator, int position) throws IOException {
- while(enumerator.position < position)
- if (!enumerator.next())
- return null;
-
- return enumerator.term();
+ // called only from asserts
+ private final boolean sameTermInfo(TermInfo ti1, TermInfo ti2, SegmentTermEnum enumerator) {
+ if (ti1.docFreq != ti2.docFreq) {
+ return false;
+ }
+ if (ti1.freqPointer != ti2.freqPointer) {
+ return false;
+ }
+ if (ti1.proxPointer != ti2.proxPointer) {
+ return false;
+ }
+ // skipOffset is only valid when docFreq >= skipInterval:
+ if (ti1.docFreq >= enumerator.skipInterval &&
+ ti1.skipOffset != ti2.skipOffset) {
+ return false;
+ }
+ return true;
}
private void ensureIndexIsRead() {
@@ -297,14 +322,12 @@ public final class TermInfosReader {
/** Returns an enumeration of all the Terms and TermInfos in the set. */
public SegmentTermEnum terms() {
- return (SegmentTermEnum) origEnum.clone();
+ return (SegmentTermEnum)origEnum.clone();
}
/** Returns an enumeration of terms starting at or after the named term. */
public SegmentTermEnum terms(Term term) throws IOException {
- // don't use the cache in this call because we want to reposition the
- // enumeration
- get(term, false);
+ get(term, true);
return (SegmentTermEnum)getThreadResources().termEnum.clone();
}
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java Wed Mar 10 22:53:37 2010
@@ -20,9 +20,9 @@ package org.apache.lucene.index.codecs.p
import java.io.IOException;
import java.util.Set;
-import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.standard.StandardPostingsWriter;
import org.apache.lucene.index.codecs.standard.StandardPostingsWriterImpl;
@@ -95,11 +95,11 @@ public class PulsingCodec extends Codec
}
@Override
- public FieldsProducer fieldsProducer(Directory dir, FieldInfos fieldInfos, SegmentInfo si, int readBufferSize, int indexDivisor) throws IOException {
+ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
// We wrap StandardPostingsReaderImpl, but any StandardPostingsReader
// will work:
- StandardPostingsReader docsReader = new StandardPostingsReaderImpl(dir, si, readBufferSize);
+ StandardPostingsReader docsReader = new StandardPostingsReaderImpl(state.dir, state.segmentInfo, state.readBufferSize);
StandardPostingsReader pulsingReader = new PulsingPostingsReaderImpl(docsReader);
// Terms dict index reader
@@ -107,10 +107,10 @@ public class PulsingCodec extends Codec
boolean success = false;
try {
- indexReader = new SimpleStandardTermsIndexReader(dir,
- fieldInfos,
- si.name,
- indexDivisor,
+ indexReader = new SimpleStandardTermsIndexReader(state.dir,
+ state.fieldInfos,
+ state.segmentInfo.name,
+ state.termsIndexDivisor,
BytesRef.getUTF8SortedAsUTF16Comparator());
success = true;
} finally {
@@ -123,9 +123,9 @@ public class PulsingCodec extends Codec
success = false;
try {
FieldsProducer ret = new StandardTermsDictReader(indexReader,
- dir, fieldInfos, si.name,
+ state.dir, state.fieldInfos, state.segmentInfo.name,
pulsingReader,
- readBufferSize,
+ state.readBufferSize,
BytesRef.getUTF8SortedAsUTF16Comparator(),
StandardCodec.TERMS_CACHE_SIZE);
success = true;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReaderImpl.java Wed Mar 10 22:53:37 2010
@@ -269,9 +269,12 @@ public class PulsingPostingsReaderImpl e
}
@Override
- public int read(int[] docs, int[] freqs) {
+ public BulkReadResult read() {
int i=0;
// TODO: -- ob1?
+ initBulkResult();
+ final int[] docs = bulkResult.docs.ints;
+ final int[] freqs = bulkResult.freqs.ints;
while(nextRead < state.docFreq) {
doc = state.docs[nextRead++];
if (skipDocs == null || !skipDocs.get(doc.docID)) {
@@ -280,7 +283,8 @@ public class PulsingPostingsReaderImpl e
i++;
}
}
- return i;
+ bulkResult.count = i;
+ return bulkResult;
}
@Override
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepCodec.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepCodec.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepCodec.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepCodec.java Wed Mar 10 22:53:37 2010
@@ -20,9 +20,9 @@ package org.apache.lucene.index.codecs.s
import java.io.IOException;
import java.util.Set;
-import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
@@ -84,17 +84,17 @@ public class SepCodec extends Codec {
final static String PAYLOAD_EXTENSION = "pyl";
@Override
- public FieldsProducer fieldsProducer(Directory dir, FieldInfos fieldInfos, SegmentInfo si, int readBufferSize, int indexDivisor) throws IOException {
+ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
- StandardPostingsReader postingsReader = new SepPostingsReaderImpl(dir, si, readBufferSize, new SingleIntFactory());
+ StandardPostingsReader postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo, state.readBufferSize, new SingleIntFactory());
StandardTermsIndexReader indexReader;
boolean success = false;
try {
- indexReader = new SimpleStandardTermsIndexReader(dir,
- fieldInfos,
- si.name,
- indexDivisor,
+ indexReader = new SimpleStandardTermsIndexReader(state.dir,
+ state.fieldInfos,
+ state.segmentInfo.name,
+ state.termsIndexDivisor,
BytesRef.getUTF8SortedAsUTF16Comparator());
success = true;
} finally {
@@ -106,9 +106,11 @@ public class SepCodec extends Codec {
success = false;
try {
FieldsProducer ret = new StandardTermsDictReader(indexReader,
- dir, fieldInfos, si.name,
+ state.dir,
+ state.fieldInfos,
+ state.segmentInfo.name,
postingsReader,
- readBufferSize,
+ state.readBufferSize,
BytesRef.getUTF8SortedAsUTF16Comparator(),
StandardCodec.TERMS_CACHE_SIZE);
success = true;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java Wed Mar 10 22:53:37 2010
@@ -360,8 +360,11 @@ public class SepPostingsReaderImpl exten
}
@Override
- public int read(int[] docs, int[] freqs) throws IOException {
+ public BulkReadResult read() throws IOException {
// TODO: -- switch to bulk read api in IntIndexInput
+ initBulkResult();
+ final int[] docs = bulkResult.docs.ints;
+ final int[] freqs = bulkResult.freqs.ints;
int i = 0;
final int length = docs.length;
while (i < length && count < docFreq) {
@@ -378,8 +381,8 @@ public class SepPostingsReaderImpl exten
i++;
}
}
-
- return i;
+ bulkResult.count = i;
+ return bulkResult;
}
@Override
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java Wed Mar 10 22:53:37 2010
@@ -20,9 +20,9 @@ package org.apache.lucene.index.codecs.s
import java.io.IOException;
import java.util.Set;
-import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.SegmentWriteState;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.FieldsConsumer;
@@ -73,16 +73,16 @@ public class StandardCodec extends Codec
public final static int TERMS_CACHE_SIZE = 1024;
@Override
- public FieldsProducer fieldsProducer(Directory dir, FieldInfos fieldInfos, SegmentInfo si, int readBufferSize, int indexDivisor) throws IOException {
- StandardPostingsReader postings = new StandardPostingsReaderImpl(dir, si, readBufferSize);
+ public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
+ StandardPostingsReader postings = new StandardPostingsReaderImpl(state.dir, state.segmentInfo, state.readBufferSize);
StandardTermsIndexReader indexReader;
boolean success = false;
try {
- indexReader = new SimpleStandardTermsIndexReader(dir,
- fieldInfos,
- si.name,
- indexDivisor,
+ indexReader = new SimpleStandardTermsIndexReader(state.dir,
+ state.fieldInfos,
+ state.segmentInfo.name,
+ state.termsIndexDivisor,
BytesRef.getUTF8SortedAsUTF16Comparator());
success = true;
} finally {
@@ -94,9 +94,11 @@ public class StandardCodec extends Codec
success = false;
try {
FieldsProducer ret = new StandardTermsDictReader(indexReader,
- dir, fieldInfos, si.name,
+ state.dir,
+ state.fieldInfos,
+ state.segmentInfo.name,
postings,
- readBufferSize,
+ state.readBufferSize,
BytesRef.getUTF8SortedAsUTF16Comparator(),
TERMS_CACHE_SIZE);
success = true;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReaderImpl.java Wed Mar 10 22:53:37 2010
@@ -280,10 +280,13 @@ public class StandardPostingsReaderImpl
}
@Override
- public int read(int[] docs, int[] freqs) throws IOException {
+ public BulkReadResult read() throws IOException {
if (Codec.DEBUG) {
Codec.debug("sdr.bulk read: ord=" + ord + " df=" + limit + " omitTF=" + omitTF + " ord=" + ord + " of " + limit + " freq.fp=" + freqIn.getFilePointer(), desc);
}
+ initBulkResult();
+ final int[] docs = bulkResult.docs.ints;
+ final int[] freqs = bulkResult.freqs.ints;
int i = 0;
final int length = docs.length;
while (i < length && ord < limit) {
@@ -310,11 +313,13 @@ public class StandardPostingsReaderImpl
++i;
}
}
+
if (Codec.DEBUG) {
System.out.println(" return " + i);
}
- return i;
+ bulkResult.count = i;
+ return bulkResult;
}
@Override
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java?rev=921608&r1=921607&r2=921608&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java Wed Mar 10 22:53:37 2010
@@ -131,8 +131,6 @@ public class MultiTermQueryWrapperFilter
if (termsEnum.next() != null) {
// fill into a OpenBitSet
final OpenBitSet bitSet = new OpenBitSet(reader.maxDoc());
- final int[] docs = new int[32];
- final int[] freqs = new int[32];
int termCount = 0;
final Bits delDocs = MultiFields.getDeletedDocs(reader);
DocsEnum docsEnum = null;
@@ -142,9 +140,10 @@ public class MultiTermQueryWrapperFilter
// enumerator.term().toBytesString());
docsEnum = termsEnum.docs(delDocs, docsEnum);
while (true) {
- final int count = docsEnum.read(docs, freqs);
- if (count != 0) {
- for (int i = 0; i < count; i++) {
+ final DocsEnum.BulkReadResult result = docsEnum.read();
+ if (result.count != 0) {
+ final int[] docs = result.docs.ints;
+ for (int i = 0; i < result.count; i++) {
bitSet.set(docs[i]);
}
} else {