You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by us...@apache.org on 2010/04/06 21:19:36 UTC
svn commit: r931278 [4/10] - in /lucene/dev/trunk: lucene/
lucene/backwards/src/ lucene/backwards/src/java/org/apache/lucene/index/
lucene/backwards/src/java/org/apache/lucene/index/codecs/
lucene/backwards/src/java/org/apache/lucene/search/ lucene/bac...
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DirectoryReader.java Tue Apr 6 19:19:27 2010
@@ -25,7 +25,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -35,6 +35,11 @@ import org.apache.lucene.search.Similari
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
+import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.ReaderUtil;
+import org.apache.lucene.util.BytesRef;
+
import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
/**
@@ -43,13 +48,14 @@ import org.apache.lucene.search.FieldCac
class DirectoryReader extends IndexReader implements Cloneable {
protected Directory directory;
protected boolean readOnly;
+
+ protected CodecProvider codecs;
IndexWriter writer;
private IndexDeletionPolicy deletionPolicy;
private Lock writeLock;
private SegmentInfos segmentInfos;
- private SegmentInfos segmentInfosStart;
private boolean stale;
private final int termInfosIndexDivisor;
@@ -58,34 +64,57 @@ class DirectoryReader extends IndexReade
private SegmentReader[] subReaders;
private int[] starts; // 1st docno for each segment
+ private final Map<SegmentReader,ReaderUtil.Slice> subReaderToSlice = new HashMap<SegmentReader,ReaderUtil.Slice>();
private Map<String,byte[]> normsCache = new HashMap<String,byte[]>();
private int maxDoc = 0;
private int numDocs = -1;
private boolean hasDeletions = false;
+// static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly,
+// final int termInfosIndexDivisor) throws CorruptIndexException, IOException {
+// return open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, null);
+// }
+
static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly,
- final int termInfosIndexDivisor) throws CorruptIndexException, IOException {
+ final int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException {
+ final CodecProvider codecs2;
+ if (codecs == null) {
+ codecs2 = CodecProvider.getDefault();
+ } else {
+ codecs2 = codecs;
+ }
return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) {
@Override
protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
SegmentInfos infos = new SegmentInfos();
- infos.read(directory, segmentFileName);
+ infos.read(directory, segmentFileName, codecs2);
if (readOnly)
- return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor);
+ return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor, codecs2);
else
- return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor);
+ return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor, codecs2);
}
}.run(commit);
}
/** Construct reading the named set of readers. */
- DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException {
+// DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException {
+// this(directory, sis, deletionPolicy, readOnly, termInfosIndexDivisor, null);
+// }
+
+ /** Construct reading the named set of readers. */
+ DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws IOException {
this.directory = directory;
this.readOnly = readOnly;
this.segmentInfos = sis;
this.deletionPolicy = deletionPolicy;
this.termInfosIndexDivisor = termInfosIndexDivisor;
+ if (codecs == null) {
+ this.codecs = CodecProvider.getDefault();
+ } else {
+ this.codecs = codecs;
+ }
+
// To reduce the chance of hitting FileNotFound
// (and having to retry), we open segments in
// reverse because IndexWriter merges & deletes
@@ -115,12 +144,16 @@ class DirectoryReader extends IndexReade
}
// Used by near real-time search
- DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor) throws IOException {
+ DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor, CodecProvider codecs) throws IOException {
this.directory = writer.getDirectory();
this.readOnly = true;
segmentInfos = infos;
- segmentInfosStart = (SegmentInfos) infos.clone();
this.termInfosIndexDivisor = termInfosIndexDivisor;
+ if (codecs == null) {
+ this.codecs = CodecProvider.getDefault();
+ } else {
+ this.codecs = codecs;
+ }
// IndexWriter synchronizes externally before calling
// us, which ensures infos will not change; so there's
@@ -166,11 +199,17 @@ class DirectoryReader extends IndexReade
/** This constructor is only used for {@link #reopen()} */
DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts,
- Map<String,byte[]> oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor) throws IOException {
+ Map<String,byte[]> oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor, CodecProvider codecs) throws IOException {
this.directory = directory;
this.readOnly = readOnly;
this.segmentInfos = infos;
this.termInfosIndexDivisor = termInfosIndexDivisor;
+ if (codecs == null) {
+ this.codecs = CodecProvider.getDefault();
+ } else {
+ this.codecs = codecs;
+ }
+
// we put the old SegmentReaders in a map, that allows us
// to lookup a reader using its segment name
@@ -296,25 +335,45 @@ class DirectoryReader extends IndexReade
buffer.append(' ');
}
buffer.append(subReaders[i]);
+ buffer.append(' ');
}
buffer.append(')');
return buffer.toString();
}
- private void initialize(SegmentReader[] subReaders) {
+ private void initialize(SegmentReader[] subReaders) throws IOException {
this.subReaders = subReaders;
starts = new int[subReaders.length + 1]; // build starts array
+
+ final List<Fields> subFields = new ArrayList<Fields>();
+ final List<ReaderUtil.Slice> fieldSlices = new ArrayList<ReaderUtil.Slice>();
+
for (int i = 0; i < subReaders.length; i++) {
starts[i] = maxDoc;
maxDoc += subReaders[i].maxDoc(); // compute maxDocs
- if (subReaders[i].hasDeletions())
+ if (subReaders[i].hasDeletions()) {
hasDeletions = true;
+ }
+
+ final ReaderUtil.Slice slice = new ReaderUtil.Slice(starts[i], subReaders[i].maxDoc(), i);
+ subReaderToSlice.put(subReaders[i], slice);
+
+ final Fields f = subReaders[i].fields();
+ if (f != null) {
+ subFields.add(f);
+ fieldSlices.add(slice);
+ }
}
starts[subReaders.length] = maxDoc;
}
@Override
+ public Bits getDeletedDocs() {
+ throw new UnsupportedOperationException("please use MultiFields.getDeletedDocs if you really need a top level Bits deletedDocs (NOTE that it's usually better to work per segment instead)");
+ }
+
+ @Override
public final synchronized Object clone() {
try {
return clone(readOnly); // Preserve current readOnly
@@ -435,7 +494,7 @@ class DirectoryReader extends IndexReade
@Override
protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
SegmentInfos infos = new SegmentInfos();
- infos.read(directory, segmentFileName);
+ infos.read(directory, segmentFileName, codecs);
return doReopen(infos, false, openReadOnly);
}
}.run(commit);
@@ -444,9 +503,9 @@ class DirectoryReader extends IndexReade
private synchronized DirectoryReader doReopen(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException {
DirectoryReader reader;
if (openReadOnly) {
- reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor);
+ reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor, null);
} else {
- reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor);
+ reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor, null);
}
return reader;
}
@@ -640,7 +699,7 @@ class DirectoryReader extends IndexReade
// Optimize single segment case:
return subReaders[0].terms();
} else {
- return new MultiTermEnum(this, subReaders, starts, null);
+ return new MultiTermEnum(this, subReaders, starts, null);
}
}
@@ -665,6 +724,16 @@ class DirectoryReader extends IndexReade
}
@Override
+ public int docFreq(String field, BytesRef term) throws IOException {
+ ensureOpen();
+ int total = 0; // sum freqs in segments
+ for (int i = 0; i < subReaders.length; i++) {
+ total += subReaders[i].docFreq(field, term);
+ }
+ return total;
+ }
+
+ @Override
public TermDocs termDocs() throws IOException {
ensureOpen();
if (subReaders.length == 1) {
@@ -687,6 +756,11 @@ class DirectoryReader extends IndexReade
}
@Override
+ public Fields fields() throws IOException {
+ throw new UnsupportedOperationException("please use MultiFields.getFields if you really need a top level Fields (NOTE that it's usually better to work per segment instead)");
+ }
+
+ @Override
public TermPositions termPositions() throws IOException {
ensureOpen();
if (subReaders.length == 1) {
@@ -731,7 +805,7 @@ class DirectoryReader extends IndexReade
// we have to check whether index has changed since this reader was opened.
// if so, this reader is no longer valid for deletion
- if (SegmentInfos.readCurrentVersion(directory) > segmentInfos.getVersion()) {
+ if (SegmentInfos.readCurrentVersion(directory, codecs) > segmentInfos.getVersion()) {
stale = true;
this.writeLock.release();
this.writeLock = null;
@@ -751,13 +825,18 @@ class DirectoryReader extends IndexReade
*/
@Override
protected void doCommit(Map<String,String> commitUserData) throws IOException {
+ // poll subreaders for changes
+ for (int i = 0; !hasChanges && i < subReaders.length; i++) {
+ hasChanges |= subReaders[i].hasChanges;
+ }
+
if (hasChanges) {
segmentInfos.setUserData(commitUserData);
// Default deleter (for backwards compatibility) is
// KeepOnlyLastCommitDeleter:
IndexFileDeleter deleter = new IndexFileDeleter(directory,
deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy,
- segmentInfos, null, null);
+ segmentInfos, null, null, codecs);
// Checkpoint the state we are about to change, in
// case we have to roll back:
@@ -828,20 +907,30 @@ class DirectoryReader extends IndexReade
}
@Override
+ public long getUniqueTermCount() throws IOException {
+ throw new UnsupportedOperationException("");
+ }
+
+ @Override
public Map<String,String> getCommitUserData() {
ensureOpen();
return segmentInfos.getUserData();
}
+ /**
+ * Check whether this IndexReader is still using the current (i.e., most recently committed) version of the index. If
+ * a writer has committed any changes to the index since this reader was opened, this will return <code>false</code>,
+ * in which case you must open a new IndexReader in order
+ * to see the changes. Use {@link IndexWriter#commit} to
+ * commit changes to the index.
+ *
+ * @throws CorruptIndexException if the index is corrupt
+ * @throws IOException if there is a low-level IO error
+ */
@Override
public boolean isCurrent() throws CorruptIndexException, IOException {
ensureOpen();
- if (writer == null || writer.isClosed()) {
- // we loaded SegmentInfos from the directory
- return SegmentInfos.readCurrentVersion(directory) == segmentInfos.getVersion();
- } else {
- return writer.nrtIsCurrent(segmentInfosStart);
- }
+ return SegmentInfos.readCurrentVersion(directory, codecs) == segmentInfos.getVersion();
}
@Override
@@ -893,6 +982,11 @@ class DirectoryReader extends IndexReade
return subReaders;
}
+ @Override
+ public int getSubReaderDocBase(IndexReader subReader) {
+ return subReaderToSlice.get(subReader).start;
+ }
+
/** Returns the directory this index resides in. */
@Override
public Directory directory() {
@@ -919,12 +1013,17 @@ class DirectoryReader extends IndexReade
/** @see org.apache.lucene.index.IndexReader#listCommits */
public static Collection<IndexCommit> listCommits(Directory dir) throws IOException {
+ return listCommits(dir, CodecProvider.getDefault());
+ }
+
+ /** @see org.apache.lucene.index.IndexReader#listCommits */
+ public static Collection<IndexCommit> listCommits(Directory dir, CodecProvider codecs) throws IOException {
final String[] files = dir.listAll();
Collection<IndexCommit> commits = new ArrayList<IndexCommit>();
SegmentInfos latest = new SegmentInfos();
- latest.read(dir);
+ latest.read(dir, codecs);
final long currentGen = latest.getGeneration();
commits.add(new ReaderCommit(latest, dir));
@@ -941,7 +1040,7 @@ class DirectoryReader extends IndexReade
try {
// IOException allowed to throw there, in case
// segments_N is corrupt
- sis.read(dir, fileName);
+ sis.read(dir, fileName, codecs);
} catch (FileNotFoundException fnfe) {
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
@@ -1020,30 +1119,34 @@ class DirectoryReader extends IndexReade
return userData;
}
}
-
+
+ // @deprecated This is pre-flex API
+ // Exposes pre-flex API by doing on-the-fly merging
+ // pre-flex API to each segment
static class MultiTermEnum extends TermEnum {
IndexReader topReader; // used for matching TermEnum to TermDocs
- private SegmentMergeQueue queue;
+ private LegacySegmentMergeQueue queue;
private Term term;
private int docFreq;
- final SegmentMergeInfo[] matchingSegments; // null terminated array of matching segments
+ final LegacySegmentMergeInfo[] matchingSegments; // null terminated array of matching segments
public MultiTermEnum(IndexReader topReader, IndexReader[] readers, int[] starts, Term t)
throws IOException {
this.topReader = topReader;
- queue = new SegmentMergeQueue(readers.length);
- matchingSegments = new SegmentMergeInfo[readers.length+1];
+ queue = new LegacySegmentMergeQueue(readers.length);
+ matchingSegments = new LegacySegmentMergeInfo[readers.length+1];
for (int i = 0; i < readers.length; i++) {
IndexReader reader = readers[i];
TermEnum termEnum;
if (t != null) {
termEnum = reader.terms(t);
- } else
+ } else {
termEnum = reader.terms();
+ }
- SegmentMergeInfo smi = new SegmentMergeInfo(starts[i], termEnum, reader);
+ LegacySegmentMergeInfo smi = new LegacySegmentMergeInfo(starts[i], termEnum, reader);
smi.ord = i;
if (t == null ? smi.next() : termEnum.term() != null)
queue.add(smi); // initialize queue
@@ -1059,7 +1162,7 @@ class DirectoryReader extends IndexReade
@Override
public boolean next() throws IOException {
for (int i=0; i<matchingSegments.length; i++) {
- SegmentMergeInfo smi = matchingSegments[i];
+ LegacySegmentMergeInfo smi = matchingSegments[i];
if (smi==null) break;
if (smi.next())
queue.add(smi);
@@ -1070,7 +1173,7 @@ class DirectoryReader extends IndexReade
int numMatchingSegments = 0;
matchingSegments[0] = null;
- SegmentMergeInfo top = queue.top();
+ LegacySegmentMergeInfo top = queue.top();
if (top == null) {
term = null;
@@ -1107,6 +1210,9 @@ class DirectoryReader extends IndexReade
}
}
+ // @deprecated This is pre-flex API
+ // Exposes pre-flex API by doing on-the-fly merging
+ // pre-flex API to each segment
static class MultiTermDocs implements TermDocs {
IndexReader topReader; // used for matching TermEnum to TermDocs
protected IndexReader[] readers;
@@ -1121,7 +1227,7 @@ class DirectoryReader extends IndexReade
private MultiTermEnum tenum; // the term enum used for seeking... can be null
int matchingSegmentPos; // position into the matching segments from tenum
- SegmentMergeInfo smi; // current segment mere info... can be null
+ LegacySegmentMergeInfo smi; // current segment mere info... can be null
public MultiTermDocs(IndexReader topReader, IndexReader[] r, int[] s) {
this.topReader = topReader;
@@ -1217,7 +1323,7 @@ class DirectoryReader extends IndexReade
return true;
} else if (pointer < readers.length) {
if (tenum != null) {
- SegmentMergeInfo smi = tenum.matchingSegments[matchingSegmentPos++];
+ LegacySegmentMergeInfo smi = tenum.matchingSegments[matchingSegmentPos++];
if (smi==null) {
pointer = readers.length;
return false;
@@ -1258,6 +1364,9 @@ class DirectoryReader extends IndexReade
}
}
+ // @deprecated This is pre-flex API
+ // Exposes pre-flex API by doing on-the-fly merging
+ // pre-flex API to each segment
static class MultiTermPositions extends MultiTermDocs implements TermPositions {
public MultiTermPositions(IndexReader topReader, IndexReader[] r, int[] s) {
super(topReader,r,s);
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Tue Apr 6 19:19:27 2010
@@ -67,7 +67,7 @@ final class DocFieldProcessor extends Do
// consumer can alter the FieldInfo* if necessary. EG,
// FreqProxTermsWriter does this with
// FieldInfo.storePayload.
- final String fileName = state.segmentFileName(IndexFileNames.FIELD_INFOS_EXTENSION);
+ final String fileName = IndexFileNames.segmentFileName(state.segmentName, IndexFileNames.FIELD_INFOS_EXTENSION);
fieldInfos.write(state.directory, fileName);
state.flushedFiles.add(fileName);
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java Tue Apr 6 19:19:27 2010
@@ -113,8 +113,9 @@ final class DocFieldProcessorPerThread e
else
lastPerField.next = perField.next;
- if (state.docWriter.infoStream != null)
- state.docWriter.infoStream.println(" purge field=" + perField.fieldInfo.name);
+ if (state.infoStream != null) {
+ state.infoStream.println(" purge field=" + perField.fieldInfo.name);
+ }
totalFieldCount--;
@@ -247,7 +248,7 @@ final class DocFieldProcessorPerThread e
fields[i].consumer.processFields(fields[i].fields, fields[i].fieldCount);
if (docState.maxTermPrefix != null && docState.infoStream != null) {
- docState.infoStream.println("WARNING: document contains at least one immense term (longer than the max length " + DocumentsWriter.MAX_TERM_LENGTH + "), all of which were skipped. Please correct the analyzer to not produce such terms. The prefix of the first immense term is: '" + docState.maxTermPrefix + "...'");
+ docState.infoStream.println("WARNING: document contains at least one immense term (whose UTF8 encoding is longer than the max length " + DocumentsWriter.MAX_TERM_LENGTH_UTF8 + "), all of which were skipped. Please correct the analyzer to not produce such terms. The prefix of the first immense term is: '" + docState.maxTermPrefix + "...'");
docState.maxTermPrefix = null;
}
@@ -389,4 +390,4 @@ final class DocFieldProcessorPerThread e
}
}
}
-}
+}
\ No newline at end of file
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerField.java Tue Apr 6 19:19:27 2010
@@ -116,8 +116,9 @@ final class DocInverterPerField extends
reader = readerValue;
else {
String stringValue = field.stringValue();
- if (stringValue == null)
+ if (stringValue == null) {
throw new IllegalArgumentException("field must have either TokenStream, String or Reader value");
+ }
perThread.stringReader.init(stringValue);
reader = perThread.stringReader;
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerThread.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerThread.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerThread.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocInverterPerThread.java Tue Apr 6 19:19:27 2010
@@ -21,7 +21,7 @@ import java.io.IOException;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
/** This is a DocFieldConsumer that inverts each field,
* separately, from a Document, and accepts a
@@ -34,16 +34,16 @@ final class DocInverterPerThread extends
final SingleTokenAttributeSource singleToken = new SingleTokenAttributeSource();
static class SingleTokenAttributeSource extends AttributeSource {
- final TermAttribute termAttribute;
+ final CharTermAttribute termAttribute;
final OffsetAttribute offsetAttribute;
private SingleTokenAttributeSource() {
- termAttribute = addAttribute(TermAttribute.class);
+ termAttribute = addAttribute(CharTermAttribute.class);
offsetAttribute = addAttribute(OffsetAttribute.class);
}
public void reinit(String stringValue, int startOffset, int endOffset) {
- termAttribute.setTermBuffer(stringValue);
+ termAttribute.setEmpty().append(stringValue);
offsetAttribute.setOffset(startOffset, endOffset);
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java Tue Apr 6 19:19:27 2010
@@ -30,6 +30,7 @@ import java.util.Map.Entry;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
+import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
@@ -41,6 +42,7 @@ import org.apache.lucene.store.RAMFile;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.ThreadInterruptedException;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.RamUsageEstimator;
/**
@@ -282,7 +284,6 @@ final class DocumentsWriter {
// If we've allocated 5% over our RAM budget, we then
// free down to 95%
- private long freeTrigger = (long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB*1024*1024*1.05);
private long freeLevel = (long) (IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB*1024*1024*0.95);
// Flush @ this number of docs. If ramBufferSize is
@@ -353,7 +354,6 @@ final class DocumentsWriter {
ramBufferSize = (long) (mb*1024*1024);
waitQueuePauseBytes = (long) (ramBufferSize*0.1);
waitQueueResumeBytes = (long) (ramBufferSize*0.05);
- freeTrigger = (long) (1.05 * ramBufferSize);
freeLevel = (long) (0.95 * ramBufferSize);
}
}
@@ -550,7 +550,6 @@ final class DocumentsWriter {
flushPending = false;
for(int i=0;i<threadStates.length;i++)
threadStates[i].doAfterFlush();
- numBytesUsed = 0;
}
// Returns true if an abort is in progress
@@ -590,9 +589,16 @@ final class DocumentsWriter {
synchronized private void initFlushState(boolean onlyDocStore) {
initSegmentName(onlyDocStore);
- flushState = new SegmentWriteState(this, directory, segment, docStoreSegment, numDocsInRAM, numDocsInStore, writer.getConfig().getTermIndexInterval());
+ flushState = new SegmentWriteState(infoStream, directory, segment, docFieldProcessor.fieldInfos,
+ docStoreSegment, numDocsInRAM, numDocsInStore, writer.getConfig().getTermIndexInterval(),
+ writer.codecs);
}
+ /** Returns the codec used to flush the last segment */
+ Codec getCodec() {
+ return flushState.codec;
+ }
+
/** Flush all pending docs to a new segment */
synchronized int flush(boolean closeDocStore) throws IOException {
@@ -628,9 +634,10 @@ final class DocumentsWriter {
consumer.flush(threads, flushState);
if (infoStream != null) {
- SegmentInfo si = new SegmentInfo(flushState.segmentName, flushState.numDocs, directory);
+ SegmentInfo si = new SegmentInfo(flushState.segmentName, flushState.numDocs, directory, flushState.codec);
+ si.setHasProx(hasProx());
final long newSegmentSize = si.sizeInBytes();
- String message = " oldRAMSize=" + numBytesUsed +
+ String message = " ramUsed=" + nf.format(((double) numBytesUsed)/1024./1024.) + " MB" +
" newFlushedSize=" + newSegmentSize +
" docs/MB=" + nf.format(numDocsInRAM/(newSegmentSize/1024./1024.)) +
" new/old=" + nf.format(100.0*newSegmentSize/numBytesUsed) + "%";
@@ -659,8 +666,9 @@ final class DocumentsWriter {
CompoundFileWriter cfsWriter = new CompoundFileWriter(directory,
IndexFileNames.segmentFileName(segment, IndexFileNames.COMPOUND_FILE_EXTENSION));
- for (final String flushedFile : flushState.flushedFiles)
- cfsWriter.addFile(flushedFile);
+ for(String fileName : flushState.flushedFiles) {
+ cfsWriter.addFile(fileName);
+ }
// Perform the merge
cfsWriter.close();
@@ -1032,28 +1040,58 @@ final class DocumentsWriter {
// Delete by term
if (deletesFlushed.terms.size() > 0) {
- TermDocs docs = reader.termDocs();
try {
+ Fields fields = reader.fields();
+ TermsEnum termsEnum = null;
+
+ String currentField = null;
+ BytesRef termRef = new BytesRef();
+ DocsEnum docs = null;
+
for (Entry<Term, BufferedDeletes.Num> entry: deletesFlushed.terms.entrySet()) {
Term term = entry.getKey();
- // LUCENE-2086: we should be iterating a TreeMap,
- // here, so terms better be in order:
+ // Since we visit terms sorted, we gain performance
+ // by re-using the same TermsEnum and seeking only
+ // forwards
+ if (term.field() != currentField) {
+ assert currentField == null || currentField.compareTo(term.field()) < 0;
+ currentField = term.field();
+ Terms terms = fields.terms(currentField);
+ if (terms != null) {
+ termsEnum = terms.iterator();
+ } else {
+ termsEnum = null;
+ }
+ }
+
+ if (termsEnum == null) {
+ continue;
+ }
assert checkDeleteTerm(term);
- docs.seek(term);
- int limit = entry.getValue().getNum();
- while (docs.next()) {
- int docID = docs.doc();
- if (docIDStart+docID >= limit)
- break;
- reader.deleteDocument(docID);
- any = true;
+
+ termRef.copy(term.text());
+
+ if (termsEnum.seek(termRef, false) == TermsEnum.SeekStatus.FOUND) {
+ DocsEnum docsEnum = termsEnum.docs(reader.getDeletedDocs(), docs);
+
+ if (docsEnum != null) {
+ docs = docsEnum;
+ int limit = entry.getValue().getNum();
+ while (true) {
+ final int docID = docs.nextDoc();
+ if (docID == DocsEnum.NO_MORE_DOCS || docIDStart+docID >= limit) {
+ break;
+ }
+ reader.deleteDocument(docID);
+ any = true;
+ }
+ }
}
}
} finally {
- docs.close();
+ //docs.close();
}
}
-
// Delete by docID
for (Integer docIdInt : deletesFlushed.docIDs) {
int docID = docIdInt.intValue();
@@ -1118,7 +1156,7 @@ final class DocumentsWriter {
}
synchronized boolean doBalanceRAM() {
- return ramBufferSize != IndexWriterConfig.DISABLE_AUTO_FLUSH && !bufferIsFull && (numBytesUsed+deletesInRAM.bytesUsed+deletesFlushed.bytesUsed >= ramBufferSize || numBytesAlloc >= freeTrigger);
+ return ramBufferSize != IndexWriterConfig.DISABLE_AUTO_FLUSH && !bufferIsFull && (numBytesUsed+deletesInRAM.bytesUsed+deletesFlushed.bytesUsed >= ramBufferSize);
}
/** Does the synchronized work to finish/flush the
@@ -1201,7 +1239,6 @@ final class DocumentsWriter {
return numBytesUsed + deletesInRAM.bytesUsed + deletesFlushed.bytesUsed;
}
- long numBytesAlloc;
long numBytesUsed;
NumberFormat nf = NumberFormat.getInstance();
@@ -1243,6 +1280,8 @@ final class DocumentsWriter {
final static int BYTE_BLOCK_MASK = BYTE_BLOCK_SIZE - 1;
final static int BYTE_BLOCK_NOT_MASK = ~BYTE_BLOCK_MASK;
+ final static int MAX_TERM_LENGTH_UTF8 = BYTE_BLOCK_SIZE-2;
+
private class ByteBlockAllocator extends ByteBlockPool.Allocator {
final int blockSize;
@@ -1259,19 +1298,16 @@ final class DocumentsWriter {
final int size = freeByteBlocks.size();
final byte[] b;
if (0 == size) {
+ b = new byte[blockSize];
// Always record a block allocated, even if
// trackAllocations is false. This is necessary
// because this block will be shared between
// things that don't track allocations (term
// vectors) and things that do (freq/prox
// postings).
- numBytesAlloc += blockSize;
- b = new byte[blockSize];
+ numBytesUsed += blockSize;
} else
b = freeByteBlocks.remove(size-1);
- if (trackAllocations)
- numBytesUsed += blockSize;
- assert numBytesUsed <= numBytesAlloc;
return b;
}
}
@@ -1291,7 +1327,7 @@ final class DocumentsWriter {
final int size = blocks.size();
for(int i=0;i<size;i++)
freeByteBlocks.add(blocks.get(i));
- }
+ }
}
}
@@ -1308,30 +1344,21 @@ final class DocumentsWriter {
final int size = freeIntBlocks.size();
final int[] b;
if (0 == size) {
+ b = new int[INT_BLOCK_SIZE];
// Always record a block allocated, even if
// trackAllocations is false. This is necessary
// because this block will be shared between
// things that don't track allocations (term
// vectors) and things that do (freq/prox
// postings).
- numBytesAlloc += INT_BLOCK_SIZE*INT_NUM_BYTE;
- b = new int[INT_BLOCK_SIZE];
+ numBytesUsed += INT_BLOCK_SIZE*INT_NUM_BYTE;
} else
b = freeIntBlocks.remove(size-1);
- if (trackAllocations)
- numBytesUsed += INT_BLOCK_SIZE*INT_NUM_BYTE;
- assert numBytesUsed <= numBytesAlloc;
return b;
}
- synchronized void bytesAllocated(long numBytes) {
- numBytesAlloc += numBytes;
- assert numBytesUsed <= numBytesAlloc;
- }
-
synchronized void bytesUsed(long numBytes) {
numBytesUsed += numBytes;
- assert numBytesUsed <= numBytesAlloc;
}
/* Return int[]s to the pool */
@@ -1346,78 +1373,34 @@ final class DocumentsWriter {
final ByteBlockAllocator perDocAllocator = new ByteBlockAllocator(PER_DOC_BLOCK_SIZE);
-
- /* Initial chunk size of the shared char[] blocks used to
- store term text */
- final static int CHAR_BLOCK_SHIFT = 14;
- final static int CHAR_BLOCK_SIZE = 1 << CHAR_BLOCK_SHIFT;
- final static int CHAR_BLOCK_MASK = CHAR_BLOCK_SIZE - 1;
-
- final static int MAX_TERM_LENGTH = CHAR_BLOCK_SIZE-1;
-
- private ArrayList<char[]> freeCharBlocks = new ArrayList<char[]>();
-
- /* Allocate another char[] from the shared pool */
- synchronized char[] getCharBlock() {
- final int size = freeCharBlocks.size();
- final char[] c;
- if (0 == size) {
- numBytesAlloc += CHAR_BLOCK_SIZE * CHAR_NUM_BYTE;
- c = new char[CHAR_BLOCK_SIZE];
- } else
- c = freeCharBlocks.remove(size-1);
- // We always track allocations of char blocks, for now,
- // because nothing that skips allocation tracking
- // (currently only term vectors) uses its own char
- // blocks.
- numBytesUsed += CHAR_BLOCK_SIZE * CHAR_NUM_BYTE;
- assert numBytesUsed <= numBytesAlloc;
- return c;
- }
-
- /* Return char[]s to the pool */
- synchronized void recycleCharBlocks(char[][] blocks, int numBlocks) {
- for(int i=0;i<numBlocks;i++)
- freeCharBlocks.add(blocks[i]);
- }
-
String toMB(long v) {
return nf.format(v/1024./1024.);
}
- /* We have four pools of RAM: Postings, byte blocks
- * (holds freq/prox posting data), char blocks (holds
- * characters in the term) and per-doc buffers (stored fields/term vectors).
- * Different docs require varying amount of storage from
- * these four classes.
- *
- * For example, docs with many unique single-occurrence
- * short terms will use up the Postings RAM and hardly any
- * of the other two. Whereas docs with very large terms
- * will use alot of char blocks RAM and relatively less of
- * the other two. This method just frees allocations from
- * the pools once we are over-budget, which balances the
- * pools to match the current docs. */
+ /* We have three pools of RAM: Postings, byte blocks
+ * (holds freq/prox posting data) and per-doc buffers
+ * (stored fields/term vectors). Different docs require
+ * varying amount of storage from these classes. For
+ * example, docs with many unique single-occurrence short
+ * terms will use up the Postings RAM and hardly any of
+ * the other two. Whereas docs with very large terms will
+ * use alot of byte blocks RAM. This method just frees
+ * allocations from the pools once we are over-budget,
+ * which balances the pools to match the current docs. */
void balanceRAM() {
- // We flush when we've used our target usage
- final long flushTrigger = ramBufferSize;
-
final long deletesRAMUsed = deletesInRAM.bytesUsed+deletesFlushed.bytesUsed;
- if (numBytesAlloc+deletesRAMUsed > freeTrigger) {
+ if (numBytesUsed+deletesRAMUsed > ramBufferSize) {
if (infoStream != null)
message(" RAM: now balance allocations: usedMB=" + toMB(numBytesUsed) +
- " vs trigger=" + toMB(flushTrigger) +
- " allocMB=" + toMB(numBytesAlloc) +
+ " vs trigger=" + toMB(ramBufferSize) +
" deletesMB=" + toMB(deletesRAMUsed) +
- " vs trigger=" + toMB(freeTrigger) +
" byteBlockFree=" + toMB(byteBlockAllocator.freeByteBlocks.size()*BYTE_BLOCK_SIZE) +
- " perDocFree=" + toMB(perDocAllocator.freeByteBlocks.size()*PER_DOC_BLOCK_SIZE) +
- " charBlockFree=" + toMB(freeCharBlocks.size()*CHAR_BLOCK_SIZE*CHAR_NUM_BYTE));
+ " perDocFree=" + toMB(perDocAllocator.freeByteBlocks.size()*PER_DOC_BLOCK_SIZE));
- final long startBytesAlloc = numBytesAlloc + deletesRAMUsed;
+ final long startBytesUsed = numBytesUsed + deletesRAMUsed;
int iter = 0;
@@ -1427,46 +1410,38 @@ final class DocumentsWriter {
boolean any = true;
- while(numBytesAlloc+deletesRAMUsed > freeLevel) {
+ while(numBytesUsed+deletesRAMUsed > freeLevel) {
synchronized(this) {
- if (0 == perDocAllocator.freeByteBlocks.size()
- && 0 == byteBlockAllocator.freeByteBlocks.size()
- && 0 == freeCharBlocks.size()
- && 0 == freeIntBlocks.size()
- && !any) {
+ if (0 == perDocAllocator.freeByteBlocks.size() &&
+ 0 == byteBlockAllocator.freeByteBlocks.size() &&
+ 0 == freeIntBlocks.size() && !any) {
// Nothing else to free -- must flush now.
- bufferIsFull = numBytesUsed+deletesRAMUsed > flushTrigger;
+ bufferIsFull = numBytesUsed+deletesRAMUsed > ramBufferSize;
if (infoStream != null) {
- if (numBytesUsed > flushTrigger)
+ if (numBytesUsed+deletesRAMUsed > ramBufferSize)
message(" nothing to free; now set bufferIsFull");
else
message(" nothing to free");
}
- assert numBytesUsed <= numBytesAlloc;
break;
}
- if ((0 == iter % 5) && byteBlockAllocator.freeByteBlocks.size() > 0) {
+ if ((0 == iter % 4) && byteBlockAllocator.freeByteBlocks.size() > 0) {
byteBlockAllocator.freeByteBlocks.remove(byteBlockAllocator.freeByteBlocks.size()-1);
- numBytesAlloc -= BYTE_BLOCK_SIZE;
+ numBytesUsed -= BYTE_BLOCK_SIZE;
}
- if ((1 == iter % 5) && freeCharBlocks.size() > 0) {
- freeCharBlocks.remove(freeCharBlocks.size()-1);
- numBytesAlloc -= CHAR_BLOCK_SIZE * CHAR_NUM_BYTE;
- }
-
- if ((2 == iter % 5) && freeIntBlocks.size() > 0) {
+ if ((1 == iter % 4) && freeIntBlocks.size() > 0) {
freeIntBlocks.remove(freeIntBlocks.size()-1);
- numBytesAlloc -= INT_BLOCK_SIZE * INT_NUM_BYTE;
+ numBytesUsed -= INT_BLOCK_SIZE * INT_NUM_BYTE;
}
- if ((3 == iter % 5) && perDocAllocator.freeByteBlocks.size() > 0) {
+ if ((2 == iter % 4) && perDocAllocator.freeByteBlocks.size() > 0) {
// Remove upwards of 32 blocks (each block is 1K)
for (int i = 0; i < 32; ++i) {
perDocAllocator.freeByteBlocks.remove(perDocAllocator.freeByteBlocks.size() - 1);
- numBytesAlloc -= PER_DOC_BLOCK_SIZE;
+ numBytesUsed -= PER_DOC_BLOCK_SIZE;
if (perDocAllocator.freeByteBlocks.size() == 0) {
break;
}
@@ -1474,7 +1449,7 @@ final class DocumentsWriter {
}
}
- if ((4 == iter % 5) && any)
+ if ((3 == iter % 4) && any)
// Ask consumer to free any recycled state
any = consumer.freeRAM();
@@ -1482,26 +1457,7 @@ final class DocumentsWriter {
}
if (infoStream != null)
- message(" after free: freedMB=" + nf.format((startBytesAlloc-numBytesAlloc-deletesRAMUsed)/1024./1024.) + " usedMB=" + nf.format((numBytesUsed+deletesRAMUsed)/1024./1024.) + " allocMB=" + nf.format(numBytesAlloc/1024./1024.));
-
- } else {
- // If we have not crossed the 100% mark, but have
- // crossed the 95% mark of RAM we are actually
- // using, go ahead and flush. This prevents
- // over-allocating and then freeing, with every
- // flush.
- synchronized(this) {
-
- if (numBytesUsed+deletesRAMUsed > flushTrigger) {
- if (infoStream != null)
- message(" RAM: now flush @ usedMB=" + nf.format(numBytesUsed/1024./1024.) +
- " allocMB=" + nf.format(numBytesAlloc/1024./1024.) +
- " deletesMB=" + nf.format(deletesRAMUsed/1024./1024.) +
- " triggerMB=" + nf.format(flushTrigger/1024./1024.));
-
- bufferIsFull = true;
- }
- }
+ message(" after free: freedMB=" + nf.format((startBytesUsed-numBytesUsed-deletesRAMUsed)/1024./1024.) + " usedMB=" + nf.format((numBytesUsed+deletesRAMUsed)/1024./1024.));
}
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfo.java Tue Apr 6 19:19:27 2010
@@ -17,20 +17,21 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-final class FieldInfo {
- String name;
- boolean isIndexed;
- int number;
+/** @lucene.experimental */
+public final class FieldInfo {
+ public String name;
+ public boolean isIndexed;
+ public int number;
// true if term vector for this field should be stored
boolean storeTermVector;
boolean storeOffsetWithTermVector;
boolean storePositionWithTermVector;
- boolean omitNorms; // omit norms associated with indexed fields
- boolean omitTermFreqAndPositions;
-
- boolean storePayloads; // whether this field stores payloads together with term positions
+ public boolean omitNorms; // omit norms associated with indexed fields
+ public boolean omitTermFreqAndPositions;
+
+ public boolean storePayloads; // whether this field stores payloads together with term positions
FieldInfo(String na, boolean tk, int nu, boolean storeTermVector,
boolean storePositionWithTermVector, boolean storeOffsetWithTermVector,
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FieldInfos.java Tue Apr 6 19:19:27 2010
@@ -32,8 +32,9 @@ import java.util.*;
* of this class are thread-safe for multiple readers, but only one thread can
* be adding documents at a time, with no other reader or writer threads
* accessing this object.
+ * @lucene.experimental
*/
-final class FieldInfos {
+public final class FieldInfos {
// Used internally (ie not written to *.fnm files) for pre-2.9 files
public static final int FORMAT_PRE = -1;
@@ -120,7 +121,7 @@ final class FieldInfos {
}
/** Returns true if any fields do not omitTermFreqAndPositions */
- boolean hasProx() {
+ public boolean hasProx() {
final int numFields = byNumber.size();
for(int i=0;i<numFields;i++) {
final FieldInfo fi = fieldInfo(i);
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java Tue Apr 6 19:19:27 2010
@@ -20,7 +20,9 @@ package org.apache.lucene.index;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.search.FieldCache; // not great (circular); used only to purge FieldCache entry on close
+import org.apache.lucene.util.BytesRef;
import java.io.IOException;
import java.util.Collection;
@@ -116,6 +118,11 @@ public class FilterIndexReader extends I
}
@Override
+ public Bits getDeletedDocs() throws IOException {
+ return in.getDeletedDocs();
+ }
+
+ @Override
public TermFreqVector[] getTermFreqVectors(int docNumber)
throws IOException {
ensureOpen();
@@ -218,6 +225,12 @@ public class FilterIndexReader extends I
}
@Override
+ public int docFreq(String field, BytesRef t) throws IOException {
+ ensureOpen();
+ return in.docFreq(field, t);
+ }
+
+ @Override
public TermDocs termDocs() throws IOException {
ensureOpen();
return in.termDocs();
@@ -297,4 +310,4 @@ public class FilterIndexReader extends I
buffer.append(')');
return buffer.toString();
}
-}
+}
\ No newline at end of file
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxFieldMergeState.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxFieldMergeState.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxFieldMergeState.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxFieldMergeState.java Tue Apr 6 19:19:27 2010
@@ -18,6 +18,8 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
+import java.util.Comparator;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.index.FreqProxTermsWriterPerField.FreqProxPostingsArray;
@@ -31,13 +33,12 @@ final class FreqProxFieldMergeState {
final FreqProxTermsWriterPerField field;
final int numPostings;
- final CharBlockPool charPool;
+ private final ByteBlockPool bytePool;
final int[] termIDs;
final FreqProxPostingsArray postings;
int currentTermID;
- char[] text;
- int textOffset;
+ final BytesRef text = new BytesRef();
private int postingUpto = -1;
@@ -47,29 +48,31 @@ final class FreqProxFieldMergeState {
int docID;
int termFreq;
- public FreqProxFieldMergeState(FreqProxTermsWriterPerField field) {
+ public FreqProxFieldMergeState(FreqProxTermsWriterPerField field, Comparator<BytesRef> termComp) {
this.field = field;
- this.charPool = field.perThread.termsHashPerThread.charPool;
this.numPostings = field.termsHashPerField.numPostings;
- this.termIDs = field.termsHashPerField.sortPostings();
+ this.bytePool = field.perThread.termsHashPerThread.bytePool;
+ this.termIDs = field.termsHashPerField.sortPostings(termComp);
this.postings = (FreqProxPostingsArray) field.termsHashPerField.postingsArray;
}
boolean nextTerm() throws IOException {
postingUpto++;
- if (postingUpto == numPostings)
+ if (postingUpto == numPostings) {
return false;
+ }
currentTermID = termIDs[postingUpto];
docID = 0;
+ // Get BytesRef
final int textStart = postings.textStarts[currentTermID];
- text = charPool.buffers[textStart >> DocumentsWriter.CHAR_BLOCK_SHIFT];
- textOffset = textStart & DocumentsWriter.CHAR_BLOCK_MASK;
+ bytePool.setBytesRef(text, textStart);
field.termsHashPerField.initReader(freq, currentTermID, 0);
- if (!field.fieldInfo.omitTermFreqAndPositions)
+ if (!field.fieldInfo.omitTermFreqAndPositions) {
field.termsHashPerField.initReader(prox, currentTermID, 1);
+ }
// Should always be true
boolean result = nextDoc();
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java Tue Apr 6 19:19:27 2010
@@ -17,14 +17,19 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.util.UnicodeUtil;
-
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.Map;
-import java.util.ArrayList;
+import java.util.Iterator;
import java.util.List;
+import java.util.Map;
+import java.util.Comparator;
+
+import org.apache.lucene.index.codecs.PostingsConsumer;
+import org.apache.lucene.index.codecs.FieldsConsumer;
+import org.apache.lucene.index.codecs.TermsConsumer;
+import org.apache.lucene.util.BytesRef;
final class FreqProxTermsWriter extends TermsHashConsumer {
@@ -33,27 +38,13 @@ final class FreqProxTermsWriter extends
return new FreqProxTermsWriterPerThread(perThread);
}
- private static int compareText(final char[] text1, int pos1, final char[] text2, int pos2) {
- while(true) {
- final char c1 = text1[pos1++];
- final char c2 = text2[pos2++];
- if (c1 != c2) {
- if (0xffff == c2)
- return 1;
- else if (0xffff == c1)
- return -1;
- else
- return c1-c2;
- } else if (0xffff == c1)
- return 0;
- }
- }
-
@Override
void closeDocStore(SegmentWriteState state) {}
+
@Override
void abort() {}
+ private int flushedDocCount;
// TODO: would be nice to factor out more of this, eg the
// FreqProxFieldMergeState, and code to visit all Fields
@@ -66,6 +57,8 @@ final class FreqProxTermsWriter extends
// Gather all FieldData's that have postings, across all
// ThreadStates
List<FreqProxTermsWriterPerField> allFields = new ArrayList<FreqProxTermsWriterPerField>();
+
+ flushedDocCount = state.numDocs;
for (Map.Entry<TermsHashConsumerPerThread,Collection<TermsHashConsumerPerField>> entry : threadsAndFields.entrySet()) {
@@ -79,21 +72,23 @@ final class FreqProxTermsWriter extends
}
}
+ final int numAllFields = allFields.size();
+
// Sort by field name
Collections.sort(allFields);
- final int numAllFields = allFields.size();
- // TODO: allow Lucene user to customize this consumer:
- final FormatPostingsFieldsConsumer consumer = new FormatPostingsFieldsWriter(state, fieldInfos);
+ // TODO: allow Lucene user to customize this codec:
+ final FieldsConsumer consumer = state.codec.fieldsConsumer(state);
+
/*
Current writer chain:
- FormatPostingsFieldsConsumer
- -> IMPL: FormatPostingsFieldsWriter
- -> FormatPostingsTermsConsumer
- -> IMPL: FormatPostingsTermsWriter
- -> FormatPostingsDocConsumer
- -> IMPL: FormatPostingsDocWriter
- -> FormatPostingsPositionsConsumer
+ FieldsConsumer
+ -> IMPL: FormatPostingsTermsDictWriter
+ -> TermsConsumer
+ -> IMPL: FormatPostingsTermsDictWriter.TermsWriter
+ -> DocsConsumer
+ -> IMPL: FormatPostingsDocsWriter
+ -> PositionsConsumer
-> IMPL: FormatPostingsPositionsWriter
*/
@@ -134,25 +129,29 @@ final class FreqProxTermsWriter extends
FreqProxTermsWriterPerThread perThread = (FreqProxTermsWriterPerThread) entry.getKey();
perThread.termsHashPerThread.reset(true);
}
-
- consumer.finish();
+ consumer.close();
}
- private byte[] payloadBuffer;
+ BytesRef payload;
/* Walk through all unique text tokens (Posting
* instances) found in this field and serialize them
* into a single RAM segment. */
void appendPostings(FreqProxTermsWriterPerField[] fields,
- FormatPostingsFieldsConsumer consumer)
+ FieldsConsumer consumer)
throws CorruptIndexException, IOException {
int numFields = fields.length;
+ final BytesRef text = new BytesRef();
+
final FreqProxFieldMergeState[] mergeStates = new FreqProxFieldMergeState[numFields];
+ final TermsConsumer termsConsumer = consumer.addField(fields[0].fieldInfo);
+ final Comparator<BytesRef> termComp = termsConsumer.getComparator();
+
for(int i=0;i<numFields;i++) {
- FreqProxFieldMergeState fms = mergeStates[i] = new FreqProxFieldMergeState(fields[i]);
+ FreqProxFieldMergeState fms = mergeStates[i] = new FreqProxFieldMergeState(fields[i], termComp);
assert fms.field.fieldInfo == fields[0].fieldInfo;
@@ -161,45 +160,63 @@ final class FreqProxTermsWriter extends
assert result;
}
- final FormatPostingsTermsConsumer termsConsumer = consumer.addField(fields[0].fieldInfo);
-
FreqProxFieldMergeState[] termStates = new FreqProxFieldMergeState[numFields];
final boolean currentFieldOmitTermFreqAndPositions = fields[0].fieldInfo.omitTermFreqAndPositions;
+ //System.out.println("flush terms field=" + fields[0].fieldInfo.name);
+ // TODO: really TermsHashPerField should take over most
+ // of this loop, including merge sort of terms from
+ // multiple threads and interacting with the
+ // TermsConsumer, only calling out to us (passing us the
+ // DocsConsumer) to handle delivery of docs/positions
while(numFields > 0) {
// Get the next term to merge
termStates[0] = mergeStates[0];
int numToMerge = 1;
+ // TODO: pqueue
for(int i=1;i<numFields;i++) {
- final char[] text = mergeStates[i].text;
- final int textOffset = mergeStates[i].textOffset;
- final int cmp = compareText(text, textOffset, termStates[0].text, termStates[0].textOffset);
-
+ final int cmp = termComp.compare(mergeStates[i].text, termStates[0].text);
if (cmp < 0) {
termStates[0] = mergeStates[i];
numToMerge = 1;
- } else if (cmp == 0)
+ } else if (cmp == 0) {
termStates[numToMerge++] = mergeStates[i];
+ }
}
- final FormatPostingsDocsConsumer docConsumer = termsConsumer.addTerm(termStates[0].text, termStates[0].textOffset);
+ // Need shallow copy here because termStates[0].text
+ // changes by the time we call finishTerm
+ text.bytes = termStates[0].text.bytes;
+ text.offset = termStates[0].text.offset;
+ text.length = termStates[0].text.length;
+
+ //System.out.println(" term=" + text.toUnicodeString());
+ //System.out.println(" term=" + text.toString());
+
+ final PostingsConsumer postingsConsumer = termsConsumer.startTerm(text);
// Now termStates has numToMerge FieldMergeStates
// which all share the same term. Now we must
// interleave the docID streams.
+ int numDocs = 0;
while(numToMerge > 0) {
FreqProxFieldMergeState minState = termStates[0];
- for(int i=1;i<numToMerge;i++)
- if (termStates[i].docID < minState.docID)
+ for(int i=1;i<numToMerge;i++) {
+ if (termStates[i].docID < minState.docID) {
minState = termStates[i];
+ }
+ }
final int termDocFreq = minState.termFreq;
+ numDocs++;
+
+ assert minState.docID < flushedDocCount: "doc=" + minState.docID + " maxDoc=" + flushedDocCount;
- final FormatPostingsPositionsConsumer posConsumer = docConsumer.addDoc(minState.docID, termDocFreq);
+ postingsConsumer.startDoc(minState.docID, termDocFreq);
final ByteSliceReader prox = minState.prox;
@@ -213,33 +230,48 @@ final class FreqProxTermsWriter extends
for(int j=0;j<termDocFreq;j++) {
final int code = prox.readVInt();
position += code >> 1;
+ //System.out.println(" pos=" + position);
final int payloadLength;
+ final BytesRef thisPayload;
+
if ((code & 1) != 0) {
// This position has a payload
- payloadLength = prox.readVInt();
-
- if (payloadBuffer == null || payloadBuffer.length < payloadLength)
- payloadBuffer = new byte[payloadLength];
+ payloadLength = prox.readVInt();
+
+ if (payload == null) {
+ payload = new BytesRef();
+ payload.bytes = new byte[payloadLength];
+ } else if (payload.bytes.length < payloadLength) {
+ payload.grow(payloadLength);
+ }
+
+ prox.readBytes(payload.bytes, 0, payloadLength);
+ payload.length = payloadLength;
+ thisPayload = payload;
- prox.readBytes(payloadBuffer, 0, payloadLength);
-
- } else
+ } else {
payloadLength = 0;
+ thisPayload = null;
+ }
- posConsumer.addPosition(position, payloadBuffer, 0, payloadLength);
+ postingsConsumer.addPosition(position, thisPayload);
} //End for
- posConsumer.finish();
+ postingsConsumer.finishDoc();
}
if (!minState.nextDoc()) {
// Remove from termStates
int upto = 0;
- for(int i=0;i<numToMerge;i++)
- if (termStates[i] != minState)
+ // TODO: inefficient O(N) where N = number of
+ // threads that had seen this term:
+ for(int i=0;i<numToMerge;i++) {
+ if (termStates[i] != minState) {
termStates[upto++] = termStates[i];
+ }
+ }
numToMerge--;
assert upto == numToMerge;
@@ -258,11 +290,10 @@ final class FreqProxTermsWriter extends
}
}
- docConsumer.finish();
+ assert numDocs > 0;
+ termsConsumer.finishTerm(text, numDocs);
}
termsConsumer.finish();
}
-
- final UnicodeUtil.UTF8Result termsUTF8 = new UnicodeUtil.UTF8Result();
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Tue Apr 6 19:19:27 2010
@@ -185,27 +185,28 @@ final class FreqProxTermsWriterPerField
int lastDocIDs[]; // Last docID where this term occurred
int lastDocCodes[]; // Code for prior doc
int lastPositions[]; // Last position where this term occurred
-
+
@Override
- ParallelPostingsArray resize(int newSize) {
- FreqProxPostingsArray newArray = new FreqProxPostingsArray(newSize);
- copy(this, newArray);
- return newArray;
+ ParallelPostingsArray newInstance(int size) {
+ return new FreqProxPostingsArray(size);
}
-
- void copy(FreqProxPostingsArray fromArray, FreqProxPostingsArray toArray) {
- super.copy(fromArray, toArray);
- System.arraycopy(fromArray.docFreqs, 0, toArray.docFreqs, 0, fromArray.docFreqs.length);
- System.arraycopy(fromArray.lastDocIDs, 0, toArray.lastDocIDs, 0, fromArray.lastDocIDs.length);
- System.arraycopy(fromArray.lastDocCodes, 0, toArray.lastDocCodes, 0, fromArray.lastDocCodes.length);
- System.arraycopy(fromArray.lastPositions, 0, toArray.lastPositions, 0, fromArray.lastPositions.length);
+
+ void copyTo(ParallelPostingsArray toArray, int numToCopy) {
+ assert toArray instanceof FreqProxPostingsArray;
+ FreqProxPostingsArray to = (FreqProxPostingsArray) toArray;
+
+ super.copyTo(toArray, numToCopy);
+
+ System.arraycopy(docFreqs, 0, to.docFreqs, 0, numToCopy);
+ System.arraycopy(lastDocIDs, 0, to.lastDocIDs, 0, numToCopy);
+ System.arraycopy(lastDocCodes, 0, to.lastDocCodes, 0, numToCopy);
+ System.arraycopy(lastPositions, 0, to.lastPositions, 0, numToCopy);
+ }
+
+ @Override
+ int bytesPerPosting() {
+ return ParallelPostingsArray.BYTES_PER_POSTING + 4 * DocumentsWriter.INT_NUM_BYTE;
}
-
- }
-
- @Override
- int bytesPerPosting() {
- return ParallelPostingsArray.BYTES_PER_POSTING + 4 * DocumentsWriter.INT_NUM_BYTE;
}
public void abort() {}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java Tue Apr 6 19:19:27 2010
@@ -17,18 +17,20 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.store.Directory;
-
-import java.io.IOException;
import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.io.IOException;
import java.io.PrintStream;
-import java.util.Map;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Collection;
+import java.util.Map;
+
+import org.apache.lucene.index.codecs.CodecProvider;
+import org.apache.lucene.store.Directory;
/*
* This class keeps track of each SegmentInfos instance that
@@ -114,6 +116,8 @@ final class IndexFileDeleter {
infoStream.println("IFD [" + Thread.currentThread().getName() + "]: " + message);
}
+ private final FilenameFilter indexFilenameFilter;
+
/**
* Initialize the deleter: find all previous commits in
* the Directory, incref the files they reference, call
@@ -122,7 +126,8 @@ final class IndexFileDeleter {
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, DocumentsWriter docWriter)
+ public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, DocumentsWriter docWriter,
+ CodecProvider codecs)
throws CorruptIndexException, IOException {
this.docWriter = docWriter;
@@ -137,8 +142,8 @@ final class IndexFileDeleter {
// First pass: walk the files and initialize our ref
// counts:
long currentGen = segmentInfos.getGeneration();
- IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
-
+ indexFilenameFilter = new IndexFileNameFilter(codecs);
+
String[] files = directory.listAll();
CommitPoint currentCommitPoint = null;
@@ -147,7 +152,7 @@ final class IndexFileDeleter {
String fileName = files[i];
- if (filter.accept(null, fileName) && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
+ if ((indexFilenameFilter.accept(null, fileName)) && !fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
// Add this file to refCounts with initial count 0:
getRefCount(fileName);
@@ -163,7 +168,7 @@ final class IndexFileDeleter {
}
SegmentInfos sis = new SegmentInfos();
try {
- sis.read(directory, fileName);
+ sis.read(directory, fileName, codecs);
} catch (FileNotFoundException e) {
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
@@ -200,7 +205,7 @@ final class IndexFileDeleter {
// try now to explicitly open this commit point:
SegmentInfos sis = new SegmentInfos();
try {
- sis.read(directory, segmentInfos.getCurrentSegmentFileName());
+ sis.read(directory, segmentInfos.getCurrentSegmentFileName(), codecs);
} catch (IOException e) {
throw new CorruptIndexException("failed to locate current segments_N file");
}
@@ -296,7 +301,6 @@ final class IndexFileDeleter {
*/
public void refresh(String segmentName) throws IOException {
String[] files = directory.listAll();
- IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
String segmentPrefix1;
String segmentPrefix2;
if (segmentName != null) {
@@ -309,8 +313,8 @@ final class IndexFileDeleter {
for(int i=0;i<files.length;i++) {
String fileName = files[i];
- if (filter.accept(null, fileName) &&
- (segmentName == null || fileName.startsWith(segmentPrefix1) || fileName.startsWith(segmentPrefix2)) &&
+ if ((segmentName == null || fileName.startsWith(segmentPrefix1) || fileName.startsWith(segmentPrefix2)) &&
+ indexFilenameFilter.accept(null, fileName) &&
!refCounts.containsKey(fileName) &&
!fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
// Unreferenced file, so remove it
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileNames.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileNames.java?rev=931278&r1=931277&r2=931278&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileNames.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/IndexFileNames.java Tue Apr 6 19:19:27 2010
@@ -17,6 +17,8 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import org.apache.lucene.index.codecs.Codec; // for javadocs
+
/**
* This class contains useful constants representing filenames and extensions
* used by lucene, as well as convenience methods for querying whether a file
@@ -25,16 +27,24 @@ package org.apache.lucene.index;
* generation and extension (
* {@link #fileNameFromGeneration(String, String, long) fileNameFromGeneration},
* {@link #segmentFileName(String, String) segmentFileName}).
- *
+ *
+ * <p><b>NOTE</b>: extensions used by codecs are not
+ * listed here. You must interact with the {@link Codec}
+ * directly.
+ *
* @lucene.internal
*/
+
public final class IndexFileNames {
/** Name of the index segment file */
public static final String SEGMENTS = "segments";
+ /** Extension of gen file */
+ public static final String GEN_EXTENSION = "gen";
+
/** Name of the generation reference file name */
- public static final String SEGMENTS_GEN = "segments.gen";
+ public static final String SEGMENTS_GEN = "segments." + GEN_EXTENSION;
/** Name of the index deletable file (only used in
* pre-lockless indices) */
@@ -43,18 +53,6 @@ public final class IndexFileNames {
/** Extension of norms file */
public static final String NORMS_EXTENSION = "nrm";
- /** Extension of freq postings file */
- public static final String FREQ_EXTENSION = "frq";
-
- /** Extension of prox postings file */
- public static final String PROX_EXTENSION = "prx";
-
- /** Extension of terms file */
- public static final String TERMS_EXTENSION = "tis";
-
- /** Extension of terms index file */
- public static final String TERMS_INDEX_EXTENSION = "tii";
-
/** Extension of stored fields index file */
public static final String FIELDS_INDEX_EXTENSION = "fdx";
@@ -88,9 +86,6 @@ public final class IndexFileNames {
/** Extension of separate norms */
public static final String SEPARATE_NORMS_EXTENSION = "s";
- /** Extension of gen file */
- public static final String GEN_EXTENSION = "gen";
-
/**
* This array contains all filename extensions used by
* Lucene's index files, with two exceptions, namely the
@@ -104,10 +99,6 @@ public final class IndexFileNames {
FIELD_INFOS_EXTENSION,
FIELDS_INDEX_EXTENSION,
FIELDS_EXTENSION,
- TERMS_INDEX_EXTENSION,
- TERMS_EXTENSION,
- FREQ_EXTENSION,
- PROX_EXTENSION,
DELETES_EXTENSION,
VECTORS_INDEX_EXTENSION,
VECTORS_DOCUMENTS_EXTENSION,
@@ -117,22 +108,6 @@ public final class IndexFileNames {
COMPOUND_FILE_STORE_EXTENSION,
};
- /** File extensions that are added to a compound file
- * (same as above, minus "del", "gen", "cfs"). */
- public static final String[] INDEX_EXTENSIONS_IN_COMPOUND_FILE = new String[] {
- FIELD_INFOS_EXTENSION,
- FIELDS_INDEX_EXTENSION,
- FIELDS_EXTENSION,
- TERMS_INDEX_EXTENSION,
- TERMS_EXTENSION,
- FREQ_EXTENSION,
- PROX_EXTENSION,
- VECTORS_INDEX_EXTENSION,
- VECTORS_DOCUMENTS_EXTENSION,
- VECTORS_FIELDS_EXTENSION,
- NORMS_EXTENSION
- };
-
public static final String[] STORE_INDEX_EXTENSIONS = new String[] {
VECTORS_INDEX_EXTENSION,
VECTORS_FIELDS_EXTENSION,
@@ -143,22 +118,13 @@ public final class IndexFileNames {
public static final String[] NON_STORE_INDEX_EXTENSIONS = new String[] {
FIELD_INFOS_EXTENSION,
- FREQ_EXTENSION,
- PROX_EXTENSION,
- TERMS_EXTENSION,
- TERMS_INDEX_EXTENSION,
NORMS_EXTENSION
};
- /** File extensions of old-style index files */
- public static final String COMPOUND_EXTENSIONS[] = new String[] {
+ static final String COMPOUND_EXTENSIONS_NOT_CODEC[] = new String[] {
FIELD_INFOS_EXTENSION,
- FREQ_EXTENSION,
- PROX_EXTENSION,
FIELDS_INDEX_EXTENSION,
FIELDS_EXTENSION,
- TERMS_INDEX_EXTENSION,
- TERMS_EXTENSION
};
/** File extensions for term vector support */
@@ -222,6 +188,7 @@ public final class IndexFileNames {
*/
public static final String segmentFileName(String segmentName, String ext) {
if (ext.length() > 0) {
+ assert !ext.startsWith(".");
return new StringBuilder(segmentName.length() + 1 + ext.length()).append(
segmentName).append('.').append(ext).toString();
} else {