You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by mi...@apache.org on 2009/10/13 22:44:59 UTC
svn commit: r824918 [2/11] - in /lucene/java/branches/flex_1458:
contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/
contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/
contrib/benchmark/src/test/org/apache/lucene/benc...
Modified: lucene/java/branches/flex_1458/contrib/surround/src/java/org/apache/lucene/queryParser/surround/query/SrndTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/surround/src/java/org/apache/lucene/queryParser/surround/query/SrndTermQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/surround/src/java/org/apache/lucene/queryParser/surround/query/SrndTermQuery.java (original)
+++ lucene/java/branches/flex_1458/contrib/surround/src/java/org/apache/lucene/queryParser/surround/query/SrndTermQuery.java Tue Oct 13 20:44:51 2009
@@ -20,7 +20,9 @@
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermRef;
public class SrndTermQuery extends SimpleTerm {
@@ -44,18 +46,16 @@
MatchingTermVisitor mtv) throws IOException
{
/* check term presence in index here for symmetry with other SimpleTerm's */
- TermEnum enumerator = reader.terms(getLuceneTerm(fieldName));
- try {
- Term it= enumerator.term(); /* same or following index term */
- if ((it != null)
- && it.text().equals(getTermText())
- && it.field().equals(fieldName)) {
- mtv.visitMatchingTerm(it);
+ Terms terms = reader.fields().terms(fieldName);
+ if (terms != null) {
+ TermsEnum termsEnum = terms.iterator();
+
+ TermsEnum.SeekStatus status = termsEnum.seek(new TermRef(getTermText()));
+ if (status == TermsEnum.SeekStatus.FOUND) {
+ mtv.visitMatchingTerm(getLuceneTerm(fieldName));
} else {
System.out.println("No term in " + fieldName + " field for: " + toString());
}
- } finally {
- enumerator.close();
}
}
}
Modified: lucene/java/branches/flex_1458/contrib/surround/src/java/org/apache/lucene/queryParser/surround/query/SrndTruncQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/contrib/surround/src/java/org/apache/lucene/queryParser/surround/query/SrndTruncQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/contrib/surround/src/java/org/apache/lucene/queryParser/surround/query/SrndTruncQuery.java (original)
+++ lucene/java/branches/flex_1458/contrib/surround/src/java/org/apache/lucene/queryParser/surround/query/SrndTruncQuery.java Tue Oct 13 20:44:51 2009
@@ -17,7 +17,9 @@
*/
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermRef;
import org.apache.lucene.index.IndexReader;
import java.io.IOException;
@@ -40,6 +42,7 @@
private final char mask;
private String prefix;
+ private TermRef prefixRef;
private Pattern pattern;
@@ -67,6 +70,7 @@
i++;
}
prefix = truncated.substring(0, i);
+ prefixRef = new TermRef(prefix);
StringBuilder re = new StringBuilder();
while (i < truncated.length()) {
@@ -83,28 +87,40 @@
{
boolean expanded = false;
int prefixLength = prefix.length();
- TermEnum enumerator = reader.terms(new Term(fieldName, prefix));
- Matcher matcher = pattern.matcher("");
- try {
- do {
- Term term = enumerator.term();
- if (term != null) {
- String text = term.text();
- if ((! text.startsWith(prefix)) || (! term.field().equals(fieldName))) {
- break;
- } else {
- matcher.reset( text.substring(prefixLength));
+ Terms terms = reader.fields().terms(fieldName);
+ if (terms != null) {
+ Matcher matcher = pattern.matcher("");
+ try {
+ TermsEnum termsEnum = terms.iterator();
+
+ TermsEnum.SeekStatus status = termsEnum.seek(prefixRef);
+ TermRef text;
+ if (status == TermsEnum.SeekStatus.FOUND) {
+ text = prefixRef;
+ } else if (status == TermsEnum.SeekStatus.NOT_FOUND) {
+ text = termsEnum.term();
+ } else {
+ text = null;
+ }
+
+ while(text != null) {
+ if (text != null && text.startsWith(prefixRef)) {
+ String textString = text.toString();
+ matcher.reset(textString.substring(prefixLength));
if (matcher.matches()) {
- mtv.visitMatchingTerm(term);
+ mtv.visitMatchingTerm(new Term(fieldName, textString));
expanded = true;
}
+ } else {
+ break;
}
+ text = termsEnum.next();
}
- } while (enumerator.next());
- } finally {
- enumerator.close();
- matcher.reset();
+ } finally {
+ matcher.reset();
+ }
}
+
if (! expanded) {
System.out.println("No terms in " + fieldName + " field for: " + toString());
}
Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.index;
+
+import org.apache.lucene.util.Bits;
+import java.io.IOException;
+
+class AllDocsEnum extends DocsEnum {
+ protected final Bits skipDocs;
+ protected final int maxDoc;
+ protected final IndexReader reader;
+ protected int doc = -1;
+
+ protected AllDocsEnum(IndexReader reader, Bits skipDocs) {
+ this.skipDocs = skipDocs;
+ this.maxDoc = reader.maxDoc();
+ this.reader = reader;
+ }
+
+ public int freq() {
+ return 1;
+ }
+
+ public int next() throws IOException {
+ return advance(doc+1);
+ }
+
+ public int read(int[] docs, int[] freqs) throws IOException {
+ final int length = docs.length;
+ int i = 0;
+ while (i < length && doc < maxDoc) {
+ if (skipDocs == null || !skipDocs.get(doc)) {
+ docs[i] = doc;
+ freqs[i] = 1;
+ ++i;
+ }
+ doc++;
+ }
+ return i;
+ }
+
+ public int advance(int target) throws IOException {
+ doc = target;
+ while (doc < maxDoc) {
+ if (skipDocs == null || !skipDocs.get(doc)) {
+ return doc;
+ }
+ doc++;
+ }
+ doc = NO_MORE_DOCS;
+ return doc;
+ }
+
+ public PositionsEnum positions() {
+ throw new UnsupportedOperationException();
+ }
+}
Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllDocsEnum.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllTermDocs.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllTermDocs.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllTermDocs.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/AllTermDocs.java Tue Oct 13 20:44:51 2009
@@ -20,6 +20,7 @@
import org.apache.lucene.util.BitVector;
import java.io.IOException;
+/** @deprecated Switch to AllDocsEnum */
class AllTermDocs implements TermDocs {
protected BitVector deletedDocs;
protected int maxDoc;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CheckIndex.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/CheckIndex.java Tue Oct 13 20:44:51 2009
@@ -22,6 +22,8 @@
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.document.AbstractField; // for javadocs
import org.apache.lucene.document.Document;
+import org.apache.lucene.index.codecs.Codecs;
+import org.apache.lucene.util.Bits;
import java.text.NumberFormat;
import java.io.PrintStream;
@@ -271,24 +273,6 @@
infoStream.println(msg);
}
- private static class MySegmentTermDocs extends SegmentTermDocs {
-
- int delCount;
-
- MySegmentTermDocs(SegmentReader p) {
- super(p);
- }
-
- public void seek(Term term) throws IOException {
- super.seek(term);
- delCount = 0;
- }
-
- protected void skippingDoc() throws IOException {
- delCount++;
- }
- }
-
/** Returns true if index is clean, else false.
* @deprecated Please instantiate a CheckIndex and then use {@link #checkIndex()} instead */
public static boolean check(Directory dir, boolean doFix) throws IOException {
@@ -319,6 +303,10 @@
return checkIndex(null);
}
+ protected Status checkIndex(List onlySegments) throws IOException {
+ return checkIndex(onlySegments, Codecs.getDefault());
+ }
+
/** Returns a {@link Status} instance detailing
* the state of the index.
*
@@ -331,13 +319,13 @@
* <p><b>WARNING</b>: make sure
* you only call this when the index is not opened by any
* writer. */
- public Status checkIndex(List onlySegments) throws IOException {
+ protected Status checkIndex(List onlySegments, Codecs codecs) throws IOException {
NumberFormat nf = NumberFormat.getInstance();
SegmentInfos sis = new SegmentInfos();
Status result = new Status();
result.dir = dir;
try {
- sis.read(dir);
+ sis.read(dir, codecs);
} catch (Throwable t) {
msg("ERROR: could not read any segments file in directory");
result.missingSegments = true;
@@ -394,6 +382,8 @@
sFormat = "FORMAT_USER_DATA [Lucene 2.9]";
else if (format == SegmentInfos.FORMAT_DIAGNOSTICS)
sFormat = "FORMAT_DIAGNOSTICS [Lucene 2.9]";
+ else if (format == SegmentInfos.FORMAT_FLEX_POSTINGS)
+ sFormat = "FORMAT_FLEX_POSTINGS [Lucene 2.9]";
else if (format < SegmentInfos.CURRENT_FORMAT) {
sFormat = "int=" + format + " [newer version of Lucene than this tool]";
skip = true;
@@ -615,66 +605,87 @@
private Status.TermIndexStatus testTermIndex(SegmentInfo info, SegmentReader reader) {
final Status.TermIndexStatus status = new Status.TermIndexStatus();
+ final int maxDoc = reader.maxDoc();
+ final Bits delDocs = reader.getDeletedDocs();
+
try {
+
if (infoStream != null) {
infoStream.print(" test: terms, freq, prox...");
}
+
+ final FieldsEnum fields = reader.fields().iterator();
+ while(true) {
+ final String field = fields.next();
+ if (field == null) {
+ break;
+ }
+
+ final TermsEnum terms = fields.terms();
+ while(true) {
- final TermEnum termEnum = reader.terms();
- final TermPositions termPositions = reader.termPositions();
-
- // Used only to count up # deleted docs for this term
- final MySegmentTermDocs myTermDocs = new MySegmentTermDocs(reader);
+ final TermRef term = terms.next();
+ if (term == null) {
+ break;
+ }
+ final int docFreq = terms.docFreq();
+ status.totFreq += docFreq;
- final int maxDoc = reader.maxDoc();
+ final DocsEnum docs = terms.docs(delDocs);
+ status.termCount++;
- while (termEnum.next()) {
- status.termCount++;
- final Term term = termEnum.term();
- final int docFreq = termEnum.docFreq();
- termPositions.seek(term);
- int lastDoc = -1;
- int freq0 = 0;
- status.totFreq += docFreq;
- while (termPositions.next()) {
- freq0++;
- final int doc = termPositions.doc();
- final int freq = termPositions.freq();
- if (doc <= lastDoc)
- throw new RuntimeException("term " + term + ": doc " + doc + " <= lastDoc " + lastDoc);
- if (doc >= maxDoc)
- throw new RuntimeException("term " + term + ": doc " + doc + " >= maxDoc " + maxDoc);
-
- lastDoc = doc;
- if (freq <= 0)
- throw new RuntimeException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds");
+ int lastDoc = -1;
+ int freq0 = 0;
+ while(true) {
+ final int doc = docs.next();
+ if (doc == DocsEnum.NO_MORE_DOCS) {
+ break;
+ }
+ final int freq = docs.freq();
+ status.totPos += freq;
+
+ freq0++;
+ if (doc <= lastDoc) {
+ throw new RuntimeException("term " + term + ": doc " + doc + " <= lastDoc " + lastDoc);
+ }
+ if (doc >= maxDoc) {
+ throw new RuntimeException("term " + term + ": doc " + doc + " >= maxDoc " + maxDoc);
+ }
+
+ lastDoc = doc;
+ if (freq <= 0) {
+ throw new RuntimeException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds");
+ }
- int lastPos = -1;
- status.totPos += freq;
- for(int j=0;j<freq;j++) {
- final int pos = termPositions.nextPosition();
- if (pos < -1)
- throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds");
- if (pos < lastPos)
- throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " < lastPos " + lastPos);
- lastPos = pos;
+ int lastPos = -1;
+ final PositionsEnum positions = docs.positions();
+ if (positions != null) {
+ for(int j=0;j<freq;j++) {
+ final int pos = positions.next();
+ if (pos < -1) {
+ throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " is out of bounds");
+ }
+ if (pos < lastPos) {
+ throw new RuntimeException("term " + term + ": doc " + doc + ": pos " + pos + " < lastPos " + lastPos);
+ }
+ lastPos = pos;
+ }
+ }
}
- }
- // Now count how many deleted docs occurred in
- // this term:
- final int delCount;
- if (reader.hasDeletions()) {
- myTermDocs.seek(term);
- while(myTermDocs.next()) { }
- delCount = myTermDocs.delCount;
- } else {
- delCount = 0;
- }
+ // Now count how many deleted docs occurred in
+ // this term:
- if (freq0 + delCount != docFreq) {
- throw new RuntimeException("term " + term + " docFreq=" +
- docFreq + " != num docs seen " + freq0 + " + num docs deleted " + delCount);
+ if (reader.hasDeletions()) {
+ final DocsEnum docsNoDel = terms.docs(null);
+ int count = 0;
+ while(docsNoDel.next() != DocsEnum.NO_MORE_DOCS) {
+ count++;
+ }
+ if (count != docFreq) {
+ throw new RuntimeException("term " + term + " docFreq=" + docFreq + " != tot docs w/o deletions " + count);
+ }
+ }
}
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DirectoryReader.java Tue Oct 13 20:44:51 2009
@@ -17,25 +17,30 @@
* limitations under the License.
*/
-import java.io.IOException;
import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.Collections;
-import java.util.ArrayList;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.search.DefaultSimilarity;
+import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
-import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.index.codecs.Codecs;
+import org.apache.lucene.util.PriorityQueue;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.ReaderUtil;
/**
* An IndexReader which reads indexes with multiple segments.
@@ -43,6 +48,8 @@
class DirectoryReader extends IndexReader implements Cloneable {
protected Directory directory;
protected boolean readOnly;
+
+ protected Codecs codecs;
IndexWriter writer;
@@ -63,28 +70,52 @@
private int numDocs = -1;
private boolean hasDeletions = false;
+ private MultiFields fields;
+
+// static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly,
+// final int termInfosIndexDivisor) throws CorruptIndexException, IOException {
+// return open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, null);
+// }
+
static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly,
- final int termInfosIndexDivisor) throws CorruptIndexException, IOException {
+ final int termInfosIndexDivisor, Codecs codecs) throws CorruptIndexException, IOException {
+ final Codecs codecs2;
+ if (codecs == null) {
+ codecs2 = Codecs.getDefault();
+ } else {
+ codecs2 = codecs;
+ }
return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) {
protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
SegmentInfos infos = new SegmentInfos();
- infos.read(directory, segmentFileName);
+ infos.read(directory, segmentFileName, codecs2);
if (readOnly)
- return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor);
+ return new ReadOnlyDirectoryReader(directory, infos, deletionPolicy, termInfosIndexDivisor, codecs2);
else
- return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor);
+ return new DirectoryReader(directory, infos, deletionPolicy, false, termInfosIndexDivisor, codecs2);
}
}.run(commit);
}
/** Construct reading the named set of readers. */
- DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException {
+// DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException {
+// this(directory, sis, deletionPolicy, readOnly, termInfosIndexDivisor, null);
+// }
+
+ /** Construct reading the named set of readers. */
+ DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, Codecs codecs) throws IOException {
this.directory = directory;
this.readOnly = readOnly;
this.segmentInfos = sis;
this.deletionPolicy = deletionPolicy;
this.termInfosIndexDivisor = termInfosIndexDivisor;
+ if (codecs == null) {
+ this.codecs = Codecs.getDefault();
+ } else {
+ this.codecs = codecs;
+ }
+
if (!readOnly) {
// We assume that this segments_N was previously
// properly sync'd:
@@ -120,11 +151,18 @@
}
// Used by near real-time search
- DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor) throws IOException {
+ DirectoryReader(IndexWriter writer, SegmentInfos infos, int termInfosIndexDivisor, Codecs codecs) throws IOException {
this.directory = writer.getDirectory();
this.readOnly = true;
this.segmentInfos = infos;
this.termInfosIndexDivisor = termInfosIndexDivisor;
+ if (codecs == null) {
+ this.codecs = Codecs.getDefault();
+ } else {
+ this.codecs = codecs;
+ }
+
+
if (!readOnly) {
// We assume that this segments_N was previously
// properly sync'd:
@@ -175,11 +213,17 @@
/** This constructor is only used for {@link #reopen()} */
DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, int[] oldStarts,
- Map oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor) throws IOException {
+ Map oldNormsCache, boolean readOnly, boolean doClone, int termInfosIndexDivisor, Codecs codecs) throws IOException {
this.directory = directory;
this.readOnly = readOnly;
this.segmentInfos = infos;
this.termInfosIndexDivisor = termInfosIndexDivisor;
+ if (codecs == null) {
+ this.codecs = Codecs.getDefault();
+ } else {
+ this.codecs = codecs;
+ }
+
if (!readOnly) {
// We assume that this segments_N was previously
// properly sync'd:
@@ -301,14 +345,75 @@
private void initialize(SegmentReader[] subReaders) {
this.subReaders = subReaders;
starts = new int[subReaders.length + 1]; // build starts array
+ Bits[] subs = new Bits[subReaders.length];
for (int i = 0; i < subReaders.length; i++) {
starts[i] = maxDoc;
maxDoc += subReaders[i].maxDoc(); // compute maxDocs
- if (subReaders[i].hasDeletions())
+ if (subReaders[i].hasDeletions()) {
hasDeletions = true;
+ }
+ subs[i] = subReaders[i].getDeletedDocs();
}
starts[subReaders.length] = maxDoc;
+
+ if (hasDeletions) {
+ deletedDocs = new MultiBits(subs, starts);
+ } else {
+ deletedDocs = null;
+ }
+
+ fields = new MultiFields(subReaders, starts);
+ }
+
+ private MultiBits deletedDocs;
+
+ // Exposes a slice of an existing Bits as a new Bits
+ final static class SubBits implements Bits {
+ private final Bits parent;
+ private final int start;
+ private final int length;
+
+ // start is inclusive; end is exclusive (length = end-start)
+ public SubBits(Bits parent, int start, int end) {
+ this.parent = parent;
+ this.start = start;
+ this.length = end - start;
+ }
+
+ public boolean get(int doc) {
+ if (doc >= length) {
+ throw new RuntimeException("doc " + doc + " is out of bounds 0 .. " + (length-1));
+ }
+ return parent.get(doc-start);
+ }
+ }
+
+ // Concatenates multiple Bits together
+ // nocommit -- if none of the subs have deletions we
+ // should return null from getDeletedDocs:
+ static final class MultiBits implements Bits {
+ private final Bits[] subs;
+ final int[] starts;
+
+ public MultiBits(Bits[] subs, int[] starts) {
+ this.subs = subs;
+ this.starts = starts;
+ }
+
+ public boolean get(int doc) {
+ final int reader = ReaderUtil.subIndex(doc, starts);
+ final Bits bits = subs[reader];
+ if (bits == null) {
+ return false;
+ } else {
+ return bits.get(doc-starts[reader]);
+ }
+ }
+ }
+
+ public Bits getDeletedDocs() {
+ return deletedDocs;
}
public final synchronized Object clone() {
@@ -423,7 +528,7 @@
return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) {
protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException {
SegmentInfos infos = new SegmentInfos();
- infos.read(directory, segmentFileName);
+ infos.read(directory, segmentFileName, codecs);
return doReopen(infos, false, openReadOnly);
}
}.run(commit);
@@ -432,9 +537,9 @@
private synchronized DirectoryReader doReopen(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException {
DirectoryReader reader;
if (openReadOnly) {
- reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor);
+ reader = new ReadOnlyDirectoryReader(directory, infos, subReaders, starts, normsCache, doClone, termInfosIndexDivisor, null);
} else {
- reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor);
+ reader = new DirectoryReader(directory, infos, subReaders, starts, normsCache, false, doClone, termInfosIndexDivisor, null);
}
reader.setDisableFakeNorms(getDisableFakeNorms());
return reader;
@@ -626,10 +731,23 @@
return total;
}
+ public int docFreq(String field, TermRef term) throws IOException {
+ ensureOpen();
+ int total = 0; // sum freqs in segments
+ for (int i = 0; i < subReaders.length; i++) {
+ total += subReaders[i].docFreq(field, term);
+ }
+ return total;
+ }
+
public TermDocs termDocs() throws IOException {
ensureOpen();
return new MultiTermDocs(this, subReaders, starts);
}
+
+ public Fields fields() throws IOException {
+ return fields;
+ }
public TermPositions termPositions() throws IOException {
ensureOpen();
@@ -669,7 +787,7 @@
// we have to check whether index has changed since this reader was opened.
// if so, this reader is no longer valid for deletion
- if (SegmentInfos.readCurrentVersion(directory) > segmentInfos.getVersion()) {
+ if (SegmentInfos.readCurrentVersion(directory, codecs) > segmentInfos.getVersion()) {
stale = true;
this.writeLock.release();
this.writeLock = null;
@@ -699,7 +817,7 @@
// KeepOnlyLastCommitDeleter:
IndexFileDeleter deleter = new IndexFileDeleter(directory,
deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy,
- segmentInfos, null, null);
+ segmentInfos, null, null, codecs);
// Checkpoint the state we are about to change, in
// case we have to roll back:
@@ -794,7 +912,7 @@
*/
public boolean isCurrent() throws CorruptIndexException, IOException {
ensureOpen();
- return SegmentInfos.readCurrentVersion(directory) == segmentInfos.getVersion();
+ return SegmentInfos.readCurrentVersion(directory, codecs) == segmentInfos.getVersion();
}
protected synchronized void doClose() throws IOException {
@@ -861,12 +979,17 @@
/** @see org.apache.lucene.index.IndexReader#listCommits */
public static Collection listCommits(Directory dir) throws IOException {
+ return listCommits(dir, Codecs.getDefault());
+ }
+
+ /** @see org.apache.lucene.index.IndexReader#listCommits */
+ public static Collection listCommits(Directory dir, Codecs codecs) throws IOException {
final String[] files = dir.listAll();
Collection commits = new ArrayList();
SegmentInfos latest = new SegmentInfos();
- latest.read(dir);
+ latest.read(dir, codecs);
final long currentGen = latest.getGeneration();
commits.add(new ReaderCommit(latest, dir));
@@ -883,7 +1006,7 @@
try {
// IOException allowed to throw there, in case
// segments_N is corrupt
- sis.read(dir, fileName);
+ sis.read(dir, fileName, codecs);
} catch (FileNotFoundException fnfe) {
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
@@ -954,20 +1077,496 @@
return userData;
}
}
+
+ private final static class TermsWithBase {
+ Terms terms;
+ int base;
+ int length;
+ Bits deletedDocs;
+
+ public TermsWithBase(IndexReader reader, int base, String field) throws IOException {
+ this.base = base;
+ length = reader.maxDoc();
+ deletedDocs = reader.getDeletedDocs();
+ terms = reader.fields().terms(field);
+ }
+ }
+
+ private final static class FieldsEnumWithBase {
+ FieldsEnum fields;
+ String current;
+ int base;
+ int length;
+ Bits deletedDocs;
+
+ public FieldsEnumWithBase(IndexReader reader, int base) throws IOException {
+ this.base = base;
+ length = reader.maxDoc();
+ deletedDocs = reader.getDeletedDocs();
+ fields = reader.fields().iterator();
+ }
+ }
+
+ private final static class TermsEnumWithBase {
+ TermsEnum terms;
+ int base;
+ int length;
+ TermRef current;
+ Bits deletedDocs;
+
+ public TermsEnumWithBase(FieldsEnumWithBase start, TermsEnum terms, TermRef term) {
+ this.terms = terms;
+ current = term;
+ deletedDocs = start.deletedDocs;
+ base = start.base;
+ length = start.length;
+ }
+
+ public TermsEnumWithBase(TermsWithBase start, TermsEnum terms, TermRef term) {
+ this.terms = terms;
+ current = term;
+ deletedDocs = start.deletedDocs;
+ base = start.base;
+ length = start.length;
+ }
+ }
+
+ private final static class DocsEnumWithBase {
+ DocsEnum docs;
+ int base;
+ }
+
+ private final static class FieldMergeQueue extends PriorityQueue {
+ FieldMergeQueue(int size) {
+ initialize(size);
+ }
+
+ protected final boolean lessThan(Object a, Object b) {
+ FieldsEnumWithBase fieldsA = (FieldsEnumWithBase) a;
+ FieldsEnumWithBase fieldsB = (FieldsEnumWithBase) b;
+ return fieldsA.current.compareTo(fieldsB.current) < 0;
+ }
+ }
+
+ private final static class TermMergeQueue extends PriorityQueue {
+ TermMergeQueue(int size) {
+ initialize(size);
+ }
+
+ protected final boolean lessThan(Object a, Object b) {
+ TermsEnumWithBase termsA = (TermsEnumWithBase) a;
+ TermsEnumWithBase termsB = (TermsEnumWithBase) b;
+ final int cmp = termsA.current.compareTerm(termsB.current);
+ if (cmp != 0) {
+ return cmp < 0;
+ } else {
+ return termsA.base < termsB.base;
+ }
+ }
+ }
+
+ final static class MultiFields extends Fields {
+ private final IndexReader[] readers;
+ private final int[] starts;
+ private final HashMap<String,MultiTerms> terms = new HashMap<String,MultiTerms>();
+
+ public MultiFields(IndexReader[] readers, int[] starts) {
+ this.readers = readers;
+ this.starts = starts;
+ }
+
+ public FieldsEnum iterator() throws IOException {
+ FieldsEnumWithBase[] subs = new FieldsEnumWithBase[readers.length];
+ for(int i=0;i<subs.length;i++) {
+ subs[i] = new FieldsEnumWithBase(readers[i], starts[i]);
+ }
+ return new MultiFieldsEnum(subs);
+ }
+
+ public Terms terms(String field) throws IOException {
+ MultiTerms result = terms.get(field);
+ if (result == null) {
+
+ List<TermsWithBase> subs = new ArrayList<TermsWithBase>();
+
+ // Gather all sub-readers that have this field
+ for(int i=0;i<readers.length;i++) {
+ Terms subTerms = readers[i].fields().terms(field);
+ if (subTerms != null) {
+ subs.add(new TermsWithBase(readers[i], starts[i], field));
+ }
+ }
+ result = new MultiTerms(subs.toArray(new TermsWithBase[]{}));
+ terms.put(field, result);
+ }
+ return result;
+ }
+
+ public void close() {
+ }
+ }
+
+ private final static class MultiTerms extends Terms {
+ private final TermsWithBase[] subs;
+
+ public MultiTerms(TermsWithBase[] subs) {
+ this.subs = subs;
+ }
+
+ public TermsEnum iterator() throws IOException {
+ return new MultiTermsEnum(subs.length).reset(subs);
+ }
+ }
+
+ private final static class MultiFieldsEnum extends FieldsEnum {
+ private final FieldMergeQueue queue;
+
+ private String currentField;
+
+ private final FieldsEnumWithBase[] top;
+ private int numTop;
+
+ private final MultiTermsEnum terms;
+
+ MultiFieldsEnum(FieldsEnumWithBase[] subs) throws IOException {
+ terms = new MultiTermsEnum(subs.length);
+ queue = new FieldMergeQueue(subs.length);
+ top = new FieldsEnumWithBase[subs.length];
+ for(int i=0;i<subs.length;i++) {
+ subs[i].current = subs[i].fields.next();
+ if (subs[i].current != null) {
+ queue.add(subs[i]);
+ }
+ }
+ }
+
+ public String field() {
+ assert currentField != null;
+ return currentField;
+ }
+
+ public String next() throws IOException {
+
+ // restore queue
+ for(int i=0;i<numTop;i++) {
+ top[i].current = top[i].fields.next();
+ if (top[i].current != null) {
+ queue.add(top[i]);
+ } else {
+ // no more fields in this reader
+ }
+ }
+
+ numTop = 0;
+
+ // gather equal top fields
+ if (queue.size() > 0) {
+ while(true) {
+ top[numTop++] = (FieldsEnumWithBase) queue.pop();
+ if (queue.size() == 0 || ((FieldsEnumWithBase) queue.top()).current != top[0].current) {
+ break;
+ }
+ }
+ currentField = top[0].current;
+ } else {
+ currentField = null;
+ }
+
+ return currentField;
+ }
+
+ public TermsEnum terms() throws IOException {
+ return terms.reset(top, numTop);
+ }
+ }
+
+ private static final class MultiTermsEnum extends TermsEnum {
+
+ private final TermMergeQueue queue;
+ private final TermsEnumWithBase[] subs;
+ private final TermsEnumWithBase[] top;
+ int numTop;
+ int numSubs;
+ private TermRef current;
+ private final MultiDocsEnum docs;
+
+ MultiTermsEnum(int size) {
+ queue = new TermMergeQueue(size);
+ top = new TermsEnumWithBase[size];
+ subs = new TermsEnumWithBase[size];
+ docs = new MultiDocsEnum(size);
+ }
+
+ public TermRef term() {
+ return current;
+ }
+
+ MultiTermsEnum reset(TermsWithBase[] terms) throws IOException {
+ assert terms.length <= top.length;
+ numSubs = 0;
+ numTop = 0;
+ for(int i=0;i<terms.length;i++) {
+ final TermsEnum termsEnum = terms[i].terms.iterator();
+ if (termsEnum != null) {
+ final TermRef term = termsEnum.next();
+ if (term != null) {
+ subs[numSubs] = new TermsEnumWithBase(terms[i], termsEnum, term);
+ queue.add(subs[numSubs]);
+ numSubs++;
+ } else {
+ // field has no terms
+ }
+ }
+ }
+
+ return this;
+ }
+
+ MultiTermsEnum reset(FieldsEnumWithBase[] fields, int numFields) throws IOException {
+ assert numFields <= top.length;
+ numSubs = 0;
+ numTop = 0;
+ for(int i=0;i<numFields;i++) {
+ final TermsEnum terms = fields[i].fields.terms();
+ if (terms != null) {
+ final TermRef term = terms.next();
+ if (term != null) {
+ subs[numSubs] = new TermsEnumWithBase(fields[i], terms, term);
+ queue.add(subs[numSubs]);
+ numSubs++;
+ } else {
+ // field has no terms
+ }
+ }
+ }
+
+ return this;
+ }
+
+ public SeekStatus seek(TermRef term) throws IOException {
+ queue.clear();
+ numTop = 0;
+ for(int i=0;i<numSubs;i++) {
+ final SeekStatus status = subs[i].terms.seek(term);
+ if (status == SeekStatus.FOUND) {
+ top[numTop++] = subs[i];
+ current = subs[i].current = term;
+ } else if (status == SeekStatus.NOT_FOUND) {
+ queue.add(subs[i]);
+ current = subs[i].current = subs[i].terms.term();
+ } else {
+ // enum exhausted
+ }
+ }
+
+ if (numTop > 0) {
+ return SeekStatus.FOUND;
+ } else if (queue.size() > 0) {
+ pullTop();
+ return SeekStatus.NOT_FOUND;
+ } else {
+ return SeekStatus.END;
+ }
+ }
+
+ public SeekStatus seek(long ord) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ public long ord() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ private final void pullTop() {
+ assert numTop == 0;
+ while(true) {
+ top[numTop++] = (TermsEnumWithBase) queue.pop();
+ if (queue.size() == 0 || !((TermsEnumWithBase) queue.top()).current.termEquals(top[0].current)) {
+ break;
+ }
+ }
+ current = top[0].current;
+ }
+
+ private final void pushTop() throws IOException {
+ for(int i=0;i<numTop;i++) {
+ top[i].current = top[i].terms.next();
+ if (top[i].current != null) {
+ queue.add(top[i]);
+ } else {
+ // no more fields in this reader
+ }
+ }
+
+ numTop = 0;
+ }
+
+ public TermRef next() throws IOException {
+ // restore queue
+ pushTop();
+
+ // gather equal top fields
+ if (queue.size() > 0) {
+ pullTop();
+ } else {
+ current = null;
+ }
+
+ return current;
+ }
+
+ public int docFreq() {
+ int sum = 0;
+ for(int i=0;i<numTop;i++) {
+ sum += top[i].terms.docFreq();
+ }
+ return sum;
+ }
+
+ public DocsEnum docs(Bits skipDocs) throws IOException {
+ return docs.reset(top, numTop, skipDocs);
+ }
+ }
+
+ private static final class MultiDocsEnum extends DocsEnum {
+ final DocsEnumWithBase[] subs;
+ int numSubs;
+ int upto;
+ DocsEnum currentDocs;
+ int currentBase;
+ Bits skipDocs;
+
+ MultiDocsEnum(int count) {
+ subs = new DocsEnumWithBase[count];
+ }
+
+ MultiDocsEnum reset(TermsEnumWithBase[] subs, final int numSubs, final Bits skipDocs) throws IOException {
+ this.numSubs = 0;
+ this.skipDocs = skipDocs;
+ for(int i=0;i<numSubs;i++) {
+ Bits bits = null;
+ boolean handled = false;
+
+ // Optimize for common case: requested skip docs is simply our
+ // deleted docs
+ if (skipDocs instanceof MultiBits) {
+ MultiBits multiBits = (MultiBits) skipDocs;
+ int reader = ReaderUtil.subIndex(subs[i].base, multiBits.starts);
+ // System.out.println("bits=" + multiBits + " starts=" + multiBits.starts + " subs=" + subs + " subs[i]=" + subs[i] + " subs[1+i]=" + subs[1+i] + " i=" + i + " numSubs=" + numSubs);
+ if (multiBits.starts[reader] == subs[i].base &&
+ (i == numSubs-1 ||
+ reader == multiBits.starts.length-1 ||
+ multiBits.starts[1+reader] == subs[1+i].base)) {
+ bits = multiBits.subs[reader];
+ handled = true;
+ }
+ }
+
+ if (!handled && skipDocs != null) {
+ // custom case: requested skip docs is foreign
+ bits = new SubBits(skipDocs, subs[i].base, subs[i].length);
+ }
+
+ final DocsEnum docs = subs[i].terms.docs(bits);
+ if (docs != null) {
+ this.subs[this.numSubs] = new DocsEnumWithBase();
+ this.subs[this.numSubs].docs = docs;
+ this.subs[this.numSubs].base = subs[i].base;
+ this.numSubs++;
+ }
+ }
+ upto = -1;
+ currentDocs = null;
+ return this;
+ }
+
+ public int freq() {
+ return currentDocs.freq();
+ }
+
+ public int read(final int docs[], final int freqs[]) throws IOException {
+ while (true) {
+ while (currentDocs == null) {
+ if (upto == numSubs-1) {
+ return 0;
+ } else {
+ upto++;
+ currentDocs = subs[upto].docs;
+ currentBase = subs[upto].base;
+ }
+ }
+ final int end = currentDocs.read(docs, freqs);
+ if (end == 0) { // none left in segment
+ currentDocs = null;
+ } else { // got some
+ for (int i = 0; i < end; i++) {
+ docs[i] += currentBase;
+ }
+ return end;
+ }
+ }
+ }
+
+ public int advance(int target) throws IOException {
+ while(true) {
+ if (currentDocs != null) {
+ final int doc = currentDocs.advance(target-currentBase);
+ if (doc == NO_MORE_DOCS) {
+ currentDocs = null;
+ } else {
+ return doc + currentBase;
+ }
+ } else if (upto == numSubs-1) {
+ return NO_MORE_DOCS;
+ } else {
+ upto++;
+ currentDocs = subs[upto].docs;
+ currentBase = subs[upto].base;
+ }
+ }
+ }
+
+ public int next() throws IOException {
+ while(true) {
+ if (currentDocs == null) {
+ if (upto == numSubs-1) {
+ return NO_MORE_DOCS;
+ } else {
+ upto++;
+ currentDocs = subs[upto].docs;
+ currentBase = subs[upto].base;
+ }
+ }
+
+ final int doc = currentDocs.next();
+ if (doc != NO_MORE_DOCS) {
+ return currentBase + doc;
+ } else {
+ currentDocs = null;
+ }
+ }
+ }
+
+ public PositionsEnum positions() throws IOException {
+ return currentDocs.positions();
+ }
+ }
+
+ // Legacy API
static class MultiTermEnum extends TermEnum {
IndexReader topReader; // used for matching TermEnum to TermDocs
- private SegmentMergeQueue queue;
+ private LegacySegmentMergeQueue queue;
private Term term;
private int docFreq;
- final SegmentMergeInfo[] matchingSegments; // null terminated array of matching segments
+ final LegacySegmentMergeInfo[] matchingSegments; // null terminated array of matching segments
public MultiTermEnum(IndexReader topReader, IndexReader[] readers, int[] starts, Term t)
throws IOException {
this.topReader = topReader;
- queue = new SegmentMergeQueue(readers.length);
- matchingSegments = new SegmentMergeInfo[readers.length+1];
+ queue = new LegacySegmentMergeQueue(readers.length);
+ matchingSegments = new LegacySegmentMergeInfo[readers.length+1];
for (int i = 0; i < readers.length; i++) {
IndexReader reader = readers[i];
TermEnum termEnum;
@@ -977,7 +1576,7 @@
} else
termEnum = reader.terms();
- SegmentMergeInfo smi = new SegmentMergeInfo(starts[i], termEnum, reader);
+ LegacySegmentMergeInfo smi = new LegacySegmentMergeInfo(starts[i], termEnum, reader);
smi.ord = i;
if (t == null ? smi.next() : termEnum.term() != null)
queue.add(smi); // initialize queue
@@ -992,7 +1591,7 @@
public boolean next() throws IOException {
for (int i=0; i<matchingSegments.length; i++) {
- SegmentMergeInfo smi = matchingSegments[i];
+ LegacySegmentMergeInfo smi = matchingSegments[i];
if (smi==null) break;
if (smi.next())
queue.add(smi);
@@ -1003,7 +1602,7 @@
int numMatchingSegments = 0;
matchingSegments[0] = null;
- SegmentMergeInfo top = (SegmentMergeInfo)queue.top();
+ LegacySegmentMergeInfo top = (LegacySegmentMergeInfo)queue.top();
if (top == null) {
term = null;
@@ -1017,7 +1616,7 @@
matchingSegments[numMatchingSegments++] = top;
queue.pop();
docFreq += top.termEnum.docFreq(); // increment freq
- top = (SegmentMergeInfo)queue.top();
+ top = (LegacySegmentMergeInfo)queue.top();
}
matchingSegments[numMatchingSegments] = null;
@@ -1037,6 +1636,7 @@
}
}
+ // Legacy API
static class MultiTermDocs implements TermDocs {
IndexReader topReader; // used for matching TermEnum to TermDocs
protected IndexReader[] readers;
@@ -1051,7 +1651,7 @@
private MultiTermEnum tenum; // the term enum used for seeking... can be null
int matchingSegmentPos; // position into the matching segments from tenum
- SegmentMergeInfo smi; // current segment mere info... can be null
+ LegacySegmentMergeInfo smi; // current segment mere info... can be null
public MultiTermDocs(IndexReader topReader, IndexReader[] r, int[] s) {
this.topReader = topReader;
@@ -1147,7 +1747,7 @@
return true;
} else if (pointer < readers.length) {
if (tenum != null) {
- SegmentMergeInfo smi = tenum.matchingSegments[matchingSegmentPos++];
+ LegacySegmentMergeInfo smi = tenum.matchingSegments[matchingSegmentPos++];
if (smi==null) {
pointer = readers.length;
return false;
@@ -1188,6 +1788,7 @@
}
}
+ // Legacy API
static class MultiTermPositions extends MultiTermDocs implements TermPositions {
public MultiTermPositions(IndexReader topReader, IndexReader[] r, int[] s) {
super(topReader,r,s);
Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,67 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.util.AttributeSource;
+
+/** On obtaining a DocsEnum, you must first call next() */
+
+public abstract class DocsEnum extends AttributeSource {
+ // nocommit
+ public String desc;
+
+ public final static int NO_MORE_DOCS = Integer.MAX_VALUE;
+
+ /** Moves forward to the doc id >= target */
+ public abstract int advance(int target) throws IOException;
+
+ /** Returns the next docID, {@link #NO_MORE_DOCS} at the end. */
+ public abstract int next() throws IOException;
+
+ public abstract int freq();
+
+ // nocommit -- fix this API so that intblock codecs are
+ // able to return their own int arrays, to save a copy
+ /** Bulk read: returns number of docs read. Subclass may
+ * do this more efficiently. */
+ public int read(int[] docs, int[] freqs) throws IOException {
+ int count = 0;
+ while(count < docs.length) {
+ final int doc = next();
+ if (doc != NO_MORE_DOCS) {
+ docs[count] = doc;
+ freqs[count] = freq();
+ count++;
+ } else {
+ break;
+ }
+ }
+ return count;
+ }
+
+ // nocommit -- maybe move this up to TermsEnum? that
+ // would disallow changing positions format/reader of each
+ // doc, though
+ // nocommit - doc whether this returns null if there are
+ // no positions, or a faker
+ /** Don't call next() or skipTo() or read() until you're
+ * done consuming the positions */
+ public abstract PositionsEnum positions() throws IOException;
+}
Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocsEnum.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocumentsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocumentsWriter.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocumentsWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/DocumentsWriter.java Tue Oct 13 20:44:51 2009
@@ -40,6 +40,8 @@
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Constants;
+import org.apache.lucene.index.codecs.Codec;
+
/**
* This class accepts multiple added documents and directly
* writes a single segment file. It does this more
@@ -545,9 +547,16 @@
synchronized private void initFlushState(boolean onlyDocStore) {
initSegmentName(onlyDocStore);
- flushState = new SegmentWriteState(this, directory, segment, docStoreSegment, numDocsInRAM, numDocsInStore, writer.getTermIndexInterval());
+ flushState = new SegmentWriteState(this, directory, segment, docFieldProcessor.fieldInfos,
+ docStoreSegment, numDocsInRAM, numDocsInStore, writer.getTermIndexInterval(),
+ writer.codecs);
}
+ /** Returns the codec used to flush the last segment */
+ Codec getCodec() {
+ return flushState.codec;
+ }
+
/** Flush all pending docs to a new segment */
synchronized int flush(boolean closeDocStore) throws IOException {
@@ -583,7 +592,8 @@
consumer.flush(threads, flushState);
if (infoStream != null) {
- final long newSegmentSize = segmentSize(flushState.segmentName);
+ SegmentInfo si = new SegmentInfo(flushState.segmentName, flushState.numDocs, directory, flushState.codec);
+ final long newSegmentSize = si.sizeInBytes();
String message = " oldRAMSize=" + numBytesUsed +
" newFlushedSize=" + newSegmentSize +
" docs/MB=" + nf.format(numDocsInRAM/(newSegmentSize/1024./1024.)) +
@@ -613,8 +623,12 @@
CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION);
Iterator it = flushState.flushedFiles.iterator();
- while(it.hasNext())
- cfsWriter.addFile((String) it.next());
+ while(it.hasNext()) {
+ final String fileName = (String) it.next();
+ if (Codec.DEBUG)
+ System.out.println("make cfs " + fileName);
+ cfsWriter.addFile(fileName);
+ }
// Perform the merge
cfsWriter.close();
@@ -970,24 +984,27 @@
// Delete by term
Iterator iter = deletesFlushed.terms.entrySet().iterator();
- TermDocs docs = reader.termDocs();
+
try {
while (iter.hasNext()) {
Entry entry = (Entry) iter.next();
Term term = (Term) entry.getKey();
- docs.seek(term);
- int limit = ((BufferedDeletes.Num) entry.getValue()).getNum();
- while (docs.next()) {
- int docID = docs.doc();
- if (docIDStart+docID >= limit)
- break;
- reader.deleteDocument(docID);
- any = true;
+ DocsEnum docs = reader.termDocsEnum(reader.getDeletedDocs(), term.field, new TermRef(term.text));
+ if (docs != null) {
+ int limit = ((BufferedDeletes.Num) entry.getValue()).getNum();
+ while (true) {
+ final int docID = docs.next();
+ if (docID == DocsEnum.NO_MORE_DOCS || docIDStart+docID >= limit) {
+ break;
+ }
+ reader.deleteDocument(docID);
+ any = true;
+ }
}
}
} finally {
- docs.close();
+ //docs.close();
}
// Delete by docID
@@ -1140,24 +1157,6 @@
NumberFormat nf = NumberFormat.getInstance();
- // TODO FI: this is not flexible -- we can't hardwire
- // extensions in here:
- private long segmentSize(String segmentName) throws IOException {
- // Used only when infoStream != null
- assert infoStream != null;
-
- long size = directory.fileLength(segmentName + ".tii") +
- directory.fileLength(segmentName + ".tis") +
- directory.fileLength(segmentName + ".frq") +
- directory.fileLength(segmentName + ".prx");
-
- final String normFileName = segmentName + ".nrm";
- if (directory.fileExists(normFileName))
- size += directory.fileLength(normFileName);
-
- return size;
- }
-
// Coarse estimates used to measure RAM usage of buffered deletes
final static int OBJECT_HEADER_BYTES = 8;
final static int POINTER_NUM_BYTE = Constants.JRE_IS_64BIT ? 8 : 4;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldInfo.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldInfo.java Tue Oct 13 20:44:51 2009
@@ -17,20 +17,27 @@
* limitations under the License.
*/
-final class FieldInfo {
- String name;
- boolean isIndexed;
- int number;
+// nocommit -- made this public:
+public final class FieldInfo {
+ // nocommit -- made this public
+ public String name;
+ // nocommit -- made this public
+ public boolean isIndexed;
+ // nocommit -- made this public
+ public int number;
// true if term vector for this field should be stored
boolean storeTermVector;
boolean storeOffsetWithTermVector;
boolean storePositionWithTermVector;
- boolean omitNorms; // omit norms associated with indexed fields
- boolean omitTermFreqAndPositions;
-
- boolean storePayloads; // whether this field stores payloads together with term positions
+ // nocommit -- made this public
+ public boolean omitNorms; // omit norms associated with indexed fields
+ // nocommit -- made this public
+ public boolean omitTermFreqAndPositions;
+
+ // nocommit -- made public
+ public boolean storePayloads; // whether this field stores payloads together with term positions
FieldInfo(String na, boolean tk, int nu, boolean storeTermVector,
boolean storePositionWithTermVector, boolean storeOffsetWithTermVector,
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldInfos.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldInfos.java Tue Oct 13 20:44:51 2009
@@ -33,7 +33,8 @@
* be adding documents at a time, with no other reader or writer threads
* accessing this object.
*/
-final class FieldInfos {
+// nocommit -- made this public:
+public final class FieldInfos {
// Used internally (ie not written to *.fnm files) for pre-2.9 files
public static final int FORMAT_PRE = -1;
@@ -121,14 +122,19 @@
}
/** Returns true if any fields do not omitTermFreqAndPositions */
- boolean hasProx() {
+ // nocommit -- made public
+ public boolean hasProx() {
final int numFields = byNumber.size();
for(int i=0;i<numFields;i++) {
final FieldInfo fi = fieldInfo(i);
if (fi.isIndexed && !fi.omitTermFreqAndPositions) {
+ // mxx
+ // System.out.println(Thread.currentThread().getName() + ": fieldInfos: hasProx=true");
return true;
}
}
+ // mxx
+ //System.out.println(Thread.currentThread().getName() + ": fieldInfos: hasProx=false");
return false;
}
Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,46 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+// TODO: split out an "iterator" api from the terms(String
+// field) API?
+
+// nocommit -- intended to be forward only? eg no "reset"?
+
+/** Access to fields and terms
+ *
+ * NOTE: this API is experimental and will likely change */
+
+// TODO: someday expose public version of FieldInfos here
+public abstract class Fields {
+
+ // nocommit -- clarify if this is forwards only. should
+ // this be "skipTo"?
+ // nocommit -- clarify: when this returns false, what is
+ // its internal state? eg if i call field() after getting
+ // false back?
+ /** Returns an iterator that will step through all fields
+ * names */
+ public abstract FieldsEnum iterator() throws IOException;
+
+ /** Get the {@link Terms} for this field */
+ public abstract Terms terms(String field) throws IOException;
+}
+
Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/Fields.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java?rev=824918&view=auto
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java (added)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java Tue Oct 13 20:44:51 2009
@@ -0,0 +1,41 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import org.apache.lucene.util.AttributeSource;
+
+/** Enumerates indexed fields.
+ *
+ * NOTE: this API is experimental and will likely change */
+
+public abstract class FieldsEnum extends AttributeSource {
+
+ // nocommit -- do we need seek?
+
+ /** Increments the enumeration to the next field.
+ * Returns null when there are no more fields.*/
+ public abstract String next() throws IOException;
+
+ /** Get TermsEnum for the current field. You should not
+ * call {@link #next()} until you're done using this
+ * TermsEnum. */
+ public abstract TermsEnum terms() throws IOException;
+}
+
Propchange: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FieldsEnum.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FilterIndexReader.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FilterIndexReader.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FilterIndexReader.java Tue Oct 13 20:44:51 2009
@@ -20,6 +20,7 @@
import org.apache.lucene.document.Document;
import org.apache.lucene.document.FieldSelector;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Bits;
import java.io.IOException;
import java.util.Collection;
@@ -109,6 +110,10 @@
return in.directory();
}
+ public Bits getDeletedDocs() throws IOException {
+ return in.getDeletedDocs();
+ }
+
public TermFreqVector[] getTermFreqVectors(int docNumber)
throws IOException {
ensureOpen();
@@ -194,6 +199,11 @@
return in.docFreq(t);
}
+ public int docFreq(String field, TermRef t) throws IOException {
+ ensureOpen();
+ return in.docFreq(field, t);
+ }
+
public TermDocs termDocs() throws IOException {
ensureOpen();
return in.termDocs();
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/FreqProxTermsWriter.java Tue Oct 13 20:44:51 2009
@@ -17,17 +17,19 @@
* limitations under the License.
*/
-import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.util.UnicodeUtil;
-
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
-import java.util.Map;
-import java.util.ArrayList;
-import java.util.List;
import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.index.codecs.DocsConsumer;
+import org.apache.lucene.index.codecs.FieldsConsumer;
+import org.apache.lucene.index.codecs.PositionsConsumer;
+import org.apache.lucene.index.codecs.TermsConsumer;
+import org.apache.lucene.util.UnicodeUtil;
final class FreqProxTermsWriter extends TermsHashConsumer {
@@ -60,6 +62,7 @@
void closeDocStore(SegmentWriteState state) {}
void abort() {}
+ private int flushedDocCount;
// TODO: would be nice to factor out more of this, eg the
// FreqProxFieldMergeState, and code to visit all Fields
@@ -71,6 +74,8 @@
// Gather all FieldData's that have postings, across all
// ThreadStates
List allFields = new ArrayList();
+
+ flushedDocCount = state.numDocs;
Iterator it = threadsAndFields.entrySet().iterator();
while(it.hasNext()) {
@@ -88,21 +93,23 @@
}
}
+ final int numAllFields = allFields.size();
+
// Sort by field name
Collections.sort(allFields);
- final int numAllFields = allFields.size();
- // TODO: allow Lucene user to customize this consumer:
- final FormatPostingsFieldsConsumer consumer = new FormatPostingsFieldsWriter(state, fieldInfos);
+ // TODO: allow Lucene user to customize this codec:
+ final FieldsConsumer consumer = state.codec.fieldsConsumer(state);
+
/*
Current writer chain:
- FormatPostingsFieldsConsumer
- -> IMPL: FormatPostingsFieldsWriter
- -> FormatPostingsTermsConsumer
- -> IMPL: FormatPostingsTermsWriter
- -> FormatPostingsDocConsumer
- -> IMPL: FormatPostingsDocWriter
- -> FormatPostingsPositionsConsumer
+ FieldsConsumer
+ -> IMPL: FormatPostingsTermsDictWriter
+ -> TermsConsumer
+ -> IMPL: FormatPostingsTermsDictWriter.TermsWriter
+ -> DocsConsumer
+ -> IMPL: FormatPostingsDocsWriter
+ -> PositionsConsumer
-> IMPL: FormatPostingsPositionsWriter
*/
@@ -145,8 +152,7 @@
FreqProxTermsWriterPerThread perThread = (FreqProxTermsWriterPerThread) entry.getKey();
perThread.termsHashPerThread.reset(true);
}
-
- consumer.finish();
+ consumer.close();
}
private byte[] payloadBuffer;
@@ -155,7 +161,7 @@
* instances) found in this field and serialize them
* into a single RAM segment. */
void appendPostings(FreqProxTermsWriterPerField[] fields,
- FormatPostingsFieldsConsumer consumer)
+ FieldsConsumer consumer)
throws CorruptIndexException, IOException {
int numFields = fields.length;
@@ -172,7 +178,7 @@
assert result;
}
- final FormatPostingsTermsConsumer termsConsumer = consumer.addField(fields[0].fieldInfo);
+ final TermsConsumer termsConsumer = consumer.addField(fields[0].fieldInfo);
FreqProxFieldMergeState[] termStates = new FreqProxFieldMergeState[numFields];
@@ -196,11 +202,18 @@
termStates[numToMerge++] = mergeStates[i];
}
- final FormatPostingsDocsConsumer docConsumer = termsConsumer.addTerm(termStates[0].text, termStates[0].textOffset);
+ final char[] termText = termStates[0].text;
+ final int termTextOffset = termStates[0].textOffset;
+
+ // nocommit
+ //System.out.println("FLUSH term=" + new String(termText, termTextOffset, 10));
+
+ final DocsConsumer docConsumer = termsConsumer.startTerm(termText, termTextOffset);
// Now termStates has numToMerge FieldMergeStates
// which all share the same term. Now we must
// interleave the docID streams.
+ int numDocs = 0;
while(numToMerge > 0) {
FreqProxFieldMergeState minState = termStates[0];
@@ -209,8 +222,12 @@
minState = termStates[i];
final int termDocFreq = minState.termFreq;
+ numDocs++;
- final FormatPostingsPositionsConsumer posConsumer = docConsumer.addDoc(minState.docID, termDocFreq);
+ assert minState.docID < flushedDocCount: "doc=" + minState.docID + " maxDoc=" + flushedDocCount;
+
+ //System.out.println(" docID=" + minState.docID);
+ final PositionsConsumer posConsumer = docConsumer.addDoc(minState.docID, termDocFreq);
final ByteSliceReader prox = minState.prox;
@@ -224,6 +241,7 @@
for(int j=0;j<termDocFreq;j++) {
final int code = prox.readVInt();
position += code >> 1;
+ //System.out.println(" pos=" + position);
final int payloadLength;
if ((code & 1) != 0) {
@@ -241,7 +259,7 @@
posConsumer.addPosition(position, payloadBuffer, 0, payloadLength);
} //End for
- posConsumer.finish();
+ posConsumer.finishDoc();
}
if (!minState.nextDoc()) {
@@ -269,14 +287,12 @@
}
}
- docConsumer.finish();
+ termsConsumer.finishTerm(termText, termTextOffset, numDocs);
}
termsConsumer.finish();
}
- private final TermInfo termInfo = new TermInfo(); // minimize consing
-
final UnicodeUtil.UTF8Result termsUTF8 = new UnicodeUtil.UTF8Result();
void files(Collection files) {}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileDeleter.java Tue Oct 13 20:44:51 2009
@@ -17,18 +17,21 @@
* limitations under the License.
*/
-import org.apache.lucene.store.Directory;
-
-import java.io.IOException;
+import java.io.File;
import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.io.IOException;
import java.io.PrintStream;
-import java.util.Map;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Collection;
+import java.util.Map;
+
+import org.apache.lucene.index.codecs.Codecs;
+import org.apache.lucene.store.Directory;
/*
* This class keeps track of each SegmentInfos instance that
@@ -114,6 +117,8 @@
infoStream.println("IFD [" + Thread.currentThread().getName() + "]: " + message);
}
+ private final FilenameFilter indexFilenameFilter;
+
/**
* Initialize the deleter: find all previous commits in
* the Directory, incref the files they reference, call
@@ -122,7 +127,8 @@
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, DocumentsWriter docWriter)
+ public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, DocumentsWriter docWriter,
+ Codecs codecs)
throws CorruptIndexException, IOException {
this.docWriter = docWriter;
@@ -137,8 +143,28 @@
// First pass: walk the files and initialize our ref
// counts:
long currentGen = segmentInfos.getGeneration();
- IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
+ final Collection codecsExtensions = codecs.getAllExtensions();
+ final FilenameFilter mainFilter = IndexFileNameFilter.getFilter();
+ indexFilenameFilter = new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ if (mainFilter.accept(dir, name)) {
+ return true;
+ } else {
+ // See if any of the codecs claim this
+ // extension:
+ int i = name.lastIndexOf('.');
+ if (i != -1) {
+ String extension = name.substring(1+i);
+ if (codecsExtensions.contains(extension)) {
+ return true;
+ }
+ }
+ return false;
+ }
+ }
+ };
+
String[] files = directory.listAll();
CommitPoint currentCommitPoint = null;
@@ -147,7 +173,7 @@
String fileName = files[i];
- if (filter.accept(null, fileName) && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
+ if ((indexFilenameFilter.accept(null, fileName)) && !fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
// Add this file to refCounts with initial count 0:
getRefCount(fileName);
@@ -163,7 +189,7 @@
}
SegmentInfos sis = new SegmentInfos();
try {
- sis.read(directory, fileName);
+ sis.read(directory, fileName, codecs);
} catch (FileNotFoundException e) {
// LUCENE-948: on NFS (and maybe others), if
// you have writers switching back and forth
@@ -200,7 +226,7 @@
// try now to explicitly open this commit point:
SegmentInfos sis = new SegmentInfos();
try {
- sis.read(directory, segmentInfos.getCurrentSegmentFileName());
+ sis.read(directory, segmentInfos.getCurrentSegmentFileName(), codecs);
} catch (IOException e) {
throw new CorruptIndexException("failed to locate current segments_N file");
}
@@ -298,7 +324,6 @@
*/
public void refresh(String segmentName) throws IOException {
String[] files = directory.listAll();
- IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
String segmentPrefix1;
String segmentPrefix2;
if (segmentName != null) {
@@ -311,8 +336,8 @@
for(int i=0;i<files.length;i++) {
String fileName = files[i];
- if (filter.accept(null, fileName) &&
- (segmentName == null || fileName.startsWith(segmentPrefix1) || fileName.startsWith(segmentPrefix2)) &&
+ if ((segmentName == null || fileName.startsWith(segmentPrefix1) || fileName.startsWith(segmentPrefix2)) &&
+ indexFilenameFilter.accept(null, fileName) &&
!refCounts.containsKey(fileName) &&
!fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
// Unreferenced file, so remove it
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileNames.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileNames.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileNames.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/index/IndexFileNames.java Tue Oct 13 20:44:51 2009
@@ -20,7 +20,8 @@
/**
* Useful constants representing filenames and extensions used by lucene
*/
-final class IndexFileNames {
+// nocommit -- made public
+public final class IndexFileNames {
/** Name of the index segment file */
static final String SEGMENTS = "segments";
@@ -36,16 +37,16 @@
static final String NORMS_EXTENSION = "nrm";
/** Extension of freq postings file */
- static final String FREQ_EXTENSION = "frq";
+ //static final String FREQ_EXTENSION = "frq";
/** Extension of prox postings file */
- static final String PROX_EXTENSION = "prx";
+ //static final String PROX_EXTENSION = "prx";
/** Extension of terms file */
- static final String TERMS_EXTENSION = "tis";
+ //static final String TERMS_EXTENSION = "tis";
/** Extension of terms index file */
- static final String TERMS_INDEX_EXTENSION = "tii";
+ //static final String TERMS_INDEX_EXTENSION = "tii";
/** Extension of stored fields index file */
static final String FIELDS_INDEX_EXTENSION = "fdx";
@@ -96,10 +97,10 @@
FIELD_INFOS_EXTENSION,
FIELDS_INDEX_EXTENSION,
FIELDS_EXTENSION,
- TERMS_INDEX_EXTENSION,
- TERMS_EXTENSION,
- FREQ_EXTENSION,
- PROX_EXTENSION,
+ //TERMS_INDEX_EXTENSION,
+ //TERMS_EXTENSION,
+ //FREQ_EXTENSION,
+ //PROX_EXTENSION,
DELETES_EXTENSION,
VECTORS_INDEX_EXTENSION,
VECTORS_DOCUMENTS_EXTENSION,
@@ -107,6 +108,11 @@
GEN_EXTENSION,
NORMS_EXTENSION,
COMPOUND_FILE_STORE_EXTENSION,
+ // nocommit -- need cleaner way!
+ "doc",
+ "pos",
+ "pyl",
+ "skp"
};
/** File extensions that are added to a compound file
@@ -115,10 +121,10 @@
FIELD_INFOS_EXTENSION,
FIELDS_INDEX_EXTENSION,
FIELDS_EXTENSION,
- TERMS_INDEX_EXTENSION,
- TERMS_EXTENSION,
- FREQ_EXTENSION,
- PROX_EXTENSION,
+ //TERMS_INDEX_EXTENSION,
+ //TERMS_EXTENSION,
+ //FREQ_EXTENSION,
+ //PROX_EXTENSION,
VECTORS_INDEX_EXTENSION,
VECTORS_DOCUMENTS_EXTENSION,
VECTORS_FIELDS_EXTENSION,
@@ -135,22 +141,28 @@
static final String[] NON_STORE_INDEX_EXTENSIONS = new String[] {
FIELD_INFOS_EXTENSION,
- FREQ_EXTENSION,
- PROX_EXTENSION,
- TERMS_EXTENSION,
- TERMS_INDEX_EXTENSION,
+ //FREQ_EXTENSION,
+ //PROX_EXTENSION,
+ //TERMS_EXTENSION,
+ //TERMS_INDEX_EXTENSION,
NORMS_EXTENSION
};
/** File extensions of old-style index files */
static final String COMPOUND_EXTENSIONS[] = new String[] {
FIELD_INFOS_EXTENSION,
- FREQ_EXTENSION,
- PROX_EXTENSION,
+ //FREQ_EXTENSION,
+ //PROX_EXTENSION,
+ FIELDS_INDEX_EXTENSION,
+ FIELDS_EXTENSION,
+ //TERMS_INDEX_EXTENSION,
+ //TERMS_EXTENSION
+ };
+
+ static final String COMPOUND_EXTENSIONS_NOT_CODEC[] = new String[] {
+ FIELD_INFOS_EXTENSION,
FIELDS_INDEX_EXTENSION,
FIELDS_EXTENSION,
- TERMS_INDEX_EXTENSION,
- TERMS_EXTENSION
};
/** File extensions for term vector support */
@@ -194,7 +206,8 @@
return false;
}
- static String segmentFileName(String segmentName, String ext) {
+ // nocommit -- made public
+ public static String segmentFileName(String segmentName, String ext) {
return segmentName + "." + ext;
}
}