You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sa...@apache.org on 2011/06/28 21:42:44 UTC
svn commit: r1140809 [3/11] - in /lucene/dev/branches/solr2452: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/
dev-tools/idea/lucene/contrib/demo/
dev-tools/idea/lucene/contrib/highlighter/
dev-tools/idea/lucene/contrib/inst...
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/document/Field.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/document/Field.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/document/Field.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/document/Field.java Tue Jun 28 19:42:31 2011
@@ -21,7 +21,6 @@ import java.io.Reader;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.util.StringHelper;
/**
A field is a section of a Document. Each field has two parts, a name and a
@@ -367,28 +366,6 @@ public final class Field extends Abstrac
* </ul>
*/
public Field(String name, String value, Store store, Index index, TermVector termVector) {
- this(name, true, value, store, index, termVector);
- }
-
- /**
- * Create a field by specifying its name, value and how it will
- * be saved in the index.
- *
- * @param name The name of the field
- * @param internName Whether to .intern() name or not
- * @param value The string to process
- * @param store Whether <code>value</code> should be stored in the index
- * @param index Whether the field should be indexed, and if so, if it should
- * be tokenized before indexing
- * @param termVector Whether term vector should be stored
- * @throws NullPointerException if name or value is <code>null</code>
- * @throws IllegalArgumentException in any of the following situations:
- * <ul>
- * <li>the field is neither stored nor indexed</li>
- * <li>the field is not indexed but termVector is <code>TermVector.YES</code></li>
- * </ul>
- */
- public Field(String name, boolean internName, String value, Store store, Index index, TermVector termVector) {
if (name == null)
throw new NullPointerException("name cannot be null");
if (value == null)
@@ -402,9 +379,6 @@ public final class Field extends Abstrac
throw new IllegalArgumentException("cannot store term vector information "
+ "for a field that is not indexed");
- if (internName) // field names are optionally interned
- name = StringHelper.intern(name);
-
this.name = name;
this.fieldsData = value;
@@ -422,7 +396,7 @@ public final class Field extends Abstrac
setStoreTermVector(termVector);
}
-
+
/**
* Create a tokenized and indexed field that is not stored. Term vectors will
* not be stored. The Reader is read only when the Document is added to the index,
@@ -454,7 +428,7 @@ public final class Field extends Abstrac
if (reader == null)
throw new NullPointerException("reader cannot be null");
- this.name = StringHelper.intern(name); // field names are interned
+ this.name = name;
this.fieldsData = reader;
this.isStored = false;
@@ -500,7 +474,7 @@ public final class Field extends Abstrac
if (tokenStream == null)
throw new NullPointerException("tokenStream cannot be null");
- this.name = StringHelper.intern(name); // field names are interned
+ this.name = name;
this.fieldsData = null;
this.tokenStream = tokenStream;
@@ -540,7 +514,7 @@ public final class Field extends Abstrac
if (value == null)
throw new IllegalArgumentException("value cannot be null");
- this.name = StringHelper.intern(name); // field names are interned
+ this.name = name;
fieldsData = value;
isStored = true;
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/document/Fieldable.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/document/Fieldable.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/document/Fieldable.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/document/Fieldable.java Tue Jun 28 19:42:31 2011
@@ -71,7 +71,7 @@ public interface Fieldable {
*/
float getBoost();
- /** Returns the name of the field as an interned string.
+ /** Returns the name of the field.
* For example "date", "title", "body", ...
*/
String name();
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/document/IndexDocValuesField.java Tue Jun 28 19:42:31 2011
@@ -98,12 +98,99 @@ public class IndexDocValuesField extends
/**
* Sets the given <code>long</code> value and sets the field's {@link ValueType} to
- * {@link ValueType#INTS} unless already set. If you want to change the
+ * {@link ValueType#VAR_INTS} unless already set. If you want to change the
* default type use {@link #setType(ValueType)}.
*/
public void setInt(long value) {
+ setInt(value, false);
+ }
+
+ /**
+ * Sets the given <code>long</code> value as a 64 bit signed integer.
+ *
+ * @param value
+ * the value to set
+ * @param fixed
+ * if <code>true</code> {@link ValueType#FIXED_INTS_64} is used
+ * otherwise {@link ValueType#VAR_INTS}
+ */
+ public void setInt(long value, boolean fixed) {
+ if (type == null) {
+ type = fixed ? ValueType.FIXED_INTS_64 : ValueType.VAR_INTS;
+ }
+ longValue = value;
+ }
+
+ /**
+ * Sets the given <code>int</code> value and sets the field's {@link ValueType} to
+ * {@link ValueType#VAR_INTS} unless already set. If you want to change the
+ * default type use {@link #setType(ValueType)}.
+ */
+ public void setInt(int value) {
+ setInt(value, false);
+ }
+
+ /**
+ * Sets the given <code>int</code> value as a 32 bit signed integer.
+ *
+ * @param value
+ * the value to set
+ * @param fixed
+ * if <code>true</code> {@link ValueType#FIXED_INTS_32} is used
+ * otherwise {@link ValueType#VAR_INTS}
+ */
+ public void setInt(int value, boolean fixed) {
+ if (type == null) {
+ type = fixed ? ValueType.FIXED_INTS_32 : ValueType.VAR_INTS;
+ }
+ longValue = value;
+ }
+
+ /**
+ * Sets the given <code>short</code> value and sets the field's {@link ValueType} to
+ * {@link ValueType#VAR_INTS} unless already set. If you want to change the
+ * default type use {@link #setType(ValueType)}.
+ */
+ public void setInt(short value) {
+ setInt(value, false);
+ }
+
+ /**
+ * Sets the given <code>short</code> value as a 16 bit signed integer.
+ *
+ * @param value
+ * the value to set
+ * @param fixed
+ * if <code>true</code> {@link ValueType#FIXED_INTS_16} is used
+ * otherwise {@link ValueType#VAR_INTS}
+ */
+ public void setInt(short value, boolean fixed) {
+ if (type == null) {
+ type = fixed ? ValueType.FIXED_INTS_16 : ValueType.VAR_INTS;
+ }
+ longValue = value;
+ }
+
+ /**
+ * Sets the given <code>byte</code> value and sets the field's {@link ValueType} to
+ * {@link ValueType#VAR_INTS} unless already set. If you want to change the
+ * default type use {@link #setType(ValueType)}.
+ */
+ public void setInt(byte value) {
+ setInt(value, false);
+ }
+ /**
+ * Sets the given <code>byte</code> value as a 8 bit signed integer.
+ *
+ * @param value
+ * the value to set
+ * @param fixed
+ * if <code>true</code> {@link ValueType#FIXED_INTS_8} is used
+ * otherwise {@link ValueType#VAR_INTS}
+ */
+ public void setInt(byte value, boolean fixed) {
if (type == null) {
- type = ValueType.INTS;
+ type = fixed ? ValueType.FIXED_INTS_8 : ValueType.VAR_INTS;
}
longValue = value;
}
@@ -268,7 +355,7 @@ public class IndexDocValuesField extends
field.stringValue());
valField.setBytes(ref, type);
break;
- case INTS:
+ case VAR_INTS:
valField.setInt(Long.parseLong(field.stringValue()));
break;
case FLOAT_32:
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/document/NumericField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/document/NumericField.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/document/NumericField.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/document/NumericField.java Tue Jun 28 19:42:31 2011
@@ -24,7 +24,6 @@ import org.apache.lucene.analysis.Numeri
import org.apache.lucene.util.NumericUtils;
import org.apache.lucene.search.NumericRangeQuery; // javadocs
import org.apache.lucene.search.NumericRangeFilter; // javadocs
-import org.apache.lucene.search.SortField; // javadocs
import org.apache.lucene.search.FieldCache; // javadocs
/**
@@ -69,8 +68,8 @@ import org.apache.lucene.search.FieldCac
* <code>NumericField</code>, use {@link NumericRangeQuery} or {@link
* NumericRangeFilter}. To sort according to a
* <code>NumericField</code>, use the normal numeric sort types, eg
- * {@link SortField#INT}. <code>NumericField</code> values
- * can also be loaded directly from {@link FieldCache}.</p>
+ * {@link org.apache.lucene.search.SortField.Type#INT}. <code>NumericField</code>
+ * values can also be loaded directly from {@link FieldCache}.</p>
*
* <p>By default, a <code>NumericField</code>'s value is not stored but
* is indexed for range filtering and sorting. You can use
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/BufferedDeletes.java Tue Jun 28 19:42:31 2011
@@ -47,11 +47,12 @@ class BufferedDeletes {
key, Integer val, int hash, Entry next
(OBJ_HEADER + 3*POINTER + INT). Term is object w/
String field and String text (OBJ_HEADER + 2*POINTER).
- We don't count Term's field since it's interned.
+ Term's field is String (OBJ_HEADER + 4*INT + POINTER +
+ OBJ_HEADER + string.length*CHAR).
Term's text is String (OBJ_HEADER + 4*INT + POINTER +
OBJ_HEADER + string.length*CHAR). Integer is
OBJ_HEADER + INT. */
- final static int BYTES_PER_DEL_TERM = 8*RamUsageEstimator.NUM_BYTES_OBJECT_REF + 5*RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 6*RamUsageEstimator.NUM_BYTES_INT;
+ final static int BYTES_PER_DEL_TERM = 9*RamUsageEstimator.NUM_BYTES_OBJECT_REF + 7*RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 10*RamUsageEstimator.NUM_BYTES_INT;
/* Rough logic: del docIDs are List<Integer>. Say list
allocates ~2X size (2*POINTER). Integer is OBJ_HEADER
@@ -189,7 +190,7 @@ class BufferedDeletes {
terms.put(term, Integer.valueOf(docIDUpto));
numTermDeletes.incrementAndGet();
if (current == null) {
- bytesUsed.addAndGet(BYTES_PER_DEL_TERM + term.bytes.length);
+ bytesUsed.addAndGet(BYTES_PER_DEL_TERM + term.bytes.length + (RamUsageEstimator.NUM_BYTES_CHAR * term.field().length()));
}
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java Tue Jun 28 19:42:31 2011
@@ -380,7 +380,7 @@ class BufferedDeletesStream {
// Since we visit terms sorted, we gain performance
// by re-using the same TermsEnum and seeking only
// forwards
- if (term.field() != currentField) {
+ if (!term.field().equals(currentField)) {
assert currentField == null || currentField.compareTo(term.field()) < 0;
currentField = term.field();
Terms terms = fields.terms(currentField);
@@ -398,7 +398,7 @@ class BufferedDeletesStream {
// System.out.println(" term=" + term);
- if (termsEnum.seek(term.bytes(), false) == TermsEnum.SeekStatus.FOUND) {
+ if (termsEnum.seekExact(term.bytes(), false)) {
DocsEnum docsEnum = termsEnum.docs(reader.getDeletedDocs(), docs);
if (docsEnum != null) {
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/CheckIndex.java Tue Jun 28 19:42:31 2011
@@ -32,6 +32,7 @@ import org.apache.lucene.index.values.In
import org.apache.lucene.index.values.ValuesEnum;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.StringHelper;
import java.text.NumberFormat;
import java.io.PrintStream;
@@ -338,6 +339,27 @@ public class CheckIndex {
return result;
}
+ // find the oldest and newest segment versions
+ String oldest = Integer.toString(Integer.MAX_VALUE), newest = Integer.toString(Integer.MIN_VALUE);
+ String oldSegs = null;
+ boolean foundNonNullVersion = false;
+ Comparator<String> versionComparator = StringHelper.getVersionComparator();
+ for (SegmentInfo si : sis) {
+ String version = si.getVersion();
+ if (version == null) {
+ // pre-3.1 segment
+ oldSegs = "pre-3.1";
+ } else {
+ foundNonNullVersion = true;
+ if (versionComparator.compare(version, oldest) < 0) {
+ oldest = version;
+ }
+ if (versionComparator.compare(version, newest) > 0) {
+ newest = version;
+ }
+ }
+ }
+
final int numSegments = sis.size();
final String segmentsFileName = sis.getCurrentSegmentFileName();
IndexInput input = null;
@@ -372,7 +394,7 @@ public class CheckIndex {
} else if (format == DefaultSegmentInfosWriter.FORMAT_HAS_VECTORS) {
sFormat = "FORMAT_HAS_VECTORS [Lucene 3.1]";
} else if (format == DefaultSegmentInfosWriter.FORMAT_3_1) {
- sFormat = "FORMAT_3_1 [Lucene 3.1]";
+ sFormat = "FORMAT_3_1 [Lucene 3.1+]";
} else if (format == DefaultSegmentInfosWriter.FORMAT_4_0) {
sFormat = "FORMAT_4_0 [Lucene 4.0]";
} else if (format == DefaultSegmentInfosWriter.FORMAT_CURRENT) {
@@ -396,7 +418,19 @@ public class CheckIndex {
userDataString = "";
}
- msg("Segments file=" + segmentsFileName + " numSegments=" + numSegments + " version=" + sFormat + userDataString);
+ String versionString = null;
+ if (oldSegs != null) {
+ if (foundNonNullVersion) {
+ versionString = "versions=[" + oldSegs + " .. " + newest + "]";
+ } else {
+ versionString = "version=" + oldSegs;
+ }
+ } else {
+ versionString = oldest.equals(newest) ? ( "version=" + oldest ) : ("versions=[" + oldest + " .. " + newest + "]");
+ }
+
+ msg("Segments file=" + segmentsFileName + " numSegments=" + numSegments
+ + " " + versionString + " format=" + sFormat + userDataString);
if (onlySegments != null) {
result.partial = true;
@@ -847,7 +881,7 @@ public class CheckIndex {
// Test seek to last term:
if (lastTerm != null) {
- if (terms.seek(lastTerm) != TermsEnum.SeekStatus.FOUND) {
+ if (terms.seekCeil(lastTerm) != TermsEnum.SeekStatus.FOUND) {
throw new RuntimeException("seek to last term " + lastTerm + " failed");
}
@@ -874,14 +908,14 @@ public class CheckIndex {
// Seek by ord
for(int i=seekCount-1;i>=0;i--) {
long ord = i*(termCount/seekCount);
- terms.seek(ord);
+ terms.seekExact(ord);
seekTerms[i] = new BytesRef(terms.term());
}
// Seek by term
long totDocCount = 0;
for(int i=seekCount-1;i>=0;i--) {
- if (terms.seek(seekTerms[i]) != TermsEnum.SeekStatus.FOUND) {
+ if (terms.seekCeil(seekTerms[i]) != TermsEnum.SeekStatus.FOUND) {
throw new RuntimeException("seek to existing term " + seekTerms[i] + " failed");
}
@@ -991,7 +1025,11 @@ public class CheckIndex {
case FLOAT_64:
values.getFloat();
break;
- case INTS:
+ case VAR_INTS:
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
values.getInt();
break;
default:
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/DocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/DocTermOrds.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/DocTermOrds.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/DocTermOrds.java Tue Jun 28 19:42:31 2011
@@ -237,7 +237,7 @@ public class DocTermOrds {
final TermsEnum te = terms.iterator();
final BytesRef seekStart = termPrefix != null ? termPrefix : new BytesRef();
//System.out.println("seekStart=" + seekStart.utf8ToString());
- if (te.seek(seekStart) == TermsEnum.SeekStatus.END) {
+ if (te.seekCeil(seekStart) == TermsEnum.SeekStatus.END) {
// No terms match
return;
}
@@ -693,7 +693,7 @@ public class DocTermOrds {
}
@Override
- public SeekStatus seek(BytesRef target, boolean useCache) throws IOException {
+ public SeekStatus seekCeil(BytesRef target, boolean useCache) throws IOException {
// already here
if (term != null && term.equals(target)) {
@@ -704,7 +704,7 @@ public class DocTermOrds {
if (startIdx >= 0) {
// we hit the term exactly... lucky us!
- TermsEnum.SeekStatus seekStatus = termsEnum.seek(target);
+ TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(target);
assert seekStatus == TermsEnum.SeekStatus.FOUND;
ord = startIdx << indexIntervalBits;
setTerm();
@@ -717,7 +717,7 @@ public class DocTermOrds {
if (startIdx == 0) {
// our target occurs *before* the first term
- TermsEnum.SeekStatus seekStatus = termsEnum.seek(target);
+ TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(target);
assert seekStatus == TermsEnum.SeekStatus.NOT_FOUND;
ord = 0;
setTerm();
@@ -733,7 +733,7 @@ public class DocTermOrds {
// so we don't need to seek.
} else {
// seek to the right block
- TermsEnum.SeekStatus seekStatus = termsEnum.seek(indexedTermsArray[startIdx]);
+ TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(indexedTermsArray[startIdx]);
assert seekStatus == TermsEnum.SeekStatus.FOUND;
ord = startIdx << indexIntervalBits;
setTerm();
@@ -754,16 +754,16 @@ public class DocTermOrds {
}
@Override
- public SeekStatus seek(long targetOrd) throws IOException {
+ public void seekExact(long targetOrd) throws IOException {
int delta = (int) (targetOrd - ordBase - ord);
- //System.out.println(" seek(ord) targetOrd=" + targetOrd + " delta=" + delta + " ord=" + ord);
+ //System.out.println(" seek(ord) targetOrd=" + targetOrd + " delta=" + delta + " ord=" + ord + " ii=" + indexInterval);
if (delta < 0 || delta > indexInterval) {
final int idx = (int) (targetOrd >>> indexIntervalBits);
final BytesRef base = indexedTermsArray[idx];
//System.out.println(" do seek term=" + base.utf8ToString());
ord = idx << indexIntervalBits;
delta = (int) (targetOrd - ord);
- final TermsEnum.SeekStatus seekStatus = termsEnum.seek(base, true);
+ final TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(base, true);
assert seekStatus == TermsEnum.SeekStatus.FOUND;
} else {
//System.out.println("seek w/in block");
@@ -772,15 +772,14 @@ public class DocTermOrds {
while (--delta >= 0) {
BytesRef br = termsEnum.next();
if (br == null) {
- term = null;
- return null;
+ assert false;
+ return;
}
ord++;
}
setTerm();
- return term == null ? SeekStatus.END : SeekStatus.FOUND;
- //System.out.println(" return term=" + term.utf8ToString());
+ assert term != null;
}
private BytesRef setTerm() throws IOException {
@@ -794,8 +793,7 @@ public class DocTermOrds {
}
public BytesRef lookupTerm(TermsEnum termsEnum, int ord) throws IOException {
- TermsEnum.SeekStatus status = termsEnum.seek(ord);
- assert status == TermsEnum.SeekStatus.FOUND;
+ termsEnum.seekExact(ord);
return termsEnum.term();
}
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java Tue Jun 28 19:42:31 2011
@@ -99,7 +99,7 @@ public final class DocumentsWriterFlushC
final long ram = flushBytes + activeBytes;
final long ramBufferBytes = (long) (maxConfiguredRamBuffer * 1024 * 1024);
// take peakDelta into account - worst case is that all flushing, pending and blocked DWPT had maxMem and the last doc had the peakDelta
- final long expected = (long)(2 * (ramBufferBytes)) + ((numPending + numFlushingDWPT() + numBlockedFlushes()) * peakDelta);
+ final long expected = (2 * (ramBufferBytes)) + ((numPending + numFlushingDWPT() + numBlockedFlushes()) * peakDelta);
if (peakDelta < (ramBufferBytes >> 1)) {
/*
* if we are indexing with very low maxRamBuffer like 0.1MB memory can
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FieldInfos.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FieldInfos.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FieldInfos.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FieldInfos.java Tue Jun 28 19:42:31 2011
@@ -36,7 +36,6 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.CodecUtil;
-import org.apache.lucene.util.StringHelper;
/** Access to the Fieldable Info file that describes document fields and whether or
* not they are indexed. Each segment has a separate Fieldable Info file. Objects
@@ -475,12 +474,11 @@ public final class FieldInfos implements
boolean storeTermVector, boolean storePositionWithTermVector,
boolean storeOffsetWithTermVector, boolean omitNorms, boolean storePayloads, boolean omitTermFreqAndPositions, ValueType docValuesType) {
// don't check modifiable here since we use that to initially build up FIs
- name = StringHelper.intern(name);
if (globalFieldNumbers != null) {
globalFieldNumbers.setIfNotSet(fieldNumber, name);
}
final FieldInfo fi = new FieldInfo(name, isIndexed, fieldNumber, storeTermVector, storePositionWithTermVector,
- storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions, docValuesType);
+ storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions, docValuesType);
putInternal(fi);
return fi;
}
@@ -611,7 +609,7 @@ public final class FieldInfos implements
b = 0;
} else {
switch(fi.docValues) {
- case INTS:
+ case VAR_INTS:
b = 1;
break;
case FLOAT_32:
@@ -638,6 +636,19 @@ public final class FieldInfos implements
case BYTES_VAR_SORTED:
b = 9;
break;
+ case FIXED_INTS_16:
+ b = 10;
+ break;
+ case FIXED_INTS_32:
+ b = 11;
+ break;
+ case FIXED_INTS_64:
+ b = 12;
+ break;
+ case FIXED_INTS_8:
+ b = 13;
+ break;
+
default:
throw new IllegalStateException("unhandled indexValues type " + fi.docValues);
}
@@ -659,7 +670,7 @@ public final class FieldInfos implements
final int size = input.readVInt(); //read in the size
for (int i = 0; i < size; i++) {
- String name = StringHelper.intern(input.readString());
+ String name = input.readString();
// if this is a previous format codec 0 will be preflex!
final int fieldNumber = format <= FORMAT_PER_FIELD_CODEC? input.readInt():i;
final int codecId = format <= FORMAT_PER_FIELD_CODEC? input.readInt():0;
@@ -688,7 +699,7 @@ public final class FieldInfos implements
docValuesType = null;
break;
case 1:
- docValuesType = ValueType.INTS;
+ docValuesType = ValueType.VAR_INTS;
break;
case 2:
docValuesType = ValueType.FLOAT_32;
@@ -714,6 +725,19 @@ public final class FieldInfos implements
case 9:
docValuesType = ValueType.BYTES_VAR_SORTED;
break;
+ case 10:
+ docValuesType = ValueType.FIXED_INTS_16;
+ break;
+ case 11:
+ docValuesType = ValueType.FIXED_INTS_32;
+ break;
+ case 12:
+ docValuesType = ValueType.FIXED_INTS_64;
+ break;
+ case 13:
+ docValuesType = ValueType.FIXED_INTS_8;
+ break;
+
default:
throw new IllegalStateException("unhandled indexValues type " + b);
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FieldsEnum.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FieldsEnum.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FieldsEnum.java Tue Jun 28 19:42:31 2011
@@ -43,10 +43,8 @@ public abstract class FieldsEnum {
return atts;
}
- /** Increments the enumeration to the next field. The
- * returned field is always interned, so simple ==
- * comparison is allowed. Returns null when there are no
- * more fields.*/
+ /** Increments the enumeration to the next field. Returns
+ * null when there are no more fields.*/
public abstract String next() throws IOException;
/** Get {@link TermsEnum} for the current field. You
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FieldsReader.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FieldsReader.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FieldsReader.java Tue Jun 28 19:42:31 2011
@@ -364,7 +364,6 @@ public final class FieldsReader implemen
Field.Index index = Field.Index.toIndex(fi.isIndexed, tokenize);
Field.TermVector termVector = Field.TermVector.toTermVector(fi.storeTermVector, fi.storeOffsetWithTermVector, fi.storePositionWithTermVector);
f = new Field(fi.name, // name
- false,
fieldsStream.readString(), // read value
Field.Store.YES,
index,
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java Tue Jun 28 19:42:31 2011
@@ -132,13 +132,18 @@ public class FilterIndexReader extends I
public FilterTermsEnum(TermsEnum in) { this.in = in; }
@Override
- public SeekStatus seek(BytesRef text, boolean useCache) throws IOException {
- return in.seek(text, useCache);
+ public boolean seekExact(BytesRef text, boolean useCache) throws IOException {
+ return in.seekExact(text, useCache);
}
@Override
- public SeekStatus seek(long ord) throws IOException {
- return in.seek(ord);
+ public SeekStatus seekCeil(BytesRef text, boolean useCache) throws IOException {
+ return in.seekCeil(text, useCache);
+ }
+
+ @Override
+ public void seekExact(long ord) throws IOException {
+ in.seekExact(ord);
}
@Override
@@ -182,8 +187,8 @@ public class FilterIndexReader extends I
}
@Override
- public void seek(BytesRef term, TermState state) throws IOException {
- in.seek(term, state);
+ public void seekExact(BytesRef term, TermState state) throws IOException {
+ in.seekExact(term, state);
}
@Override
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java Tue Jun 28 19:42:31 2011
@@ -234,8 +234,6 @@ final class FreqProxTermsWriterPerField
final TermsConsumer termsConsumer = consumer.addField(fieldInfo);
final Comparator<BytesRef> termComp = termsConsumer.getComparator();
- final Term protoTerm = new Term(fieldName);
-
final boolean currentFieldOmitTermFreqAndPositions = fieldInfo.omitTermFreqAndPositions;
final Map<Term,Integer> segDeletes;
@@ -274,7 +272,7 @@ final class FreqProxTermsWriterPerField
final int delDocLimit;
if (segDeletes != null) {
- final Integer docIDUpto = segDeletes.get(protoTerm.createTerm(text));
+ final Integer docIDUpto = segDeletes.get(new Term(fieldName, text));
if (docIDUpto != null) {
delDocLimit = docIDUpto;
} else {
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FrozenBufferedDeletes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FrozenBufferedDeletes.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FrozenBufferedDeletes.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/FrozenBufferedDeletes.java Tue Jun 28 19:42:31 2011
@@ -32,11 +32,12 @@ import org.apache.lucene.index.BufferedD
class FrozenBufferedDeletes {
/* Rough logic: Term is object w/
- String field and String text (OBJ_HEADER + 2*POINTER).
- We don't count Term's field since it's interned.
- Term's text is String (OBJ_HEADER + 4*INT + POINTER +
- OBJ_HEADER + string.length*CHAR). */
- final static int BYTES_PER_DEL_TERM = 3*RamUsageEstimator.NUM_BYTES_OBJECT_REF + 3*RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 4*RamUsageEstimator.NUM_BYTES_INT;
+ String field and BytesRef text (OBJ_HEADER + 2*POINTER).
+ String field is (OBJ_HEADER + 4*INT +
+ POINTER + OBJ_HEADER + CHAR*field.length).
+ Term's text is BytesRef (OBJ_HEADER + 2*INT + POINTER +
+ OBJ_HEADER + bytes.length). */
+ final static int BYTES_PER_DEL_TERM = 4*RamUsageEstimator.NUM_BYTES_OBJECT_REF + 5*RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + 6*RamUsageEstimator.NUM_BYTES_INT;
/* Query we often undercount (say 24 bytes), plus int. */
final static int BYTES_PER_DEL_QUERY = RamUsageEstimator.NUM_BYTES_OBJECT_REF + RamUsageEstimator.NUM_BYTES_INT + 24;
@@ -70,7 +71,13 @@ class FrozenBufferedDeletes {
queryLimits[upto] = ent.getValue();
upto++;
}
- bytesUsed = terms.length * BYTES_PER_DEL_TERM + queries.length * BYTES_PER_DEL_QUERY;
+ int termDataBytes = 0;
+ for(Map.Entry<Term,Integer> ent : deletes.terms.entrySet()) {
+ final Term term = ent.getKey();
+ termDataBytes += term.bytes().length;
+ termDataBytes += term.field().length() * RamUsageEstimator.NUM_BYTES_CHAR;
+ }
+ bytesUsed = terms.length * BYTES_PER_DEL_TERM + queries.length * BYTES_PER_DEL_QUERY + termDataBytes;
numTermDeletes = deletes.numTermDeletes.get();
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/IndexFileNames.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/IndexFileNames.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/IndexFileNames.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/IndexFileNames.java Tue Jun 28 19:42:31 2011
@@ -68,6 +68,9 @@ public final class IndexFileNames {
/** Extension of compound file */
public static final String COMPOUND_FILE_EXTENSION = "cfs";
+
+ /** Extension of compound file entries */
+ public static final String COMPOUND_FILE_ENTRIES_EXTENSION = "cfe";
/** Extension of compound file for doc store files*/
public static final String COMPOUND_FILE_STORE_EXTENSION = "cfx";
@@ -93,6 +96,7 @@ public final class IndexFileNames {
*/
public static final String INDEX_EXTENSIONS[] = new String[] {
COMPOUND_FILE_EXTENSION,
+ COMPOUND_FILE_ENTRIES_EXTENSION,
FIELD_INFOS_EXTENSION,
FIELDS_INDEX_EXTENSION,
FIELDS_EXTENSION,
@@ -244,7 +248,15 @@ public final class IndexFileNames {
}
return filename;
}
-
+
+ public static String stripExtension(String filename) {
+ int idx = filename.indexOf('.');
+ if (idx != -1) {
+ filename = filename.substring(0, idx);
+ }
+ return filename;
+ }
+
/**
* Returns true if the given filename ends with the separate norms file
* pattern: {@code SEPARATE_NORMS_EXTENSION + "[0-9]+"}.
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/IndexReader.java Tue Jun 28 19:42:31 2011
@@ -1432,14 +1432,14 @@ public abstract class IndexReader implem
}
Directory dir = null;
- CompoundFileReader cfr = null;
+ CompoundFileDirectory cfr = null;
try {
File file = new File(filename);
String dirname = file.getAbsoluteFile().getParent();
filename = file.getName();
dir = FSDirectory.open(new File(dirname));
- cfr = new CompoundFileReader(dir, filename);
+ cfr = dir.openCompoundInput(filename, BufferedIndexInput.BUFFER_SIZE);
String [] files = cfr.listAll();
ArrayUtil.mergeSort(files); // sort the array of filename so that the output is more readable
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/IndexWriter.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/IndexWriter.java Tue Jun 28 19:42:31 2011
@@ -46,12 +46,14 @@ import org.apache.lucene.index.codecs.Co
import org.apache.lucene.search.Query;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.BufferedIndexInput;
+import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.Lock;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.BitVector;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.ThreadInterruptedException;
import org.apache.lucene.util.MapBackedSet;
@@ -2190,13 +2192,19 @@ public class IndexWriter implements Clos
String compoundFileName = IndexFileNames.segmentFileName(newSegment.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION);
message("creating compound file " + compoundFileName);
// Now build compound file
- CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, compoundFileName);
- for(String fileName : newSegment.files()) {
- cfsWriter.addFile(fileName);
+ final Directory cfsDir = directory.createCompoundOutput(compoundFileName);
+ IOException prior = null;
+ try {
+ for(String fileName : newSegment.files()) {
+ directory.copy(cfsDir, fileName, fileName);
+ }
+ } catch(IOException ex) {
+ prior = ex;
+ } finally {
+ IOUtils.closeSafely(prior, cfsDir);
}
-
// Perform the merge
- cfsWriter.close();
+
synchronized(this) {
deleter.deleteNewFiles(newSegment.files());
}
@@ -2502,21 +2510,22 @@ public class IndexWriter implements Clos
private void copySegmentIntoCFS(SegmentInfo info, String segName) throws IOException {
String segFileName = IndexFileNames.segmentFileName(segName, "", IndexFileNames.COMPOUND_FILE_EXTENSION);
Collection<String> files = info.files();
- CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, segFileName);
- for (String file : files) {
- String newFileName = segName + IndexFileNames.stripSegmentName(file);
- if (!IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION)
- && !IndexFileNames.isSeparateNormsFile(file)) {
- cfsWriter.addFile(file, info.dir);
- } else {
- assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists";
- info.dir.copy(directory, file, newFileName);
+ final CompoundFileDirectory cfsdir = directory.createCompoundOutput(segFileName);
+ try {
+ for (String file : files) {
+ String newFileName = segName + IndexFileNames.stripSegmentName(file);
+ if (!IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION)
+ && !IndexFileNames.isSeparateNormsFile(file)) {
+ info.dir.copy(cfsdir, file, file);
+ } else {
+ assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists";
+ info.dir.copy(directory, file, newFileName);
+ }
}
+ } finally {
+ IOUtils.closeSafely(true, cfsdir);
}
- // Create the .cfs
- cfsWriter.close();
-
info.dir = directory;
info.name = segName;
info.setUseCompoundFile(true);
@@ -3515,6 +3524,8 @@ public class IndexWriter implements Clos
synchronized(this) {
deleter.deleteFile(compoundFileName);
+
+ deleter.deleteFile(IndexFileNames.segmentFileName(mergedName, "", IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
deleter.deleteNewFiles(merge.info.files());
}
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/MultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/MultiFields.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/MultiFields.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/MultiFields.java Tue Jun 28 19:42:31 2011
@@ -18,15 +18,16 @@ package org.apache.lucene.index;
*/
import java.io.IOException;
-import java.util.Map;
-import java.util.List;
import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
-import org.apache.lucene.util.ReaderUtil;
-import org.apache.lucene.util.ReaderUtil.Gather; // for javadocs
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.MultiBits;
+import org.apache.lucene.util.ReaderUtil.Gather; // for javadocs
+import org.apache.lucene.util.ReaderUtil;
/**
* Exposes flex API, merged from flex API of sub-segments.
@@ -99,49 +100,11 @@ public final class MultiFields extends F
}
}
- private static class MultiReaderBits implements Bits {
- private final int[] starts;
- private final IndexReader[] readers;
- private final Bits[] delDocs;
-
- public MultiReaderBits(int[] starts, IndexReader[] readers) {
- assert readers.length == starts.length-1;
- this.starts = starts;
- this.readers = readers;
- delDocs = new Bits[readers.length];
- for(int i=0;i<readers.length;i++) {
- delDocs[i] = readers[i].getDeletedDocs();
- }
- }
-
- public boolean get(int doc) {
- final int sub = ReaderUtil.subIndex(doc, starts);
- Bits dels = delDocs[sub];
- if (dels == null) {
- // NOTE: this is not sync'd but multiple threads can
- // come through here; I think this is OK -- worst
- // case is more than 1 thread ends up filling in the
- // sub Bits
- dels = readers[sub].getDeletedDocs();
- if (dels == null) {
- return false;
- } else {
- delDocs[sub] = dels;
- }
- }
- return dels.get(doc-starts[sub]);
- }
-
- public int length() {
- return starts[starts.length-1];
- }
- }
-
public static Bits getDeletedDocs(IndexReader r) {
Bits result;
if (r.hasDeletions()) {
- final List<IndexReader> readers = new ArrayList<IndexReader>();
+ final List<Bits> delDocs = new ArrayList<Bits>();
final List<Integer> starts = new ArrayList<Integer>();
try {
@@ -149,7 +112,7 @@ public final class MultiFields extends F
@Override
protected void add(int base, IndexReader r) throws IOException {
// record all delDocs, even if they are null
- readers.add(r);
+ delDocs.add(r.getDeletedDocs());
starts.add(base);
}
}.run();
@@ -159,16 +122,12 @@ public final class MultiFields extends F
throw new RuntimeException(ioe);
}
- assert readers.size() > 0;
- if (readers.size() == 1) {
+ assert delDocs.size() > 0;
+ if (delDocs.size() == 1) {
// Only one actual sub reader -- optimize this case
- result = readers.get(0).getDeletedDocs();
+ result = delDocs.get(0);
} else {
- int[] startsArray = new int[starts.size()];
- for(int i=0;i<startsArray.length;i++) {
- startsArray[i] = starts.get(i);
- }
- result = new MultiReaderBits(startsArray, readers.toArray(new IndexReader[readers.size()]));
+ result = new MultiBits(delDocs, starts);
}
} else {
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/MultiFieldsEnum.java Tue Jun 28 19:42:31 2011
@@ -94,7 +94,7 @@ public final class MultiFieldsEnum exte
if (queue.size() > 0) {
while(true) {
top[numTop++] = queue.pop();
- if (queue.size() == 0 || (queue.top()).current != top[0].current) {
+ if (queue.size() == 0 || !(queue.top()).current.equals(top[0].current)) {
break;
}
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java Tue Jun 28 19:42:31 2011
@@ -139,7 +139,7 @@ public final class MultiTermsEnum extend
}
@Override
- public SeekStatus seek(BytesRef term, boolean useCache) throws IOException {
+ public boolean seekExact(BytesRef term, boolean useCache) throws IOException {
queue.clear();
numTop = 0;
@@ -147,6 +147,56 @@ public final class MultiTermsEnum extend
if (lastSeek != null && termComp.compare(lastSeek, term) <= 0) {
seekOpt = true;
}
+
+ lastSeek = null;
+
+ for(int i=0;i<numSubs;i++) {
+ final boolean status;
+ // LUCENE-2130: if we had just seek'd already, prior
+ // to this seek, and the new seek term is after the
+ // previous one, don't try to re-seek this sub if its
+ // current term is already beyond this new seek term.
+ // Doing so is a waste because this sub will simply
+ // seek to the same spot.
+ if (seekOpt) {
+ final BytesRef curTerm = currentSubs[i].current;
+ if (curTerm != null) {
+ final int cmp = termComp.compare(term, curTerm);
+ if (cmp == 0) {
+ status = true;
+ } else if (cmp < 0) {
+ status = false;
+ } else {
+ status = currentSubs[i].terms.seekExact(term, useCache);
+ }
+ } else {
+ status = false;
+ }
+ } else {
+ status = currentSubs[i].terms.seekExact(term, useCache);
+ }
+
+ if (status) {
+ top[numTop++] = currentSubs[i];
+ current = currentSubs[i].current = currentSubs[i].terms.term();
+ }
+ }
+
+ // if at least one sub had exact match to the requested
+ // term then we found match
+ return numTop > 0;
+ }
+
+ @Override
+ public SeekStatus seekCeil(BytesRef term, boolean useCache) throws IOException {
+ queue.clear();
+ numTop = 0;
+
+ boolean seekOpt = false;
+ if (lastSeek != null && termComp.compare(lastSeek, term) <= 0) {
+ seekOpt = true;
+ }
+
lastSeekScratch.copy(term);
lastSeek = lastSeekScratch;
@@ -167,25 +217,27 @@ public final class MultiTermsEnum extend
} else if (cmp < 0) {
status = SeekStatus.NOT_FOUND;
} else {
- status = currentSubs[i].terms.seek(term, useCache);
+ status = currentSubs[i].terms.seekCeil(term, useCache);
}
} else {
status = SeekStatus.END;
}
} else {
- status = currentSubs[i].terms.seek(term, useCache);
+ status = currentSubs[i].terms.seekCeil(term, useCache);
}
if (status == SeekStatus.FOUND) {
top[numTop++] = currentSubs[i];
current = currentSubs[i].current = currentSubs[i].terms.term();
- } else if (status == SeekStatus.NOT_FOUND) {
- currentSubs[i].current = currentSubs[i].terms.term();
- assert currentSubs[i].current != null;
- queue.add(currentSubs[i]);
} else {
- // enum exhausted
- currentSubs[i].current = null;
+ if (status == SeekStatus.NOT_FOUND) {
+ currentSubs[i].current = currentSubs[i].terms.term();
+ assert currentSubs[i].current != null;
+ queue.add(currentSubs[i]);
+ } else {
+ // enum exhausted
+ currentSubs[i].current = null;
+ }
}
}
@@ -204,7 +256,7 @@ public final class MultiTermsEnum extend
}
@Override
- public SeekStatus seek(long ord) throws IOException {
+ public void seekExact(long ord) throws IOException {
throw new UnsupportedOperationException();
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java Tue Jun 28 19:42:31 2011
@@ -23,6 +23,7 @@ import java.util.concurrent.atomic.Atomi
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.PerDocValues;
+import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
/** Holds core readers that are shared (unchanged) when
@@ -52,8 +53,8 @@ final class SegmentCoreReaders {
FieldsReader fieldsReaderOrig;
TermVectorsReader termVectorsReaderOrig;
- CompoundFileReader cfsReader;
- CompoundFileReader storeCFSReader;
+ CompoundFileDirectory cfsReader;
+ CompoundFileDirectory storeCFSReader;
@@ -73,7 +74,7 @@ final class SegmentCoreReaders {
try {
Directory dir0 = dir;
if (si.getUseCompoundFile()) {
- cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
+ cfsReader = dir.openCompoundInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
dir0 = cfsReader;
}
cfsDir = dir0;
@@ -161,7 +162,7 @@ final class SegmentCoreReaders {
if (si.getDocStoreOffset() != -1) {
if (si.getDocStoreIsCompoundFile()) {
assert storeCFSReader == null;
- storeCFSReader = new CompoundFileReader(dir,
+ storeCFSReader = dir.openCompoundInput(
IndexFileNames.segmentFileName(si.getDocStoreSegment(), "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION),
readBufferSize);
storeDir = storeCFSReader;
@@ -175,7 +176,7 @@ final class SegmentCoreReaders {
// was not used, but then we are asked to open doc
// stores after the segment has switched to CFS
if (cfsReader == null) {
- cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
+ cfsReader = dir.openCompoundInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize);
}
storeDir = cfsReader;
assert storeDir != null;
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentInfo.java Tue Jun 28 19:42:31 2011
@@ -30,10 +30,12 @@ import java.util.Set;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter;
+import org.apache.lucene.store.BufferedIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.Constants;
+import org.apache.lucene.util.StringHelper;
/**
* Information about a segment such as it's name, directory, and files related
@@ -245,7 +247,7 @@ public final class SegmentInfo implement
}
final Directory dirToTest;
if (isCompoundFile) {
- dirToTest = new CompoundFileReader(dir, IndexFileNames.segmentFileName(storesSegment, "", ext));
+ dirToTest = dir.openCompoundInput(IndexFileNames.segmentFileName(storesSegment, "", ext), BufferedIndexInput.BUFFER_SIZE);
} else {
dirToTest = dir;
}
@@ -263,8 +265,8 @@ public final class SegmentInfo implement
if (fieldInfos == null) {
Directory dir0 = dir;
if (isCompoundFile && checkCompoundFile) {
- dir0 = new CompoundFileReader(dir, IndexFileNames.segmentFileName(name,
- "", IndexFileNames.COMPOUND_FILE_EXTENSION));
+ dir0 = dir.openCompoundInput(IndexFileNames.segmentFileName(name,
+ "", IndexFileNames.COMPOUND_FILE_EXTENSION), BufferedIndexInput.BUFFER_SIZE);
}
try {
fieldInfos = new FieldInfos(dir0, IndexFileNames.segmentFileName(name,
@@ -454,7 +456,7 @@ public final class SegmentInfo implement
this.isCompoundFile = isCompoundFile;
clearFilesCache();
}
-
+
/**
* Returns true if this segment is stored as a compound
* file; else, false.
@@ -617,6 +619,10 @@ public final class SegmentInfo implement
if (useCompoundFile) {
fileSet.add(IndexFileNames.segmentFileName(name, "", IndexFileNames.COMPOUND_FILE_EXTENSION));
+ if (version != null && StringHelper.getVersionComparator().compare("3.4", version) <= 0) {
+ fileSet.add(IndexFileNames.segmentFileName(name, "",
+ IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
+ }
} else {
for(String ext : IndexFileNames.NON_STORE_INDEX_EXTENSIONS) {
addIfExists(fileSet, IndexFileNames.segmentFileName(name, "", ext));
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Tue Jun 28 19:42:31 2011
@@ -31,6 +31,7 @@ import org.apache.lucene.index.codecs.Fi
import org.apache.lucene.index.codecs.MergeState;
import org.apache.lucene.index.codecs.PerDocConsumer;
import org.apache.lucene.index.codecs.PerDocValues;
+import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
@@ -132,18 +133,20 @@ final class SegmentMerger {
// Now merge all added files
Collection<String> files = info.files();
- CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName, checkAbort);
- for (String file : files) {
- assert !IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION)
- : ".del file is not allowed in .cfs: " + file;
- assert !IndexFileNames.isSeparateNormsFile(file)
- : "separate norms file (.s[0-9]+) is not allowed in .cfs: " + file;
- cfsWriter.addFile(file);
+ CompoundFileDirectory cfsDir = directory.createCompoundOutput(fileName);
+ try {
+ for (String file : files) {
+ assert !IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION)
+ : ".del file is not allowed in .cfs: " + file;
+ assert !IndexFileNames.isSeparateNormsFile(file)
+ : "separate norms file (.s[0-9]+) is not allowed in .cfs: " + file;
+ directory.copy(cfsDir, file, file);
+ checkAbort.work(directory.fileLength(file));
+ }
+ } finally {
+ cfsDir.close();
}
- // Perform the merge
- cfsWriter.close();
-
return files;
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/Term.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/Term.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/Term.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/Term.java Tue Jun 28 19:42:31 2011
@@ -17,15 +17,12 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import java.util.Comparator;
-
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.StringHelper;
/**
A Term represents a word from text. This is the unit of search. It is
composed of two elements, the text of the word, as a string, and the name of
- the field that the text occurred in, an interned string.
+ the field that the text occurred in.
Note that terms may represent more than words from text fields, but also
things like dates, email addresses, urls, etc. */
@@ -37,13 +34,14 @@ public final class Term implements Compa
/** Constructs a Term with the given field and bytes.
* <p>Note that a null field or null bytes value results in undefined
* behavior for most Lucene APIs that accept a Term parameter.
+ *
* <p>WARNING: the provided BytesRef is not copied, but used directly.
* Therefore the bytes should not be modified after construction, for
* example, you should clone a copy rather than pass reused bytes from
* a TermsEnum.
*/
public Term(String fld, BytesRef bytes) {
- field = fld == null ? null : StringHelper.intern(fld);
+ field = fld;
this.bytes = bytes;
}
@@ -61,28 +59,10 @@ public final class Term implements Compa
* @param fld
*/
public Term(String fld) {
- this(fld, new BytesRef(), true);
- }
-
- /**
- * WARNING: the provided BytesRef is not copied, but used directly.
- * Therefore the bytes should not be modified after construction, for
- * example, you should clone a copy rather than pass reused bytes from
- * a TermsEnum.
- *
- * @lucene.experimental
- */
- public Term(String fld, BytesRef bytes, boolean intern) {
- field = intern ? StringHelper.intern(fld) : fld; // field names are interned
- this.bytes = bytes; // unless already known to be
+ this(fld, new BytesRef());
}
- /** @lucene.experimental */
- public Term(String fld, String text, boolean intern) {
- this(fld, new BytesRef(text), intern);
- }
-
- /** Returns the field of this term, an interned string. The field indicates
+ /** Returns the field of this term. The field indicates
the part of a document which this term came from. */
public final String field() { return field; }
@@ -94,32 +74,6 @@ public final class Term implements Compa
/** Returns the bytes of this term. */
public final BytesRef bytes() { return bytes; }
- /**
- * Optimized construction of new Terms by reusing same field as this Term
- * - avoids field.intern() overhead
- * <p>WARNING: the provided BytesRef is not copied, but used directly.
- * Therefore the bytes should not be modified after construction, for
- * example, you should clone a copy rather than pass reused bytes from
- * a TermsEnum.
- * @param bytes The bytes of the new term (field is implicitly same as this Term instance)
- * @return A new Term
- */
- public Term createTerm(BytesRef bytes)
- {
- return new Term(field,bytes,false);
- }
-
- /**
- * Optimized construction of new Terms by reusing same field as this Term
- * - avoids field.intern() overhead
- * @param text The text of the new term (field is implicitly same as this Term instance)
- * @return A new Term
- */
- public Term createTerm(String text)
- {
- return new Term(field,text,false);
- }
-
@Override
public boolean equals(Object obj) {
if (this == obj)
@@ -157,26 +111,11 @@ public final class Term implements Compa
The ordering of terms is first by field, then by text.*/
public final int compareTo(Term other) {
- if (field == other.field) // fields are interned
+ if (field.equals(other.field)) {
return bytes.compareTo(other.bytes);
- else
- return field.compareTo(other.field);
- }
-
- @Deprecated
- private static final Comparator<BytesRef> legacyComparator =
- BytesRef.getUTF8SortedAsUTF16Comparator();
-
- /**
- * @deprecated (4.0) For internal backwards compatibility use only
- * @lucene.internal
- */
- @Deprecated
- public final int compareToUTF16(Term other) {
- if (field == other.field) // fields are interned
- return legacyComparator.compare(this.bytes, other.bytes);
- else
+ } else {
return field.compareTo(other.field);
+ }
}
/**
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermState.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermState.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermState.java Tue Jun 28 19:42:31 2011
@@ -21,7 +21,7 @@ package org.apache.lucene.index;
* Encapsulates all required internal state to position the associated
* {@link TermsEnum} without re-seeking.
*
- * @see TermsEnum#seek(org.apache.lucene.util.BytesRef, TermState)
+ * @see TermsEnum#seekExact(org.apache.lucene.util.BytesRef, TermState)
* @see TermsEnum#termState()
* @lucene.experimental
*/
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/Terms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/Terms.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/Terms.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/Terms.java Tue Jun 28 19:42:31 2011
@@ -50,7 +50,7 @@ public abstract class Terms {
* exist. */
public int docFreq(BytesRef text) throws IOException {
final TermsEnum termsEnum = getThreadTermsEnum();
- if (termsEnum.seek(text) == TermsEnum.SeekStatus.FOUND) {
+ if (termsEnum.seekExact(text, true)) {
return termsEnum.docFreq();
} else {
return 0;
@@ -62,7 +62,7 @@ public abstract class Terms {
* exist. */
public long totalTermFreq(BytesRef text) throws IOException {
final TermsEnum termsEnum = getThreadTermsEnum();
- if (termsEnum.seek(text) == TermsEnum.SeekStatus.FOUND) {
+ if (termsEnum.seekExact(text, true)) {
return termsEnum.totalTermFreq();
} else {
return 0;
@@ -73,7 +73,7 @@ public abstract class Terms {
* method may return null if the term does not exist. */
public DocsEnum docs(Bits skipDocs, BytesRef text, DocsEnum reuse) throws IOException {
final TermsEnum termsEnum = getThreadTermsEnum();
- if (termsEnum.seek(text) == TermsEnum.SeekStatus.FOUND) {
+ if (termsEnum.seekExact(text, true)) {
return termsEnum.docs(skipDocs, reuse);
} else {
return null;
@@ -85,7 +85,7 @@ public abstract class Terms {
* exists, or positions were not indexed. */
public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, BytesRef text, DocsAndPositionsEnum reuse) throws IOException {
final TermsEnum termsEnum = getThreadTermsEnum();
- if (termsEnum.seek(text) == TermsEnum.SeekStatus.FOUND) {
+ if (termsEnum.seekExact(text, true)) {
return termsEnum.docsAndPositions(skipDocs, reuse);
} else {
return null;
@@ -97,10 +97,10 @@ public abstract class Terms {
* This method may return <code>null</code> if the term does not exist.
*
* @see TermsEnum#termState()
- * @see TermsEnum#seek(BytesRef, TermState) */
+ * @see TermsEnum#seekExact(BytesRef, TermState) */
public DocsEnum docs(Bits skipDocs, BytesRef term, TermState termState, DocsEnum reuse) throws IOException {
final TermsEnum termsEnum = getThreadTermsEnum();
- termsEnum.seek(term, termState);
+ termsEnum.seekExact(term, termState);
return termsEnum.docs(skipDocs, reuse);
}
@@ -110,10 +110,10 @@ public abstract class Terms {
* not indexed.
*
* @see TermsEnum#termState()
- * @see TermsEnum#seek(BytesRef, TermState) */
+ * @see TermsEnum#seekExact(BytesRef, TermState) */
public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, BytesRef term, TermState termState, DocsAndPositionsEnum reuse) throws IOException {
final TermsEnum termsEnum = getThreadTermsEnum();
- termsEnum.seek(term, termState);
+ termsEnum.seekExact(term, termState);
return termsEnum.docsAndPositions(skipDocs, reuse);
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermsEnum.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermsEnum.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermsEnum.java Tue Jun 28 19:42:31 2011
@@ -24,18 +24,20 @@ import org.apache.lucene.util.AttributeS
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
-/** Iterator to seek ({@link #seek}) or step through ({@link
- * #next} terms, obtain frequency information ({@link
- * #docFreq}), and obtain a {@link DocsEnum} or {@link
+/** Iterator to seek ({@link #seekCeil(BytesRef)}, {@link
+ * #seekExact(BytesRef,boolean)}) or step through ({@link
+ * #next} terms to obtain frequency information ({@link
+ * #docFreq}), {@link DocsEnum} or {@link
* DocsAndPositionsEnum} for the current term ({@link
* #docs}.
*
* <p>Term enumerations are always ordered by
* {@link #getComparator}. Each term in the enumeration is
- * greater than all that precede it.</p>
+ * greater than the one before it.</p>
*
- * <p>On obtaining a TermsEnum, you must first call
- * {@link #next} or {@link #seek}.
+ * <p>The TermsEnum is unpositioned when you first obtain it
+ * and you must first successfully call {@link #next} or one
+ * of the <code>seek</code> methods.
*
* @lucene.experimental */
public abstract class TermsEnum {
@@ -48,31 +50,41 @@ public abstract class TermsEnum {
return atts;
}
- /** Represents returned result from {@link #seek}.
+ /** Represents returned result from {@link #seekCeil}.
* If status is FOUND, then the precise term was found.
* If status is NOT_FOUND, then a different term was
* found. If the status is END, the end of the iteration
* was hit. */
public static enum SeekStatus {END, FOUND, NOT_FOUND};
- /** Expert: just like {@link #seek(BytesRef)} but allows
+ /** Attemps to seek to the exact term, returning
+ * true if the term is found. If this returns false, the
+ * enum is unpositioned. For some codecs, seekExact may
+ * be substantially faster than {@link #seekCeil}. */
+ public boolean seekExact(BytesRef text, boolean useCache) throws IOException {
+ return seekCeil(text, useCache) == SeekStatus.FOUND;
+ }
+
+ /** Expert: just like {@link #seekCeil(BytesRef)} but allows
* you to control whether the implementation should
* attempt to use its term cache (if it uses one). */
- public abstract SeekStatus seek(BytesRef text, boolean useCache) throws IOException;
+ public abstract SeekStatus seekCeil(BytesRef text, boolean useCache) throws IOException;
- /** Seeks to the specified term. Returns SeekStatus to
+ /** Seeks to the specified term, if it exists, or to the
+ * next (ceiling) term. Returns SeekStatus to
* indicate whether exact term was found, a different
* term was found, or EOF was hit. The target term may
- * be before or after the current term. */
- public final SeekStatus seek(BytesRef text) throws IOException {
- return seek(text, true);
+ * be before or after the current term. If this returns
+ * SeekStatus.END, the enum is unpositioned. */
+ public final SeekStatus seekCeil(BytesRef text) throws IOException {
+ return seekCeil(text, true);
}
/** Seeks to the specified term by ordinal (position) as
* previously returned by {@link #ord}. The target ord
- * may be before or after the current ord. See {@link
- * #seek(BytesRef)}. */
- public abstract SeekStatus seek(long ord) throws IOException;
+ * may be before or after the current ord, and must be
+ * within bounds. */
+ public abstract void seekExact(long ord) throws IOException;
/**
* Expert: Seeks a specific position by {@link TermState} previously obtained
@@ -82,8 +94,7 @@ public abstract class TermsEnum {
* <p>
* Seeking by {@link TermState} should only be used iff the enum the state was
* obtained from and the enum the state is used for seeking are obtained from
- * the same {@link IndexReader}, otherwise a {@link #seek(BytesRef, TermState)} call can
- * leave the enum in undefined state.
+ * the same {@link IndexReader}.
* <p>
* NOTE: Using this method with an incompatible {@link TermState} might leave
* this {@link TermsEnum} in undefined state. On a segment level
@@ -97,32 +108,30 @@ public abstract class TermsEnum {
* @param term the term the TermState corresponds to
* @param state the {@link TermState}
* */
- public void seek(BytesRef term, TermState state) throws IOException {
- seek(term);
+ public void seekExact(BytesRef term, TermState state) throws IOException {
+ if (!seekExact(term, true)) {
+ throw new IllegalArgumentException("term=" + term + " does not exist");
+ }
}
- /** Increments the enumeration to the next element.
+ /** Increments the enumeration to the next term.
* Returns the resulting term, or null if the end was
- * hit. The returned BytesRef may be re-used across calls
- * to next. */
+ * hit (which means the enum is unpositioned). The
+ * returned BytesRef may be re-used across calls to next. */
public abstract BytesRef next() throws IOException;
- /** Returns current term. Do not call this before calling
- * next() for the first time, after next() returns null
- * or after seek returns {@link SeekStatus#END}.*/
+ /** Returns current term. Do not call this when the enum
+ * is unpositioned. */
public abstract BytesRef term() throws IOException;
/** Returns ordinal position for current term. This is an
* optional method (the codec may throw {@link
* UnsupportedOperationException}). Do not call this
- * before calling {@link #next} for the first time or after
- * {@link #next} returns null or {@link #seek} returns
- * END; */
+ * when the enum is unpositioned. */
public abstract long ord() throws IOException;
/** Returns the number of documents containing the current
- * term. Do not call this before calling next() for the
- * first time, after next() returns null or seek returns
+ * term. Do not call this when the enum is unpositioned.
* {@link SeekStatus#END}.*/
public abstract int docFreq() throws IOException;
@@ -135,9 +144,8 @@ public abstract class TermsEnum {
public abstract long totalTermFreq() throws IOException;
/** Get {@link DocsEnum} for the current term. Do not
- * call this before calling {@link #next} or {@link
- * #seek} for the first time. This method will not
- * return null.
+ * call this when the enum is unpositioned. This method
+ * will not return null.
*
* @param skipDocs set bits are documents that should not
* be returned
@@ -145,10 +153,9 @@ public abstract class TermsEnum {
public abstract DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException;
/** Get {@link DocsAndPositionsEnum} for the current term.
- * Do not call this before calling {@link #next} or
- * {@link #seek} for the first time. This method will
- * only return null if positions were not indexed into
- * the postings by this codec. */
+ * Do not call this when the enum is unpositioned.
+ * This method will only return null if positions were
+ * not indexed into the postings by this codec. */
public abstract DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException;
/**
@@ -160,7 +167,7 @@ public abstract class TermsEnum {
* {@link AttributeSource} states separately
*
* @see TermState
- * @see #seek(BytesRef, TermState)
+ * @see #seekExact(BytesRef, TermState)
*/
public TermState termState() throws IOException {
return new TermState() {
@@ -186,10 +193,10 @@ public abstract class TermsEnum {
*/
public static final TermsEnum EMPTY = new TermsEnum() {
@Override
- public SeekStatus seek(BytesRef term, boolean useCache) { return SeekStatus.END; }
+ public SeekStatus seekCeil(BytesRef term, boolean useCache) { return SeekStatus.END; }
@Override
- public SeekStatus seek(long ord) { return SeekStatus.END; }
+ public void seekExact(long ord) {}
@Override
public BytesRef term() {
@@ -242,7 +249,7 @@ public abstract class TermsEnum {
}
@Override
- public void seek(BytesRef term, TermState state) throws IOException {
+ public void seekExact(BytesRef term, TermState state) throws IOException {
throw new IllegalStateException("this method should never be called");
}
};
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java Tue Jun 28 19:42:31 2011
@@ -91,7 +91,7 @@ public class BlockTermsReader extends Fi
@Override
public boolean equals(Object _other) {
FieldAndTerm other = (FieldAndTerm) _other;
- return other.field == field && term.bytesEquals(other.term);
+ return other.field.equals(field) && term.bytesEquals(other.term);
}
@Override
@@ -302,7 +302,7 @@ public class BlockTermsReader extends Fi
only valid if didIndexNext is true: */
private BytesRef nextIndexTerm;
- /* True after seek(TermState), do defer seeking. If the app then
+ /* True after seekExact(TermState), do defer seeking. If the app then
calls next() (which is not "typical"), then we'll do the real seek */
private boolean seekPending;
@@ -312,13 +312,13 @@ public class BlockTermsReader extends Fi
private int blocksSinceSeek;
private byte[] termSuffixes;
- private ByteArrayDataInput termSuffixesReader = new ByteArrayDataInput(null);
+ private ByteArrayDataInput termSuffixesReader = new ByteArrayDataInput();
/* Common prefix used for all terms in this block. */
private int termBlockPrefix;
private byte[] docFreqBytes;
- private final ByteArrayDataInput freqReader = new ByteArrayDataInput(null);
+ private final ByteArrayDataInput freqReader = new ByteArrayDataInput();
private int metaDataUpto;
public SegmentTermsEnum() throws IOException {
@@ -348,7 +348,7 @@ public class BlockTermsReader extends Fi
// return NOT_FOUND so it's a waste for us to fill in
// the term that was actually NOT_FOUND
@Override
- public SeekStatus seek(final BytesRef target, final boolean useCache) throws IOException {
+ public SeekStatus seekCeil(final BytesRef target, final boolean useCache) throws IOException {
if (indexEnum == null) {
throw new IllegalStateException("terms index was not loaded");
@@ -376,7 +376,7 @@ public class BlockTermsReader extends Fi
if (cachedState != null) {
seekPending = true;
//System.out.println(" cached!");
- seek(target, cachedState);
+ seekExact(target, cachedState);
//System.out.println(" term=" + term.utf8ToString());
return SeekStatus.FOUND;
}
@@ -711,7 +711,7 @@ public class BlockTermsReader extends Fi
}
@Override
- public void seek(BytesRef target, TermState otherState) throws IOException {
+ public void seekExact(BytesRef target, TermState otherState) throws IOException {
//System.out.println("BTR.seek termState target=" + target.utf8ToString() + " " + target + " this=" + this);
assert otherState != null && otherState instanceof BlockTermState;
assert !doOrd || ((BlockTermState) otherState).ord < numTerms;
@@ -731,16 +731,13 @@ public class BlockTermsReader extends Fi
}
@Override
- public SeekStatus seek(long ord) throws IOException {
+ public void seekExact(long ord) throws IOException {
//System.out.println("BTR.seek by ord ord=" + ord);
if (indexEnum == null) {
throw new IllegalStateException("terms index was not loaded");
}
- if (ord >= numTerms) {
- state.ord = numTerms-1;
- return SeekStatus.END;
- }
+ assert ord < numTerms;
// TODO: if ord is in same terms block and
// after current ord, we should avoid this seek just
@@ -768,9 +765,6 @@ public class BlockTermsReader extends Fi
left--;
assert indexIsCurrent;
}
-
- // always found
- return SeekStatus.FOUND;
}
@Override
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java Tue Jun 28 19:42:31 2011
@@ -80,12 +80,17 @@ public class DefaultDocValuesConsumer ex
case BYTES_FIXED_STRAIGHT:
case FLOAT_32:
case FLOAT_64:
- case INTS:
+ case VAR_INTS:
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
files.add(IndexFileNames.segmentFileName(filename, "",
Writer.DATA_EXTENSION));
assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
Writer.DATA_EXTENSION));
break;
+
default:
assert false;
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java Tue Jun 28 19:42:31 2011
@@ -121,8 +121,12 @@ public class DefaultDocValuesProducer ex
protected IndexDocValues loadDocValues(int docCount, Directory dir, String id,
ValueType type) throws IOException {
switch (type) {
- case INTS:
- return Ints.getValues(dir, id, false);
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
+ case VAR_INTS:
+ return Ints.getValues(dir, id, docCount);
case FLOAT_32:
return Floats.getValues(dir, id, docCount);
case FLOAT_64:
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java Tue Jun 28 19:42:31 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.index.codecs;
import java.io.IOException;
-import org.apache.lucene.index.CompoundFileReader;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.FieldsReader;
import org.apache.lucene.index.IndexFileNames;
@@ -68,12 +67,12 @@ public class DefaultSegmentInfosReader e
Directory dir = directory;
if (si.getDocStoreOffset() != -1) {
if (si.getDocStoreIsCompoundFile()) {
- dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName(
+ dir = dir.openCompoundInput(IndexFileNames.segmentFileName(
si.getDocStoreSegment(), "",
IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), 1024);
}
} else if (si.getUseCompoundFile()) {
- dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName(
+ dir = dir.openCompoundInput(IndexFileNames.segmentFileName(
si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), 1024);
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java?rev=1140809&r1=1140808&r2=1140809&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java Tue Jun 28 19:42:31 2011
@@ -533,7 +533,7 @@ public class MemoryCodec extends Codec {
private final static class FSTTermsEnum extends TermsEnum {
private final FieldInfo field;
private final BytesRefFSTEnum<BytesRef> fstEnum;
- private final ByteArrayDataInput buffer = new ByteArrayDataInput(null);
+ private final ByteArrayDataInput buffer = new ByteArrayDataInput();
private boolean didDecode;
private int docFreq;
@@ -561,7 +561,15 @@ public class MemoryCodec extends Codec {
}
@Override
- public SeekStatus seek(BytesRef text, boolean useCache /* ignored */) throws IOException {
+ public boolean seekExact(BytesRef text, boolean useCache /* ignored */) throws IOException {
+ if (VERBOSE) System.out.println("te.seekExact text=" + field.name + ":" + text.utf8ToString() + " this=" + this);
+ current = fstEnum.seekExact(text);
+ didDecode = false;
+ return current != null;
+ }
+
+ @Override
+ public SeekStatus seekCeil(BytesRef text, boolean useCache /* ignored */) throws IOException {
if (VERBOSE) System.out.println("te.seek text=" + field.name + ":" + text.utf8ToString() + " this=" + this);
current = fstEnum.seekCeil(text);
if (current == null) {
@@ -656,7 +664,7 @@ public class MemoryCodec extends Codec {
}
@Override
- public SeekStatus seek(long ord) {
+ public void seekExact(long ord) {
// NOTE: we could add this...
throw new UnsupportedOperationException();
}