You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sa...@apache.org on 2011/07/08 08:41:31 UTC
svn commit: r1144174 [3/7] - in /lucene/dev/branches/solr2452: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/
dev-tools/idea/lucene/contrib/queryparser/ dev-tools/idea/solr/
dev-tools/idea/solr/contrib/analysis-extras/ dev-t...
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentMerger.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentMerger.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/SegmentMerger.java Fri Jul 8 06:41:23 2011
@@ -115,6 +115,7 @@ final class SegmentMerger {
mergedDocs = mergeFields();
mergeTerms();
+ mergePerDoc();
mergeNorms();
if (fieldInfos.hasVectors())
@@ -481,18 +482,11 @@ final class SegmentMerger {
// the new segment:
int docBase = 0;
-
+
final List<Fields> fields = new ArrayList<Fields>();
-
final List<ReaderUtil.Slice> slices = new ArrayList<ReaderUtil.Slice>();
final List<Bits> bits = new ArrayList<Bits>();
final List<Integer> bitsStarts = new ArrayList<Integer>();
-
- // TODO: move this into its own method - this merges currently only docvalues
- final List<PerDocValues> perDocProducers = new ArrayList<PerDocValues>();
- final List<ReaderUtil.Slice> perDocSlices = new ArrayList<ReaderUtil.Slice>();
- final List<Bits> perDocBits = new ArrayList<Bits>();
- final List<Integer> perDocBitsStarts = new ArrayList<Integer>();
for(IndexReader r : readers) {
final Fields f = r.fields();
@@ -503,18 +497,10 @@ final class SegmentMerger {
bits.add(r.getLiveDocs());
bitsStarts.add(docBase);
}
- final PerDocValues producer = r.perDocValues();
- if (producer != null) {
- perDocSlices.add(new ReaderUtil.Slice(docBase, maxDoc, fields.size()));
- perDocProducers.add(producer);
- perDocBits.add(r.getLiveDocs());
- perDocBitsStarts.add(docBase);
- }
docBase += maxDoc;
}
bitsStarts.add(docBase);
- perDocBitsStarts.add(docBase);
// we may gather more readers than mergeState.readerCount
mergeState = new MergeState();
@@ -580,19 +566,45 @@ final class SegmentMerger {
} finally {
consumer.close();
}
+ }
+
+ private void mergePerDoc() throws IOException {
+ final List<PerDocValues> perDocProducers = new ArrayList<PerDocValues>();
+ final List<ReaderUtil.Slice> perDocSlices = new ArrayList<ReaderUtil.Slice>();
+ final List<Bits> perDocBits = new ArrayList<Bits>();
+ final List<Integer> perDocBitsStarts = new ArrayList<Integer>();
+ int docBase = 0;
+ for (IndexReader r : readers) {
+ final int maxDoc = r.maxDoc();
+ final PerDocValues producer = r.perDocValues();
+ if (producer != null) {
+ perDocSlices.add(new ReaderUtil.Slice(docBase, maxDoc, perDocProducers
+ .size()));
+ perDocProducers.add(producer);
+ perDocBits.add(r.getLiveDocs());
+ perDocBitsStarts.add(docBase);
+ }
+ docBase += maxDoc;
+ }
+ perDocBitsStarts.add(docBase);
if (!perDocSlices.isEmpty()) {
- mergeState.multiLiveDocs = new MultiBits(perDocBits, perDocBitsStarts, true);
+ mergeState.multiLiveDocs = new MultiBits(perDocBits, perDocBitsStarts,
+ true);
final PerDocConsumer docsConsumer = codec
.docsConsumer(new PerDocWriteState(segmentWriteState));
+ boolean success = false;
try {
- final MultiPerDocValues multiPerDocValues = new MultiPerDocValues(perDocProducers
- .toArray(PerDocValues.EMPTY_ARRAY), perDocSlices
- .toArray(ReaderUtil.Slice.EMPTY_ARRAY));
+ final MultiPerDocValues multiPerDocValues = new MultiPerDocValues(
+ perDocProducers.toArray(PerDocValues.EMPTY_ARRAY),
+ perDocSlices.toArray(ReaderUtil.Slice.EMPTY_ARRAY));
docsConsumer.merge(mergeState, multiPerDocValues);
+ success = true;
} finally {
- docsConsumer.close();
+ IOUtils.closeSafely(!success, docsConsumer);
}
}
+ /* don't close the perDocProducers here since they are private segment producers
+ * and will be closed once the SegmentReader goes out of scope */
}
private MergeState mergeState;
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java Fri Jul 8 06:41:23 2011
@@ -22,11 +22,13 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
+import java.io.Closeable;
import java.io.IOException;
import java.util.Arrays;
-class TermVectorsReader implements Cloneable {
+class TermVectorsReader implements Cloneable, Closeable {
// NOTE: if you make a new format, it must be larger than
// the current format
@@ -192,14 +194,8 @@ class TermVectorsReader implements Clone
return format;
}
- void close() throws IOException {
- // make all effort to close up. Keep the first exception
- // and throw it as a new one.
- IOException keep = null;
- if (tvx != null) try { tvx.close(); } catch (IOException e) { keep = e; }
- if (tvd != null) try { tvd.close(); } catch (IOException e) { if (keep == null) keep = e; }
- if (tvf != null) try { tvf.close(); } catch (IOException e) { if (keep == null) keep = e; }
- if (keep != null) throw (IOException) keep.fillInStackTrace();
+ public void close() throws IOException {
+ IOUtils.closeSafely(false, tvx, tvd, tvf);
}
/**
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/Codec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/Codec.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/Codec.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/Codec.java Fri Jul 8 06:41:23 2011
@@ -18,6 +18,7 @@ package org.apache.lucene.index.codecs;
*/
import java.io.IOException;
+import java.util.Comparator;
import java.util.Set;
import org.apache.lucene.index.PerDocWriteState;
@@ -25,13 +26,21 @@ import org.apache.lucene.index.SegmentIn
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
/** @lucene.experimental */
public abstract class Codec {
public static final Codec[] EMPTY = new Codec[0];
/** Unique name that's used to retrieve this codec when
* reading the index */
- public String name;
+ public final String name;
+ private boolean dvUseCompoundFile = true;
+ private Comparator<BytesRef> docValuesSortComparator = BytesRef
+ .getUTF8SortedAsUnicodeComparator();
+
+ protected Codec(String name) {
+ this.name = name;
+ }
/** Writes a new segment */
public abstract FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException;
@@ -68,7 +77,48 @@ public abstract class Codec {
/** Records all file extensions this codec uses */
public abstract void getExtensions(Set<String> extensions);
+
+
+ /**
+ * If set to <code>true</code> this codec will use a compound file for
+ * IndexDocValues, otherwise each IndexDocValues field will create up to 2
+ * files per segment.
+ * <p>
+ * NOTE: The default values is <code>true</code>.
+ */
+ public void setDocValuesUseCFS(boolean docValuesUseCFS) {
+ this.dvUseCompoundFile = docValuesUseCFS;
+ }
+
+ /**
+ * Returns <code>true</code> iff compound file should be used for
+ * IndexDocValues, otherwise <code>false</code>.
+ *
+ * @see #setDocValuesUseCFS(boolean)
+ * @return <code>true</code> iff compound file should be used for
+ * IndexDocValues, otherwise <code>false</code>.
+ */
+ public boolean getDocValuesUseCFS() {
+ return dvUseCompoundFile;
+ }
+
+ /**
+ * Sets the {@link BytesRef} comparator for sorted IndexDocValue variants. The
+ * default is {@link BytesRef#getUTF8SortedAsUnicodeComparator()}. *
+ */
+ public void setDocValuesSortComparator(
+ Comparator<BytesRef> docValuesSortComparator) {
+ this.docValuesSortComparator = docValuesSortComparator;
+ }
+ /**
+ * Returns the {@link BytesRef} comparator for sorted IndexDocValue variants.
+ * The default is {@link BytesRef#getUTF8SortedAsUnicodeComparator()}.
+ */
+ public Comparator<BytesRef> getDocValuesSortComparator() {
+ return docValuesSortComparator;
+ }
+
@Override
public String toString() {
return name;
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java Fri Jul 8 06:41:23 2011
@@ -44,7 +44,7 @@ public class CoreCodecProvider extends C
public CoreCodecProvider() {
register(new StandardCodec());
register(new PreFlexCodec());
- register(new PulsingCodec(1));
+ register(new PulsingCodec());
register(new SimpleTextCodec());
register(new MemoryCodec());
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java Fri Jul 8 06:41:23 2011
@@ -31,79 +31,102 @@ import org.apache.lucene.index.values.Wr
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
+/**
+ *
+ * @lucene.experimental
+ */
public class DefaultDocValuesConsumer extends PerDocConsumer {
private final String segmentName;
private final int codecId;
private final Directory directory;
private final AtomicLong bytesUsed;
private final Comparator<BytesRef> comparator;
-
- public DefaultDocValuesConsumer(PerDocWriteState state, Comparator<BytesRef> comparator) {
+ private boolean useCompoundFile;
+
+ public DefaultDocValuesConsumer(PerDocWriteState state, Comparator<BytesRef> comparator, boolean useCompoundFile) throws IOException {
this.segmentName = state.segmentName;
this.codecId = state.codecId;
this.bytesUsed = state.bytesUsed;
- this.directory = state.directory;
+ //TODO maybe we should enable a global CFS that all codecs can pull on demand to further reduce the number of files?
+ this.directory = useCompoundFile ? state.directory.createCompoundOutput(IndexFileNames.segmentFileName(segmentName, state.codecId, IndexFileNames.COMPOUND_FILE_EXTENSION)) : state.directory;
this.comparator = comparator;
+ this.useCompoundFile = useCompoundFile;
}
-
+
public void close() throws IOException {
+ if (useCompoundFile) {
+ this.directory.close();
+ }
}
@Override
public DocValuesConsumer addValuesField(FieldInfo field) throws IOException {
return Writer.create(field.getDocValues(),
docValuesId(segmentName, codecId, field.number),
- // TODO can we have a compound file per segment and codec for
- // docvalues?
directory, comparator, bytesUsed);
}
@SuppressWarnings("fallthrough")
public static void files(Directory dir, SegmentInfo segmentInfo, int codecId,
- Set<String> files) throws IOException {
+ Set<String> files, boolean useCompoundFile) throws IOException {
FieldInfos fieldInfos = segmentInfo.getFieldInfos();
for (FieldInfo fieldInfo : fieldInfos) {
if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) {
String filename = docValuesId(segmentInfo.name, codecId,
fieldInfo.number);
- switch (fieldInfo.getDocValues()) {
- case BYTES_FIXED_DEREF:
- case BYTES_VAR_DEREF:
- case BYTES_VAR_SORTED:
- case BYTES_FIXED_SORTED:
- case BYTES_VAR_STRAIGHT:
- files.add(IndexFileNames.segmentFileName(filename, "",
- Writer.INDEX_EXTENSION));
- assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
- Writer.INDEX_EXTENSION));
- // until here all types use an index
- case BYTES_FIXED_STRAIGHT:
- case FLOAT_32:
- case FLOAT_64:
- case VAR_INTS:
- case FIXED_INTS_16:
- case FIXED_INTS_32:
- case FIXED_INTS_64:
- case FIXED_INTS_8:
- files.add(IndexFileNames.segmentFileName(filename, "",
- Writer.DATA_EXTENSION));
- assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
- Writer.DATA_EXTENSION));
- break;
-
- default:
- assert false;
+ if (useCompoundFile) {
+ files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION));
+ files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
+ assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION));
+ assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION));
+ return;
+ } else {
+ switch (fieldInfo.getDocValues()) {
+ case BYTES_FIXED_DEREF:
+ case BYTES_VAR_DEREF:
+ case BYTES_VAR_SORTED:
+ case BYTES_FIXED_SORTED:
+ case BYTES_VAR_STRAIGHT:
+ files.add(IndexFileNames.segmentFileName(filename, "",
+ Writer.INDEX_EXTENSION));
+ assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
+ Writer.INDEX_EXTENSION));
+ // until here all types use an index
+ case BYTES_FIXED_STRAIGHT:
+ case FLOAT_32:
+ case FLOAT_64:
+ case VAR_INTS:
+ case FIXED_INTS_16:
+ case FIXED_INTS_32:
+ case FIXED_INTS_64:
+ case FIXED_INTS_8:
+ files.add(IndexFileNames.segmentFileName(filename, "",
+ Writer.DATA_EXTENSION));
+ assert dir.fileExists(IndexFileNames.segmentFileName(filename, "",
+ Writer.DATA_EXTENSION));
+ break;
+
+ default:
+ assert false;
+ }
}
}
}
}
+
static String docValuesId(String segmentsName, int codecID, int fieldId) {
return segmentsName + "_" + codecID + "-" + fieldId;
}
-
- public static void getDocValuesExtensions(Set<String> extensions) {
- extensions.add(Writer.DATA_EXTENSION);
- extensions.add(Writer.INDEX_EXTENSION);
+
+ public static void getDocValuesExtensions(Set<String> extensions, boolean useCompoundFile) {
+ if (useCompoundFile) {
+ extensions.add(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION);
+ extensions.add(IndexFileNames.COMPOUND_FILE_EXTENSION);
+ } else {
+ extensions.add(Writer.DATA_EXTENSION);
+ extensions.add(Writer.INDEX_EXTENSION);
+ }
}
+
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java Fri Jul 8 06:41:23 2011
@@ -16,12 +16,16 @@ package org.apache.lucene.index.codecs;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+import java.io.Closeable;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collection;
+import java.util.Comparator;
import java.util.TreeMap;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.SegmentInfo;
import org.apache.lucene.index.values.Bytes;
import org.apache.lucene.index.values.IndexDocValues;
@@ -29,6 +33,8 @@ import org.apache.lucene.index.values.Fl
import org.apache.lucene.index.values.Ints;
import org.apache.lucene.index.values.ValueType;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.IOUtils;
/**
* Abstract base class for FieldsProducer implementations supporting
@@ -39,8 +45,12 @@ import org.apache.lucene.store.Directory
public class DefaultDocValuesProducer extends PerDocValues {
protected final TreeMap<String, IndexDocValues> docValues;
+ private final boolean useCompoundFile;
+ private final Closeable cfs;
+ private final Comparator<BytesRef> sortComparator;
/**
+ *
* Creates a new {@link DefaultDocValuesProducer} instance and loads all
* {@link IndexDocValues} instances for this segment and codec.
*
@@ -52,12 +62,27 @@ public class DefaultDocValuesProducer ex
* the {@link FieldInfos}
* @param codecId
* the codec ID
+ * @param useCompoundFile
+ * if <code>true</code> this producer opens a compound file to read
+ * IndexDocValues fields, otherwise each field defines its own set of
+ * files.
+ * @param sortComparator
+ * defines the sort order for sorted IndexDocValues variants
* @throws IOException
* if an {@link IOException} occurs
*/
- public DefaultDocValuesProducer(SegmentInfo si, Directory dir,
- FieldInfos fieldInfo, int codecId) throws IOException {
- docValues = load(fieldInfo, si.name, si.docCount, dir, codecId);
+ public DefaultDocValuesProducer(SegmentInfo si, Directory dir,
+ FieldInfos fieldInfo, int codecId, boolean useCompoundFile, Comparator<BytesRef> sortComparator) throws IOException {
+ this.useCompoundFile = useCompoundFile;
+ this.sortComparator = sortComparator;
+ final Directory directory;
+ if (useCompoundFile) {
+ cfs = directory = dir.openCompoundInput(IndexFileNames.segmentFileName(si.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION), 1024);
+ } else {
+ cfs = null;
+ directory = dir;
+ }
+ docValues = load(fieldInfo, si.name, si.docCount, directory, codecId);
}
/**
@@ -85,14 +110,14 @@ public class DefaultDocValuesProducer ex
final String id = DefaultDocValuesConsumer.docValuesId(segment,
codecId, fieldInfo.number);
values.put(field,
- loadDocValues(docCount, dir, id, fieldInfo.getDocValues()));
+ loadDocValues(docCount, dir, id, fieldInfo.getDocValues(), sortComparator));
}
}
success = true;
} finally {
if (!success) {
// if we fail we must close all opened resources if there are any
- closeDocValues(values.values());
+ closeInternal(values.values());
}
}
return values;
@@ -112,6 +137,7 @@ public class DefaultDocValuesProducer ex
* the unique file ID within the segment
* @param type
* the type to load
+ * @param sortComparator byte comparator used by sorted variants
* @return a {@link IndexDocValues} instance for the given type
* @throws IOException
* if an {@link IOException} occurs
@@ -119,7 +145,7 @@ public class DefaultDocValuesProducer ex
* if the given {@link ValueType} is not supported
*/
protected IndexDocValues loadDocValues(int docCount, Directory dir, String id,
- ValueType type) throws IOException {
+ ValueType type, Comparator<BytesRef> sortComparator) throws IOException {
switch (type) {
case FIXED_INTS_16:
case FIXED_INTS_32:
@@ -132,39 +158,36 @@ public class DefaultDocValuesProducer ex
case FLOAT_64:
return Floats.getValues(dir, id, docCount);
case BYTES_FIXED_STRAIGHT:
- return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount);
+ return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, true, docCount, sortComparator);
case BYTES_FIXED_DEREF:
- return Bytes.getValues(dir, id, Bytes.Mode.DEREF, true, docCount);
+ return Bytes.getValues(dir, id, Bytes.Mode.DEREF, true, docCount, sortComparator);
case BYTES_FIXED_SORTED:
- return Bytes.getValues(dir, id, Bytes.Mode.SORTED, true, docCount);
+ return Bytes.getValues(dir, id, Bytes.Mode.SORTED, true, docCount, sortComparator);
case BYTES_VAR_STRAIGHT:
- return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, false, docCount);
+ return Bytes.getValues(dir, id, Bytes.Mode.STRAIGHT, false, docCount, sortComparator);
case BYTES_VAR_DEREF:
- return Bytes.getValues(dir, id, Bytes.Mode.DEREF, false, docCount);
+ return Bytes.getValues(dir, id, Bytes.Mode.DEREF, false, docCount, sortComparator);
case BYTES_VAR_SORTED:
- return Bytes.getValues(dir, id, Bytes.Mode.SORTED, false, docCount);
+ return Bytes.getValues(dir, id, Bytes.Mode.SORTED, false, docCount, sortComparator);
default:
throw new IllegalStateException("unrecognized index values mode " + type);
}
}
public void close() throws IOException {
- closeDocValues(docValues.values());
+ closeInternal(docValues.values());
}
- private void closeDocValues(final Collection<IndexDocValues> values)
- throws IOException {
- IOException ex = null;
- for (IndexDocValues docValues : values) {
- try {
- docValues.close();
- } catch (IOException e) {
- ex = e;
- }
- }
- if (ex != null) {
- throw ex;
- }
+ private void closeInternal(Collection<? extends Closeable> closeables) throws IOException {
+ final Collection<? extends Closeable> toClose;
+ if (useCompoundFile) {
+ final ArrayList<Closeable> list = new ArrayList<Closeable>(closeables);
+ list.add(cfs);
+ toClose = list;
+ } else {
+ toClose = closeables;
+ }
+ IOUtils.closeSafely(false, toClose);
}
@Override
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java Fri Jul 8 06:41:23 2011
@@ -76,9 +76,9 @@ import org.apache.lucene.util.fst.FST;
* @lucene.experimental */
public class MemoryCodec extends Codec {
-
+
public MemoryCodec() {
- name = "Memory";
+ super("Memory");
}
private static final boolean VERBOSE = false;
@@ -778,22 +778,22 @@ public class MemoryCodec extends Codec {
@Override
public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, EXTENSION));
- DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
+ DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS());
}
@Override
public void getExtensions(Set<String> extensions) {
extensions.add(EXTENSION);
- DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
+ DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+ return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
+ return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator());
}
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java Fri Jul 8 06:41:23 2011
@@ -55,7 +55,7 @@ public class PreFlexCodec extends Codec
public static final String PROX_EXTENSION = "prx";
public PreFlexCodec() {
- name = "PreFlex";
+ super("PreFlex");
}
@Override
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java Fri Jul 8 06:41:23 2011
@@ -43,7 +43,6 @@ import org.apache.lucene.index.codecs.Te
import org.apache.lucene.index.codecs.TermsIndexWriterBase;
import org.apache.lucene.index.codecs.standard.StandardCodec;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
/** This codec "inlines" the postings for terms that have
@@ -58,10 +57,19 @@ public class PulsingCodec extends Codec
private final int freqCutoff;
+ /**
+ * Creates a {@link PulsingCodec} with <tt>freqCutoff = 1</tt>
+ *
+ * @see PulsingCodec#PulsingCodec(int)
+ */
+ public PulsingCodec() {
+ this(1);
+ }
+
/** Terms with freq <= freqCutoff are inlined into terms
* dict. */
public PulsingCodec(int freqCutoff) {
- name = "Pulsing";
+ super("Pulsing");
this.freqCutoff = freqCutoff;
}
@@ -157,22 +165,22 @@ public class PulsingCodec extends Codec
StandardPostingsReader.files(dir, segmentInfo, id, files);
BlockTermsReader.files(dir, segmentInfo, id, files);
VariableGapTermsIndexReader.files(dir, segmentInfo, id, files);
- DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
+ DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS());
}
@Override
public void getExtensions(Set<String> extensions) {
StandardCodec.getStandardExtensions(extensions);
- DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
+ DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+ return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
+ return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator());
}
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java Fri Jul 8 06:41:23 2011
@@ -33,7 +33,6 @@ import org.apache.lucene.index.codecs.Pe
import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
/** For debugging, curiosity, transparency only!! Do not
* use this codec in production.
@@ -44,11 +43,12 @@ import org.apache.lucene.util.BytesRef;
*
* @lucene.experimental */
public class SimpleTextCodec extends Codec {
-
+
public SimpleTextCodec() {
- name = "SimpleText";
+ super("SimpleText");
}
+
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
return new SimpleTextFieldsWriter(state);
@@ -69,23 +69,23 @@ public class SimpleTextCodec extends Cod
@Override
public void files(Directory dir, SegmentInfo segmentInfo, int id, Set<String> files) throws IOException {
files.add(getPostingsFileName(segmentInfo.name, id));
- DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
+ DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS());
}
@Override
public void getExtensions(Set<String> extensions) {
extensions.add(POSTINGS_EXTENSION);
- DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
+ DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
}
// TODO: would be great if these used a plain text impl
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+ return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
+ return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator());
}
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java Fri Jul 8 06:41:23 2011
@@ -40,14 +40,13 @@ import org.apache.lucene.index.codecs.Bl
import org.apache.lucene.index.codecs.BlockTermsReader;
import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.BytesRef;
/** Default codec.
* @lucene.experimental */
public class StandardCodec extends Codec {
public StandardCodec() {
- name = "Standard";
+ super("Standard");
}
@Override
@@ -140,13 +139,13 @@ public class StandardCodec extends Codec
StandardPostingsReader.files(dir, segmentInfo, id, files);
BlockTermsReader.files(dir, segmentInfo, id, files);
VariableGapTermsIndexReader.files(dir, segmentInfo, id, files);
- DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
+ DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS());
}
@Override
public void getExtensions(Set<String> extensions) {
getStandardExtensions(extensions);
- DefaultDocValuesConsumer.getDocValuesExtensions(extensions);
+ DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
}
public static void getStandardExtensions(Set<String> extensions) {
@@ -158,11 +157,11 @@ public class StandardCodec extends Codec
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state, BytesRef.getUTF8SortedAsUnicodeComparator());
+ return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId);
+ return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator());
}
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Bytes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Bytes.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Bytes.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/Bytes.java Fri Jul 8 06:41:23 2011
@@ -151,12 +151,13 @@ public final class Bytes {
* otherwise <code>false</code>
* @param maxDoc
* the number of document values stored for the given ID
+ * @param sortComparator byte comparator used by sorted variants
* @return an initialized {@link IndexDocValues} instance.
* @throws IOException
* if an {@link IOException} occurs
*/
public static IndexDocValues getValues(Directory dir, String id, Mode mode,
- boolean fixedSize, int maxDoc) throws IOException {
+ boolean fixedSize, int maxDoc, Comparator<BytesRef> sortComparator) throws IOException {
// TODO -- I can peek @ header to determing fixed/mode?
if (fixedSize) {
if (mode == Mode.STRAIGHT) {
@@ -172,7 +173,7 @@ public final class Bytes {
} else if (mode == Mode.DEREF) {
return new VarDerefBytesImpl.Reader(dir, id, maxDoc);
} else if (mode == Mode.SORTED) {
- return new VarSortedBytesImpl.Reader(dir, id, maxDoc);
+ return new VarSortedBytesImpl.Reader(dir, id, maxDoc, sortComparator);
}
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java Fri Jul 8 06:41:23 2011
@@ -130,6 +130,18 @@ public abstract class IndexDocValues imp
throws IOException {
return cache.loadSorted(this, comparator);
}
+
+ /**
+ * Returns a {@link SortedSource} instance using a default {@link BytesRef}
+ * comparator for this {@link IndexDocValues} field instance like
+ * {@link #getSource()}.
+ * <p>
+ * This method will return null iff this {@link IndexDocValues} represent a
+ * {@link Source} instead of a {@link SortedSource}.
+ */
+ public SortedSource getSortedSorted() throws IOException {
+ return getSortedSorted(null);
+ }
/**
* Loads and returns a {@link SortedSource} instance for this
@@ -142,7 +154,19 @@ public abstract class IndexDocValues imp
throws IOException {
throw new UnsupportedOperationException();
}
-
+
+ /**
+ * Loads and returns a {@link SortedSource} instance using a default
+ * {@link BytesRef} comparator for this {@link IndexDocValues} field instance
+ * like {@link #load()}.
+ * <p>
+ * This method will return null iff this {@link IndexDocValues} represent a
+ * {@link Source} instead of a {@link SortedSource}.
+ */
+ public SortedSource loadSorted() throws IOException {
+ return loadSorted(null);
+ }
+
/**
* Returns the {@link ValueType} of this {@link IndexDocValues} instance
*/
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java Fri Jul 8 06:41:23 2011
@@ -167,14 +167,16 @@ class VarSortedBytesImpl {
public static class Reader extends BytesReaderBase {
- Reader(Directory dir, String id, int maxDoc) throws IOException {
+ private final Comparator<BytesRef> defaultComp;
+ Reader(Directory dir, String id, int maxDoc, Comparator<BytesRef> comparator) throws IOException {
super(dir, id, CODEC_NAME, VERSION_START, true);
+ this.defaultComp = comparator;
}
@Override
public org.apache.lucene.index.values.IndexDocValues.Source load()
throws IOException {
- return loadSorted(null);
+ return loadSorted(defaultComp);
}
@Override
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/BooleanQuery.java Fri Jul 8 06:41:23 2011
@@ -183,14 +183,11 @@ public class BooleanQuery extends Query
public Query getQuery() { return BooleanQuery.this; }
@Override
- public float getValue() { return getBoost(); }
-
- @Override
- public float sumOfSquaredWeights() throws IOException {
+ public float getValueForNormalization() throws IOException {
float sum = 0.0f;
for (int i = 0 ; i < weights.size(); i++) {
// call sumOfSquaredWeights for all clauses in case of side effects
- float s = weights.get(i).sumOfSquaredWeights(); // sum sub weights
+ float s = weights.get(i).getValueForNormalization(); // sum sub weights
if (!clauses.get(i).isProhibited())
// only add to sum for non-prohibited clauses
sum += s;
@@ -206,11 +203,11 @@ public class BooleanQuery extends Query
}
@Override
- public void normalize(float norm) {
- norm *= getBoost(); // incorporate boost
+ public void normalize(float norm, float topLevelBoost) {
+ topLevelBoost *= getBoost(); // incorporate boost
for (Weight w : weights) {
// normalize all clauses, (even if prohibited in case of side affects)
- w.normalize(norm);
+ w.normalize(norm, topLevelBoost);
}
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java Fri Jul 8 06:41:23 2011
@@ -27,7 +27,7 @@ import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
-import org.apache.lucene.util.PerReaderTermState;
+import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.BytesRefHash.DirectBytesStartArray;
@@ -77,7 +77,7 @@ class ConstantScoreAutoRewrite extends T
}
@Override
- protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost /*ignored*/, PerReaderTermState states) {
+ protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost /*ignored*/, TermContext states) {
topLevel.add(new TermQuery(term, states), BooleanClause.Occur.SHOULD);
}
@@ -140,9 +140,9 @@ class ConstantScoreAutoRewrite extends T
assert termState != null;
if (pos < 0) {
pos = (-pos)-1;
- array.termState[pos].register(termState, readerContext.ord, termsEnum.docFreq());
+ array.termState[pos].register(termState, readerContext.ord, termsEnum.docFreq(), termsEnum.totalTermFreq());
} else {
- array.termState[pos] = new PerReaderTermState(topReaderContext, termState, readerContext.ord, termsEnum.docFreq());
+ array.termState[pos] = new TermContext(topReaderContext, termState, readerContext.ord, termsEnum.docFreq(), termsEnum.totalTermFreq());
}
return true;
}
@@ -183,9 +183,9 @@ class ConstantScoreAutoRewrite extends T
return true;
}
- /** Special implementation of BytesStartArray that keeps parallel arrays for {@link PerReaderTermState} */
+ /** Special implementation of BytesStartArray that keeps parallel arrays for {@link TermContext} */
static final class TermStateByteStart extends DirectBytesStartArray {
- PerReaderTermState[] termState;
+ TermContext[] termState;
public TermStateByteStart(int initSize) {
super(initSize);
@@ -194,7 +194,7 @@ class ConstantScoreAutoRewrite extends T
@Override
public int[] init() {
final int[] ord = super.init();
- termState = new PerReaderTermState[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ termState = new TermContext[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
assert termState.length >= ord.length;
return ord;
}
@@ -203,7 +203,7 @@ class ConstantScoreAutoRewrite extends T
public int[] grow() {
final int[] ord = super.grow();
if (termState.length < ord.length) {
- PerReaderTermState[] tmpTermState = new PerReaderTermState[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ TermContext[] tmpTermState = new TermContext[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
System.arraycopy(termState, 0, tmpTermState, 0, termState.length);
termState = tmpTermState;
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java Fri Jul 8 06:41:23 2011
@@ -110,24 +110,19 @@ public class ConstantScoreQuery extends
}
@Override
- public float getValue() {
- return queryWeight;
- }
-
- @Override
- public float sumOfSquaredWeights() throws IOException {
+ public float getValueForNormalization() throws IOException {
// we calculate sumOfSquaredWeights of the inner weight, but ignore it (just to initialize everything)
- if (innerWeight != null) innerWeight.sumOfSquaredWeights();
+ if (innerWeight != null) innerWeight.getValueForNormalization();
queryWeight = getBoost();
return queryWeight * queryWeight;
}
@Override
- public void normalize(float norm) {
- this.queryNorm = norm;
+ public void normalize(float norm, float topLevelBoost) {
+ this.queryNorm = norm * topLevelBoost;
queryWeight *= this.queryNorm;
// we normalize the inner weight, but ignore it (just to initialize everything)
- if (innerWeight != null) innerWeight.normalize(norm);
+ if (innerWeight != null) innerWeight.normalize(norm, topLevelBoost);
}
@Override
@@ -148,7 +143,7 @@ public class ConstantScoreQuery extends
if (disi == null) {
return null;
}
- return new ConstantScorer(disi, this);
+ return new ConstantScorer(disi, this, queryWeight);
}
@Override
@@ -181,9 +176,9 @@ public class ConstantScoreQuery extends
final DocIdSetIterator docIdSetIterator;
final float theScore;
- public ConstantScorer(DocIdSetIterator docIdSetIterator, Weight w) throws IOException {
+ public ConstantScorer(DocIdSetIterator docIdSetIterator, Weight w, float theScore) throws IOException {
super(w);
- theScore = w.getValue();
+ this.theScore = theScore;
this.docIdSetIterator = docIdSetIterator;
}
@@ -212,7 +207,7 @@ public class ConstantScoreQuery extends
@Override
public void setScorer(Scorer scorer) throws IOException {
// we must wrap again here, but using the scorer passed in as parameter:
- collector.setScorer(new ConstantScorer(scorer, ConstantScorer.this.weight));
+ collector.setScorer(new ConstantScorer(scorer, ConstantScorer.this.weight, ConstantScorer.this.theScore));
}
@Override
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java Fri Jul 8 06:41:23 2011
@@ -20,7 +20,7 @@ import org.apache.lucene.index.FieldInve
*/
/** Expert: Default scoring implementation. */
-public class DefaultSimilarity extends Similarity {
+public class DefaultSimilarity extends TFIDFSimilarity {
/** Implemented as
* <code>state.getBoost()*lengthNorm(numTerms)</code>, where
@@ -31,15 +31,15 @@ public class DefaultSimilarity extends S
*
* @lucene.experimental */
@Override
- public float computeNorm(FieldInvertState state) {
+ public byte computeNorm(FieldInvertState state) {
final int numTerms;
if (discountOverlaps)
numTerms = state.getLength() - state.getNumOverlap();
else
numTerms = state.getLength();
- return state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms)));
+ return encodeNormValue(state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms))));
}
-
+
/** Implemented as <code>sqrt(freq)</code>. */
@Override
public float tf(float freq) {
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java Fri Jul 8 06:41:23 2011
@@ -110,16 +110,12 @@ public class DisjunctionMaxQuery extends
@Override
public Query getQuery() { return DisjunctionMaxQuery.this; }
- /** Return our boost */
- @Override
- public float getValue() { return getBoost(); }
-
/** Compute the sub of squared weights of us applied to our subqueries. Used for normalization. */
@Override
- public float sumOfSquaredWeights() throws IOException {
+ public float getValueForNormalization() throws IOException {
float max = 0.0f, sum = 0.0f;
for (Weight currentWeight : weights) {
- float sub = currentWeight.sumOfSquaredWeights();
+ float sub = currentWeight.getValueForNormalization();
sum += sub;
max = Math.max(max, sub);
@@ -130,10 +126,10 @@ public class DisjunctionMaxQuery extends
/** Apply the computed normalization factor to our subqueries */
@Override
- public void normalize(float norm) {
- norm *= getBoost(); // Incorporate our boost
+ public void normalize(float norm, float topLevelBoost) {
+ topLevelBoost *= getBoost(); // Incorporate our boost
for (Weight wt : weights) {
- wt.normalize(norm);
+ wt.normalize(norm, topLevelBoost);
}
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java Fri Jul 8 06:41:23 2011
@@ -23,12 +23,6 @@ import java.util.Arrays;
import org.apache.lucene.index.*;
final class ExactPhraseScorer extends Scorer {
- private final byte[] norms;
- private final float value;
-
- private static final int SCORE_CACHE_SIZE = 32;
- private final float[] scoreCache = new float[SCORE_CACHE_SIZE];
-
private final int endMinus1;
private final static int CHUNK = 4096;
@@ -60,14 +54,12 @@ final class ExactPhraseScorer extends Sc
private int docID = -1;
private int freq;
- private final Similarity similarity;
+ private final Similarity.ExactDocScorer docScorer;
ExactPhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings,
- Similarity similarity, byte[] norms) throws IOException {
+ Similarity.ExactDocScorer docScorer) throws IOException {
super(weight);
- this.similarity = similarity;
- this.norms = norms;
- this.value = weight.getValue();
+ this.docScorer = docScorer;
chunkStates = new ChunkState[postings.length];
@@ -88,10 +80,6 @@ final class ExactPhraseScorer extends Sc
return;
}
}
-
- for (int i = 0; i < SCORE_CACHE_SIZE; i++) {
- scoreCache[i] = similarity.tf((float) i) * value;
- }
}
@Override
@@ -206,13 +194,7 @@ final class ExactPhraseScorer extends Sc
@Override
public float score() throws IOException {
- final float raw; // raw score
- if (freq < SCORE_CACHE_SIZE) {
- raw = scoreCache[freq];
- } else {
- raw = similarity.tf((float) freq) * value;
- }
- return norms == null ? raw : raw * similarity.decodeNormValue(norms[docID]); // normalize
+ return docScorer.score(docID, freq);
}
private int phraseFreq() throws IOException {
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/Explanation.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/Explanation.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/Explanation.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/Explanation.java Fri Jul 8 06:41:23 2011
@@ -125,25 +125,4 @@ public class Explanation {
return buffer.toString();
}
-
- /**
- * Small Util class used to pass both an idf factor as well as an
- * explanation for that factor.
- *
- * This class will likely be held on a {@link Weight}, so be aware
- * before storing any large or un-serializable fields.
- *
- */
- public static abstract class IDFExplanation {
- /**
- * @return the idf factor
- */
- public abstract float getIdf();
- /**
- * This should be calculated lazily if possible.
- *
- * @return the explanation for the idf factor.
- */
- public abstract String explain();
- }
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/FilteredQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/FilteredQuery.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/FilteredQuery.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/FilteredQuery.java Fri Jul 8 06:41:23 2011
@@ -63,21 +63,15 @@ extends Query {
public Weight createWeight(final IndexSearcher searcher) throws IOException {
final Weight weight = query.createWeight (searcher);
return new Weight() {
- private float value;
-
- // pass these methods through to enclosed query's weight
- @Override
- public float getValue() { return value; }
@Override
- public float sumOfSquaredWeights() throws IOException {
- return weight.sumOfSquaredWeights() * getBoost() * getBoost();
+ public float getValueForNormalization() throws IOException {
+ return weight.getValueForNormalization() * getBoost() * getBoost();
}
@Override
- public void normalize (float v) {
- weight.normalize(v);
- value = weight.getValue() * getBoost();
+ public void normalize (float norm, float topLevelBoost) {
+ weight.normalize(norm, topLevelBoost);
}
@Override
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/IndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/IndexSearcher.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/IndexSearcher.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/IndexSearcher.java Fri Jul 8 06:41:23 2011
@@ -674,11 +674,11 @@ public class IndexSearcher {
public Weight createNormalizedWeight(Query query) throws IOException {
query = rewrite(query);
Weight weight = query.createWeight(this);
- float sum = weight.sumOfSquaredWeights();
- float norm = getSimilarityProvider().queryNorm(sum);
+ float v = weight.getValueForNormalization();
+ float norm = getSimilarityProvider().queryNorm(v);
if (Float.isInfinite(norm) || Float.isNaN(norm))
norm = 1.0f;
- weight.normalize(norm);
+ weight.normalize(norm, 1.0f);
return weight;
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java Fri Jul 8 06:41:23 2011
@@ -32,35 +32,17 @@ import java.io.IOException;
*/
public class MatchAllDocsQuery extends Query {
- public MatchAllDocsQuery() {
- this(null);
- }
-
- private final String normsField;
-
- /**
- * @param normsField Field used for normalization factor (document boost). Null if nothing.
- */
- public MatchAllDocsQuery(String normsField) {
- this.normsField = normsField;
- }
-
private class MatchAllScorer extends Scorer {
final float score;
- final byte[] norms;
private int doc = -1;
private final int maxDoc;
private final Bits liveDocs;
- private final Similarity similarity;
-
- MatchAllScorer(IndexReader reader, Similarity similarity, Weight w,
- byte[] norms) throws IOException {
+
+ MatchAllScorer(IndexReader reader, Weight w, float score) throws IOException {
super(w);
- this.similarity = similarity;
liveDocs = reader.getLiveDocs();
- score = w.getValue();
+ this.score = score;
maxDoc = reader.maxDoc();
- this.norms = norms;
}
@Override
@@ -82,7 +64,7 @@ public class MatchAllDocsQuery extends Q
@Override
public float score() {
- return norms == null ? score : score * similarity.decodeNormValue(norms[docID()]);
+ return score;
}
@Override
@@ -93,12 +75,10 @@ public class MatchAllDocsQuery extends Q
}
private class MatchAllDocsWeight extends Weight {
- private Similarity similarity;
private float queryWeight;
private float queryNorm;
public MatchAllDocsWeight(IndexSearcher searcher) {
- this.similarity = normsField == null ? null : searcher.getSimilarityProvider().get(normsField);
}
@Override
@@ -112,33 +92,27 @@ public class MatchAllDocsQuery extends Q
}
@Override
- public float getValue() {
- return queryWeight;
- }
-
- @Override
- public float sumOfSquaredWeights() {
+ public float getValueForNormalization() {
queryWeight = getBoost();
return queryWeight * queryWeight;
}
@Override
- public void normalize(float queryNorm) {
- this.queryNorm = queryNorm;
+ public void normalize(float queryNorm, float topLevelBoost) {
+ this.queryNorm = queryNorm * topLevelBoost;
queryWeight *= this.queryNorm;
}
@Override
public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
- return new MatchAllScorer(context.reader, similarity, this,
- normsField != null ? context.reader.norms(normsField) : null);
+ return new MatchAllScorer(context.reader, this, queryWeight);
}
@Override
public Explanation explain(AtomicReaderContext context, int doc) {
// explain query weight
Explanation queryExpl = new ComplexExplanation
- (true, getValue(), "MatchAllDocsQuery, product of:");
+ (true, queryWeight, "MatchAllDocsQuery, product of:");
if (getBoost() != 1.0f) {
queryExpl.addDetail(new Explanation(getBoost(),"boost"));
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java Fri Jul 8 06:41:23 2011
@@ -22,12 +22,14 @@ import java.util.*;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.DocsAndPositionsEnum;
-import org.apache.lucene.search.Explanation.IDFExplanation;
+import org.apache.lucene.search.Similarity.SloppyDocScorer;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ToStringUtils;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.Bits;
@@ -129,45 +131,35 @@ public class MultiPhraseQuery extends Qu
private class MultiPhraseWeight extends Weight {
- private Similarity similarity;
- private float value;
- private final IDFExplanation idfExp;
- private float idf;
- private float queryNorm;
- private float queryWeight;
+ private final Similarity similarity;
+ private final Similarity.Stats stats;
public MultiPhraseWeight(IndexSearcher searcher)
throws IOException {
this.similarity = searcher.getSimilarityProvider().get(field);
-
+ final ReaderContext context = searcher.getTopReaderContext();
+
// compute idf
- ArrayList<Term> allTerms = new ArrayList<Term>();
+ ArrayList<TermContext> allTerms = new ArrayList<TermContext>();
for(final Term[] terms: termArrays) {
for (Term term: terms) {
- allTerms.add(term);
+ allTerms.add(TermContext.build(context, term, true));
}
}
- idfExp = similarity.idfExplain(allTerms, searcher);
- idf = idfExp.getIdf();
+ stats = similarity.computeStats(searcher, field, getBoost(), allTerms.toArray(new TermContext[allTerms.size()]));
}
@Override
public Query getQuery() { return MultiPhraseQuery.this; }
@Override
- public float getValue() { return value; }
-
- @Override
- public float sumOfSquaredWeights() {
- queryWeight = idf * getBoost(); // compute query weight
- return queryWeight * queryWeight; // square it
+ public float getValueForNormalization() {
+ return stats.getValueForNormalization();
}
@Override
- public void normalize(float queryNorm) {
- this.queryNorm = queryNorm;
- queryWeight *= queryNorm; // normalize query weight
- value = queryWeight * idf; // idf for document
+ public void normalize(float queryNorm, float topLevelBoost) {
+ stats.normalize(queryNorm, topLevelBoost);
}
@Override
@@ -222,8 +214,7 @@ public class MultiPhraseQuery extends Qu
}
if (slop == 0) {
- ExactPhraseScorer s = new ExactPhraseScorer(this, postingsFreqs, similarity,
- reader.norms(field));
+ ExactPhraseScorer s = new ExactPhraseScorer(this, postingsFreqs, similarity.exactDocScorer(stats, field, context));
if (s.noDocs) {
return null;
} else {
@@ -231,84 +222,29 @@ public class MultiPhraseQuery extends Qu
}
} else {
return new SloppyPhraseScorer(this, postingsFreqs, similarity,
- slop, reader.norms(field));
+ slop, similarity.sloppyDocScorer(stats, field, context));
}
}
@Override
- public Explanation explain(AtomicReaderContext context, int doc)
- throws IOException {
- ComplexExplanation result = new ComplexExplanation();
- result.setDescription("weight("+getQuery()+" in "+doc+"), product of:");
-
- Explanation idfExpl = new Explanation(idf, "idf(" + field + ":" + idfExp.explain() +")");
-
- // explain query weight
- Explanation queryExpl = new Explanation();
- queryExpl.setDescription("queryWeight(" + getQuery() + "), product of:");
-
- Explanation boostExpl = new Explanation(getBoost(), "boost");
- if (getBoost() != 1.0f)
- queryExpl.addDetail(boostExpl);
-
- queryExpl.addDetail(idfExpl);
-
- Explanation queryNormExpl = new Explanation(queryNorm,"queryNorm");
- queryExpl.addDetail(queryNormExpl);
-
- queryExpl.setValue(boostExpl.getValue() *
- idfExpl.getValue() *
- queryNormExpl.getValue());
-
- result.addDetail(queryExpl);
-
- // explain field weight
- ComplexExplanation fieldExpl = new ComplexExplanation();
- fieldExpl.setDescription("fieldWeight("+getQuery()+" in "+doc+
- "), product of:");
-
+ public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
Scorer scorer = scorer(context, ScorerContext.def());
- if (scorer == null) {
- return new Explanation(0.0f, "no matching docs");
- }
-
- Explanation tfExplanation = new Explanation();
- int d = scorer.advance(doc);
- float phraseFreq;
- if (d == doc) {
- phraseFreq = scorer.freq();
- } else {
- phraseFreq = 0.0f;
+ if (scorer != null) {
+ int newDoc = scorer.advance(doc);
+ if (newDoc == doc) {
+ float freq = scorer.freq();
+ SloppyDocScorer docScorer = similarity.sloppyDocScorer(stats, field, context);
+ ComplexExplanation result = new ComplexExplanation();
+ result.setDescription("weight("+getQuery()+" in "+doc+") [" + similarity.getClass().getSimpleName() + "], result of:");
+ Explanation scoreExplanation = docScorer.explain(doc, new Explanation(freq, "phraseFreq=" + freq));
+ result.addDetail(scoreExplanation);
+ result.setValue(scoreExplanation.getValue());
+ result.setMatch(true);
+ return result;
+ }
}
-
- tfExplanation.setValue(similarity.tf(phraseFreq));
- tfExplanation.setDescription("tf(phraseFreq=" + phraseFreq + ")");
- fieldExpl.addDetail(tfExplanation);
- fieldExpl.addDetail(idfExpl);
-
- Explanation fieldNormExpl = new Explanation();
- byte[] fieldNorms = context.reader.norms(field);
- float fieldNorm =
- fieldNorms!=null ? similarity.decodeNormValue(fieldNorms[doc]) : 1.0f;
- fieldNormExpl.setValue(fieldNorm);
- fieldNormExpl.setDescription("fieldNorm(field="+field+", doc="+doc+")");
- fieldExpl.addDetail(fieldNormExpl);
-
- fieldExpl.setMatch(Boolean.valueOf(tfExplanation.isMatch()));
- fieldExpl.setValue(tfExplanation.getValue() *
- idfExpl.getValue() *
- fieldNormExpl.getValue());
-
- result.addDetail(fieldExpl);
- result.setMatch(fieldExpl.getMatch());
-
- // combine them
- result.setValue(queryExpl.getValue() * fieldExpl.getValue());
-
- if (queryExpl.getValue() == 1.0f)
- return fieldExpl;
-
- return result;
+
+ return new ComplexExplanation(false, 0.0f, "no matching term");
}
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/MultiTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/MultiTermQuery.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/MultiTermQuery.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/MultiTermQuery.java Fri Jul 8 06:41:23 2011
@@ -25,7 +25,7 @@ import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.util.AttributeSource;
-import org.apache.lucene.util.PerReaderTermState;
+import org.apache.lucene.util.TermContext;
/**
* An abstract {@link Query} that matches documents
@@ -154,7 +154,7 @@ public abstract class MultiTermQuery ext
}
@Override
- protected void addClause(BooleanQuery topLevel, Term term, int docCount, float boost, PerReaderTermState states) {
+ protected void addClause(BooleanQuery topLevel, Term term, int docCount, float boost, TermContext states) {
final TermQuery tq = new TermQuery(term, states);
tq.setBoost(boost);
topLevel.add(tq, BooleanClause.Occur.SHOULD);
@@ -195,7 +195,7 @@ public abstract class MultiTermQuery ext
}
@Override
- protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost, PerReaderTermState states) {
+ protected void addClause(BooleanQuery topLevel, Term term, int docFreq, float boost, TermContext states) {
final Query q = new ConstantScoreQuery(new TermQuery(term, states));
q.setBoost(boost);
topLevel.add(q, BooleanClause.Occur.SHOULD);
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/PhraseQuery.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/PhraseQuery.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/PhraseQuery.java Fri Jul 8 06:41:23 2011
@@ -22,10 +22,16 @@ import java.util.Set;
import java.util.ArrayList;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.search.Explanation.IDFExplanation;
+import org.apache.lucene.index.TermState;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.search.Similarity.SloppyDocScorer;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ToStringUtils;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
@@ -171,18 +177,17 @@ public class PhraseQuery extends Query {
private class PhraseWeight extends Weight {
private final Similarity similarity;
- private float value;
- private float idf;
- private float queryNorm;
- private float queryWeight;
- private IDFExplanation idfExp;
+ private final Similarity.Stats stats;
+ private transient TermContext states[];
public PhraseWeight(IndexSearcher searcher)
throws IOException {
this.similarity = searcher.getSimilarityProvider().get(field);
-
- idfExp = similarity.idfExplain(terms, searcher);
- idf = idfExp.getIdf();
+ final ReaderContext context = searcher.getTopReaderContext();
+ states = new TermContext[terms.size()];
+ for (int i = 0; i < terms.size(); i++)
+ states[i] = TermContext.build(context, terms.get(i), true);
+ stats = similarity.computeStats(searcher, field, getBoost(), states);
}
@Override
@@ -192,19 +197,13 @@ public class PhraseQuery extends Query {
public Query getQuery() { return PhraseQuery.this; }
@Override
- public float getValue() { return value; }
-
- @Override
- public float sumOfSquaredWeights() {
- queryWeight = idf * getBoost(); // compute query weight
- return queryWeight * queryWeight; // square it
+ public float getValueForNormalization() {
+ return stats.getValueForNormalization();
}
@Override
- public void normalize(float queryNorm) {
- this.queryNorm = queryNorm;
- queryWeight *= queryNorm; // normalize query weight
- value = queryWeight * idf; // idf for document
+ public void normalize(float queryNorm, float topLevelBoost) {
+ stats.normalize(queryNorm, topLevelBoost);
}
@Override
@@ -216,21 +215,26 @@ public class PhraseQuery extends Query {
PostingsAndFreq[] postingsFreqs = new PostingsAndFreq[terms.size()];
for (int i = 0; i < terms.size(); i++) {
final Term t = terms.get(i);
+ final TermState state = states[i].get(context.ord);
+ if (state == null) { /* term doesnt exist in this segment */
+ assert termNotInReader(reader, field, t.bytes()) : "no termstate found but term exists in reader";
+ return null;
+ }
DocsAndPositionsEnum postingsEnum = reader.termPositionsEnum(liveDocs,
t.field(),
- t.bytes());
+ t.bytes(),
+ state);
// PhraseQuery on a field that did not index
// positions.
if (postingsEnum == null) {
- if (reader.termDocsEnum(liveDocs, t.field(), t.bytes()) != null) {
- // term does exist, but has no positions
- throw new IllegalStateException("field \"" + t.field() + "\" was indexed with Field.omitTermFreqAndPositions=true; cannot run PhraseQuery (term=" + t.text() + ")");
- } else {
- // term does not exist
- return null;
- }
+ assert (reader.termDocsEnum(liveDocs, t.field(), t.bytes(), state) != null) : "termstate found but no term exists in reader";
+ // term does exist, but has no positions
+ throw new IllegalStateException("field \"" + t.field() + "\" was indexed with Field.omitTermFreqAndPositions=true; cannot run PhraseQuery (term=" + t.text() + ")");
}
- postingsFreqs[i] = new PostingsAndFreq(postingsEnum, reader.docFreq(t.field(), t.bytes()), positions.get(i).intValue(), t);
+ // get the docFreq without seeking
+ TermsEnum te = reader.fields().terms(field).getThreadTermsEnum();
+ te.seekExact(t.bytes(), state);
+ postingsFreqs[i] = new PostingsAndFreq(postingsEnum, te.docFreq(), positions.get(i).intValue(), t);
}
// sort by increasing docFreq order
@@ -239,8 +243,7 @@ public class PhraseQuery extends Query {
}
if (slop == 0) { // optimize exact case
- ExactPhraseScorer s = new ExactPhraseScorer(this, postingsFreqs, similarity,
- reader.norms(field));
+ ExactPhraseScorer s = new ExactPhraseScorer(this, postingsFreqs, similarity.exactDocScorer(stats, field, context));
if (s.noDocs) {
return null;
} else {
@@ -248,96 +251,35 @@ public class PhraseQuery extends Query {
}
} else {
return
- new SloppyPhraseScorer(this, postingsFreqs, similarity, slop,
- reader.norms(field));
+ new SloppyPhraseScorer(this, postingsFreqs, similarity, slop, similarity.sloppyDocScorer(stats, field, context));
}
}
+
+ private boolean termNotInReader(IndexReader reader, String field, BytesRef bytes) throws IOException {
+ // only called from assert
+ final Terms terms = reader.terms(field);
+ return terms == null || terms.docFreq(bytes) == 0;
+ }
@Override
- public Explanation explain(AtomicReaderContext context, int doc)
- throws IOException {
-
- ComplexExplanation result = new ComplexExplanation();
- result.setDescription("weight("+getQuery()+" in "+doc+"), product of:");
-
- StringBuilder docFreqs = new StringBuilder();
- StringBuilder query = new StringBuilder();
- query.append('\"');
- docFreqs.append(idfExp.explain());
- for (int i = 0; i < terms.size(); i++) {
- if (i != 0) {
- query.append(" ");
- }
-
- Term term = terms.get(i);
-
- query.append(term.text());
- }
- query.append('\"');
-
- Explanation idfExpl =
- new Explanation(idf, "idf(" + field + ":" + docFreqs + ")");
-
- // explain query weight
- Explanation queryExpl = new Explanation();
- queryExpl.setDescription("queryWeight(" + getQuery() + "), product of:");
-
- Explanation boostExpl = new Explanation(getBoost(), "boost");
- if (getBoost() != 1.0f)
- queryExpl.addDetail(boostExpl);
- queryExpl.addDetail(idfExpl);
-
- Explanation queryNormExpl = new Explanation(queryNorm,"queryNorm");
- queryExpl.addDetail(queryNormExpl);
-
- queryExpl.setValue(boostExpl.getValue() *
- idfExpl.getValue() *
- queryNormExpl.getValue());
-
- result.addDetail(queryExpl);
-
- // explain field weight
- Explanation fieldExpl = new Explanation();
- fieldExpl.setDescription("fieldWeight("+field+":"+query+" in "+doc+
- "), product of:");
-
+ public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
Scorer scorer = scorer(context, ScorerContext.def());
- if (scorer == null) {
- return new Explanation(0.0f, "no matching docs");
- }
- Explanation tfExplanation = new Explanation();
- int d = scorer.advance(doc);
- float phraseFreq;
- if (d == doc) {
- phraseFreq = scorer.freq();
- } else {
- phraseFreq = 0.0f;
+ if (scorer != null) {
+ int newDoc = scorer.advance(doc);
+ if (newDoc == doc) {
+ float freq = scorer.freq();
+ SloppyDocScorer docScorer = similarity.sloppyDocScorer(stats, field, context);
+ ComplexExplanation result = new ComplexExplanation();
+ result.setDescription("weight("+getQuery()+" in "+doc+") [" + similarity.getClass().getSimpleName() + "], result of:");
+ Explanation scoreExplanation = docScorer.explain(doc, new Explanation(freq, "phraseFreq=" + freq));
+ result.addDetail(scoreExplanation);
+ result.setValue(scoreExplanation.getValue());
+ result.setMatch(true);
+ return result;
+ }
}
-
- tfExplanation.setValue(similarity.tf(phraseFreq));
- tfExplanation.setDescription("tf(phraseFreq=" + phraseFreq + ")");
- fieldExpl.addDetail(tfExplanation);
- fieldExpl.addDetail(idfExpl);
-
- Explanation fieldNormExpl = new Explanation();
- byte[] fieldNorms = context.reader.norms(field);
- float fieldNorm =
- fieldNorms!=null ? similarity.decodeNormValue(fieldNorms[doc]) : 1.0f;
- fieldNormExpl.setValue(fieldNorm);
- fieldNormExpl.setDescription("fieldNorm(field="+field+", doc="+doc+")");
- fieldExpl.addDetail(fieldNormExpl);
-
- fieldExpl.setValue(tfExplanation.getValue() *
- idfExpl.getValue() *
- fieldNormExpl.getValue());
-
- result.addDetail(fieldExpl);
-
- // combine them
- result.setValue(queryExpl.getValue() * fieldExpl.getValue());
- result.setMatch(tfExplanation.isMatch());
- return result;
+ return new ComplexExplanation(false, 0.0f, "no matching term");
}
}
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/PhraseScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/PhraseScorer.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/PhraseScorer.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/PhraseScorer.java Fri Jul 8 06:41:23 2011
@@ -30,9 +30,6 @@ import java.io.IOException;
* means a match.
*/
abstract class PhraseScorer extends Scorer {
- protected byte[] norms;
- protected float value;
-
private boolean firstTime = true;
private boolean more = true;
protected PhraseQueue pq;
@@ -40,14 +37,12 @@ abstract class PhraseScorer extends Scor
private float freq; //phrase frequency in current doc as computed by phraseFreq().
- protected final Similarity similarity;
+ protected final Similarity.SloppyDocScorer docScorer;
PhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings,
- Similarity similarity, byte[] norms) {
+ Similarity.SloppyDocScorer docScorer) throws IOException {
super(weight);
- this.similarity = similarity;
- this.norms = norms;
- this.value = weight.getValue();
+ this.docScorer = docScorer;
// convert tps to a list of phrase positions.
// note: phrase-position differs from term-position in that its position
@@ -107,9 +102,7 @@ abstract class PhraseScorer extends Scor
@Override
public float score() throws IOException {
- //System.out.println("scoring " + first.doc);
- float raw = similarity.tf(freq) * value; // raw score
- return norms == null ? raw : raw * similarity.decodeNormValue(norms[first.doc]); // normalize
+ return docScorer.score(first.doc, freq);
}
@Override
Modified: lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/ScoringRewrite.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/ScoringRewrite.java?rev=1144174&r1=1144173&r2=1144174&view=diff
==============================================================================
--- lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/ScoringRewrite.java (original)
+++ lucene/dev/branches/solr2452/lucene/src/java/org/apache/lucene/search/ScoringRewrite.java Fri Jul 8 06:41:23 2011
@@ -28,7 +28,7 @@ import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.ByteBlockPool;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefHash;
-import org.apache.lucene.util.PerReaderTermState;
+import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.BytesRefHash.DirectBytesStartArray;
@@ -56,7 +56,7 @@ public abstract class ScoringRewrite<Q e
@Override
protected void addClause(BooleanQuery topLevel, Term term, int docCount,
- float boost, PerReaderTermState states) {
+ float boost, TermContext states) {
final TermQuery tq = new TermQuery(term, states);
tq.setBoost(boost);
topLevel.add(tq, BooleanClause.Occur.SHOULD);
@@ -117,7 +117,7 @@ public abstract class ScoringRewrite<Q e
if (size > 0) {
final int sort[] = col.terms.sort(col.termsEnum.getComparator());
final float[] boost = col.array.boost;
- final PerReaderTermState[] termStates = col.array.termState;
+ final TermContext[] termStates = col.array.termState;
for (int i = 0; i < size; i++) {
final int pos = sort[i];
final Term term = new Term(query.getField(), col.terms.get(pos, new BytesRef()));
@@ -150,12 +150,12 @@ public abstract class ScoringRewrite<Q e
if (e < 0 ) {
// duplicate term: update docFreq
final int pos = (-e)-1;
- array.termState[pos].register(state, readerContext.ord, termsEnum.docFreq());
+ array.termState[pos].register(state, readerContext.ord, termsEnum.docFreq(), termsEnum.totalTermFreq());
assert array.boost[pos] == boostAtt.getBoost() : "boost should be equal in all segment TermsEnums";
} else {
// new entry: we populate the entry initially
array.boost[e] = boostAtt.getBoost();
- array.termState[e] = new PerReaderTermState(topReaderContext, state, readerContext.ord, termsEnum.docFreq());
+ array.termState[e] = new TermContext(topReaderContext, state, readerContext.ord, termsEnum.docFreq(), termsEnum.totalTermFreq());
ScoringRewrite.this.checkMaxClauseCount(terms.size());
}
return true;
@@ -165,7 +165,7 @@ public abstract class ScoringRewrite<Q e
/** Special implementation of BytesStartArray that keeps parallel arrays for boost and docFreq */
static final class TermFreqBoostByteStart extends DirectBytesStartArray {
float[] boost;
- PerReaderTermState[] termState;
+ TermContext[] termState;
public TermFreqBoostByteStart(int initSize) {
super(initSize);
@@ -175,7 +175,7 @@ public abstract class ScoringRewrite<Q e
public int[] init() {
final int[] ord = super.init();
boost = new float[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_FLOAT)];
- termState = new PerReaderTermState[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ termState = new TermContext[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
assert termState.length >= ord.length && boost.length >= ord.length;
return ord;
}
@@ -185,7 +185,7 @@ public abstract class ScoringRewrite<Q e
final int[] ord = super.grow();
boost = ArrayUtil.grow(boost, ord.length);
if (termState.length < ord.length) {
- PerReaderTermState[] tmpTermState = new PerReaderTermState[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
+ TermContext[] tmpTermState = new TermContext[ArrayUtil.oversize(ord.length, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
System.arraycopy(termState, 0, tmpTermState, 0, termState.length);
termState = tmpTermState;
}