You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by kl...@apache.org on 2006/11/27 23:40:22 UTC
svn commit: r479793 - in /incubator/solr/trunk: ./ example/solr/conf/
src/java/org/apache/solr/request/ src/java/org/apache/solr/schema/
src/java/org/apache/solr/search/ src/java/org/apache/solr/util/
src/test/org/apache/solr/ src/test/test-files/solr/...
Author: klaas
Date: Mon Nov 27 14:40:21 2006
New Revision: 479793
URL: http://svn.apache.org/viewvc?view=rev&rev=479793
Log:
SOLR-52 lazyfields patch implemented.
I am going to commit this with solrconfig defaulting to disabling lazyfields and use it in my
application. When I'm satisfied with the behaviour, I'll close SOLR-52 and perhaps enable
lazyfields as the default option.
Modified:
incubator/solr/trunk/CHANGES.txt
incubator/solr/trunk/example/solr/conf/solrconfig.xml
incubator/solr/trunk/src/java/org/apache/solr/request/DisMaxRequestHandler.java
incubator/solr/trunk/src/java/org/apache/solr/request/JSONResponseWriter.java
incubator/solr/trunk/src/java/org/apache/solr/request/StandardRequestHandler.java
incubator/solr/trunk/src/java/org/apache/solr/request/XMLWriter.java
incubator/solr/trunk/src/java/org/apache/solr/schema/BCDIntField.java
incubator/solr/trunk/src/java/org/apache/solr/schema/BCDLongField.java
incubator/solr/trunk/src/java/org/apache/solr/schema/BCDStrField.java
incubator/solr/trunk/src/java/org/apache/solr/schema/BoolField.java
incubator/solr/trunk/src/java/org/apache/solr/schema/CompressableField.java
incubator/solr/trunk/src/java/org/apache/solr/schema/DateField.java
incubator/solr/trunk/src/java/org/apache/solr/schema/DoubleField.java
incubator/solr/trunk/src/java/org/apache/solr/schema/FieldType.java
incubator/solr/trunk/src/java/org/apache/solr/schema/FloatField.java
incubator/solr/trunk/src/java/org/apache/solr/schema/IndexSchema.java
incubator/solr/trunk/src/java/org/apache/solr/schema/IntField.java
incubator/solr/trunk/src/java/org/apache/solr/schema/LongField.java
incubator/solr/trunk/src/java/org/apache/solr/schema/SchemaField.java
incubator/solr/trunk/src/java/org/apache/solr/schema/SortableDoubleField.java
incubator/solr/trunk/src/java/org/apache/solr/schema/SortableFloatField.java
incubator/solr/trunk/src/java/org/apache/solr/schema/SortableIntField.java
incubator/solr/trunk/src/java/org/apache/solr/schema/SortableLongField.java
incubator/solr/trunk/src/java/org/apache/solr/schema/StrField.java
incubator/solr/trunk/src/java/org/apache/solr/schema/TextField.java
incubator/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java
incubator/solr/trunk/src/java/org/apache/solr/util/HighlightingUtils.java
incubator/solr/trunk/src/java/org/apache/solr/util/SolrPluginUtils.java
incubator/solr/trunk/src/test/org/apache/solr/BasicFunctionalityTest.java
incubator/solr/trunk/src/test/test-files/solr/conf/solrconfig.xml
Modified: incubator/solr/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/CHANGES.txt?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/CHANGES.txt (original)
+++ incubator/solr/trunk/CHANGES.txt Mon Nov 27 14:40:21 2006
@@ -88,6 +88,8 @@
(hossman, SOLR-25)
8. Document update handling locking is much sparser, allowing performance gains
through multiple threads. Large commits also might be faster (klaas, SOLR-65)
+ 9. Lazy field loading can be enabled via a solrconfig directive. This will be faster when
+ not all stored fields are needed from a document (klaas, SOLR-52)
Optimizations
1. getDocListAndSet can now generate both a DocList and a DocSet from a
Modified: incubator/solr/trunk/example/solr/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/example/solr/conf/solrconfig.xml?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/example/solr/conf/solrconfig.xml (original)
+++ incubator/solr/trunk/example/solr/conf/solrconfig.xml Mon Nov 27 14:40:21 2006
@@ -132,6 +132,10 @@
initialSize="512"
autowarmCount="0"/>
+ <!-- If true, stored fields that are not requested will be loaded lazily.
+ -->
+ <enableLazyFieldLoading>false</enableLazyFieldLoading>
+
<!-- Example of a generic cache. These caches may be accessed by name
through SolrIndexSearcher.getCache(),cacheLookup(), and cacheInsert().
The purpose is to enable easy caching of user/application level data.
Modified: incubator/solr/trunk/src/java/org/apache/solr/request/DisMaxRequestHandler.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/request/DisMaxRequestHandler.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/request/DisMaxRequestHandler.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/request/DisMaxRequestHandler.java Mon Nov 27 14:40:21 2006
@@ -347,6 +347,9 @@
flags);
}
rsp.add("search-results",results.docList);
+ // pre-fetch returned documents
+ U.optimizePreFetchDocs(results.docList, query, req, rsp);
+
if (null != facetInfo) rsp.add("facet_counts", facetInfo);
Modified: incubator/solr/trunk/src/java/org/apache/solr/request/JSONResponseWriter.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/request/JSONResponseWriter.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/request/JSONResponseWriter.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/request/JSONResponseWriter.java Mon Nov 27 14:40:21 2006
@@ -382,7 +382,7 @@
DocIterator iterator = ids.iterator();
for (int i=0; i<sz; i++) {
int id = iterator.nextDoc();
- Document doc = searcher.doc(id);
+ Document doc = searcher.doc(id, fields);
if (first) {
first=false;
Modified: incubator/solr/trunk/src/java/org/apache/solr/request/StandardRequestHandler.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/request/StandardRequestHandler.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/request/StandardRequestHandler.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/request/StandardRequestHandler.java Mon Nov 27 14:40:21 2006
@@ -135,6 +135,9 @@
p.getInt(START,0), p.getInt(ROWS,10),
flags);
}
+
+ // pre-fetch returned documents
+ U.optimizePreFetchDocs(results.docList, query, req, rsp);
rsp.add(null,results.docList);
@@ -154,7 +157,7 @@
rsp.add("debug", dbg);
}
} catch (Exception e) {
- SolrException.logOnce(SolrCore.log, "Exception durring debug", e);
+ SolrException.logOnce(SolrCore.log, "Exception during debug", e);
rsp.add("exception_during_debug", SolrException.toStr(e));
}
Modified: incubator/solr/trunk/src/java/org/apache/solr/request/XMLWriter.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/request/XMLWriter.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/request/XMLWriter.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/request/XMLWriter.java Mon Nov 27 14:40:21 2006
@@ -30,7 +30,7 @@
import java.io.IOException;
import java.util.*;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
import org.apache.lucene.document.Document;
/**
* @author yonik
@@ -215,7 +215,7 @@
private static final Comparator fieldnameComparator = new Comparator() {
public int compare(Object o, Object o1) {
- Field f1 = (Field)o; Field f2 = (Field)o1;
+ Fieldable f1 = (Fieldable)o; Fieldable f2 = (Fieldable)o1;
int cmp = f1.name().compareTo(f2.name());
return cmp;
// note - the sort is stable, so this should not have affected the ordering
@@ -238,13 +238,12 @@
// an array. The fastest way to detect multiple fields
// with the same name is to sort them first.
- Enumeration ee = doc.fields();
// using global tlst here, so we shouldn't call any other
// function that uses it until we are done.
tlst.clear();
- while (ee.hasMoreElements()) {
- Field ff = (Field) ee.nextElement();
+ for (Object obj : doc.getFields()) {
+ Fieldable ff = (Fieldable)obj;
// skip this field if it is not a field to be returned.
if (returnFields!=null && !returnFields.contains(ff.name())) {
continue;
@@ -256,12 +255,12 @@
int sz = tlst.size();
int fidx1 = 0, fidx2 = 0;
while (fidx1 < sz) {
- Field f1 = (Field)tlst.get(fidx1);
+ Fieldable f1 = (Fieldable)tlst.get(fidx1);
String fname = f1.name();
// find the end of fields with this name
fidx2 = fidx1+1;
- while (fidx2 < sz && fname.equals(((Field)tlst.get(fidx2)).name()) ) {
+ while (fidx2 < sz && fname.equals(((Fieldable)tlst.get(fidx2)).name()) ) {
fidx2++;
}
@@ -297,7 +296,7 @@
indent();
cnt=0;
}
- sf.write(this, null, (Field)tlst.get(i));
+ sf.write(this, null, (Fieldable)tlst.get(i));
}
decLevel();
// if (doIndent) indent();
@@ -343,7 +342,7 @@
DocIterator iterator = ids.iterator();
for (int i=0; i<sz; i++) {
int id = iterator.nextDoc();
- Document doc = searcher.doc(id);
+ Document doc = searcher.doc(id, fields);
writeDoc(null, doc, fields, (includeScore ? iterator.score() : 0.0f), includeScore);
}
decLevel();
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/BCDIntField.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/BCDIntField.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/BCDIntField.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/BCDIntField.java Mon Nov 27 14:40:21 2006
@@ -19,7 +19,6 @@
import org.apache.lucene.search.SortField;
import org.apache.solr.search.function.ValueSource;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.solr.util.BCDUtils;
import org.apache.solr.request.XMLWriter;
@@ -47,7 +46,7 @@
return BCDUtils.base10toBase10kSortableInt(val);
}
- public String toExternal(Field f) {
+ public String toExternal(Fieldable f) {
return indexedToReadable(f.stringValue());
}
@@ -55,7 +54,7 @@
return BCDUtils.base10kSortableIntToBase10(indexedForm);
}
- public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+ public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeInt(name,toExternal(f));
}
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/BCDLongField.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/BCDLongField.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/BCDLongField.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/BCDLongField.java Mon Nov 27 14:40:21 2006
@@ -18,7 +18,7 @@
package org.apache.solr.schema;
import org.apache.solr.request.XMLWriter;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
import java.io.IOException;
/**
@@ -26,7 +26,7 @@
* @version $Id$
*/
public class BCDLongField extends BCDIntField {
- public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+ public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeLong(name,toExternal(f));
}
}
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/BCDStrField.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/BCDStrField.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/BCDStrField.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/BCDStrField.java Mon Nov 27 14:40:21 2006
@@ -18,7 +18,7 @@
package org.apache.solr.schema;
import org.apache.solr.request.XMLWriter;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
import java.io.IOException;
/**
@@ -26,7 +26,7 @@
* @version $Id$
*/
public class BCDStrField extends BCDIntField {
- public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+ public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeStr(name,toExternal(f));
}
}
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/BoolField.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/BoolField.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/BoolField.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/BoolField.java Mon Nov 27 14:40:21 2006
@@ -24,7 +24,6 @@
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
@@ -86,7 +85,7 @@
return (ch=='1' || ch=='t' || ch=='T') ? "T" : "F";
}
- public String toExternal(Field f) {
+ public String toExternal(Fieldable f) {
return indexedToReadable(f.stringValue());
}
@@ -95,7 +94,7 @@
return ch=='T' ? "true" : "false";
}
- public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+ public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeBool(name, f.stringValue().charAt(0) =='T');
}
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/CompressableField.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/CompressableField.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/CompressableField.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/CompressableField.java Mon Nov 27 14:40:21 2006
@@ -18,6 +18,7 @@
package org.apache.solr.schema;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
import org.apache.solr.request.*;
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/DateField.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/DateField.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/DateField.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/DateField.java Mon Nov 27 14:40:21 2006
@@ -20,7 +20,6 @@
import org.apache.solr.core.SolrException;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.search.SortField;
import org.apache.solr.search.function.ValueSource;
@@ -117,7 +116,7 @@
return indexedForm + 'Z';
}
- public String toExternal(Field f) {
+ public String toExternal(Fieldable f) {
return indexedToReadable(f.stringValue());
}
@@ -129,7 +128,7 @@
return new OrdFieldSource(field.name);
}
- public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+ public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeDate(name, toExternal(f));
}
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/DoubleField.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/DoubleField.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/DoubleField.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/DoubleField.java Mon Nov 27 14:40:21 2006
@@ -20,7 +20,6 @@
import org.apache.lucene.search.SortField;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.FloatFieldSource;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
@@ -47,7 +46,7 @@
return new FloatFieldSource(field.name);
}
- public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+ public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeDouble(name, f.stringValue());
}
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/FieldType.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/FieldType.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/FieldType.java Mon Nov 27 14:40:21 2006
@@ -225,7 +225,7 @@
*/
public String toExternal(Fieldable f) {
// currently used in writing XML of the search result (but perhaps
- // a more efficient toXML(Field f, Writer w) should be used
+ // a more efficient toXML(Fieldable f, Writer w) should be used
// in the future.
return f.stringValue();
}
@@ -357,7 +357,7 @@
/**
* Renders the specified field as XML
*/
- public abstract void write(XMLWriter xmlWriter, String name, Field f) throws IOException;
+ public abstract void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException;
/**
* calls back to TextResponseWriter to write the field value
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/FloatField.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/FloatField.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/FloatField.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/FloatField.java Mon Nov 27 14:40:21 2006
@@ -20,7 +20,6 @@
import org.apache.lucene.search.SortField;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.FloatFieldSource;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
@@ -44,7 +43,7 @@
return new FloatFieldSource(field.name);
}
- public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+ public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeFloat(name, f.stringValue());
}
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/IndexSchema.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/IndexSchema.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/IndexSchema.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/IndexSchema.java Mon Nov 27 14:40:21 2006
@@ -20,6 +20,7 @@
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Fieldable;
import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.Similarity;
import org.apache.solr.core.SolrException;
@@ -170,8 +171,8 @@
* @return null if this schema has no unique key field
* @see #printableUniqueKey
*/
- public Field getUniqueKeyField(org.apache.lucene.document.Document doc) {
- return doc.getField(uniqueKeyFieldName); // this should return null if name is null
+ public Fieldable getUniqueKeyField(org.apache.lucene.document.Document doc) {
+ return doc.getFieldable(uniqueKeyFieldName); // this should return null if name is null
}
/**
@@ -180,7 +181,7 @@
* @return null if this schema has no unique key field
*/
public String printableUniqueKey(org.apache.lucene.document.Document doc) {
- Field f = doc.getField(uniqueKeyFieldName);
+ Fieldable f = doc.getFieldable(uniqueKeyFieldName);
return f==null ? null : uniqueKeyFieldType.toExternal(f);
}
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/IntField.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/IntField.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/IntField.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/IntField.java Mon Nov 27 14:40:21 2006
@@ -20,7 +20,6 @@
import org.apache.lucene.search.SortField;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.IntFieldSource;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
@@ -44,7 +43,7 @@
return new IntFieldSource(field.name);
}
- public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+ public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeInt(name, f.stringValue());
}
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/LongField.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/LongField.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/LongField.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/LongField.java Mon Nov 27 14:40:21 2006
@@ -20,7 +20,6 @@
import org.apache.lucene.search.SortField;
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.IntFieldSource;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
@@ -49,7 +48,7 @@
}
- public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+ public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeLong(name, f.stringValue());
}
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/SchemaField.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/SchemaField.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/SchemaField.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/SchemaField.java Mon Nov 27 14:40:21 2006
@@ -93,7 +93,7 @@
+ "}";
}
- public void write(XMLWriter writer, String name, Field val) throws IOException {
+ public void write(XMLWriter writer, String name, Fieldable val) throws IOException {
// name is passed in because it may be null if name should not be used.
type.write(writer,name,val);
}
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/SortableDoubleField.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/SortableDoubleField.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/SortableDoubleField.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/SortableDoubleField.java Mon Nov 27 14:40:21 2006
@@ -22,7 +22,6 @@
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.FieldCacheSource;
import org.apache.solr.search.function.DocValues;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.util.NumberUtils;
@@ -51,7 +50,7 @@
return NumberUtils.double2sortableStr(val);
}
- public String toExternal(Field f) {
+ public String toExternal(Fieldable f) {
return indexedToReadable(f.stringValue());
}
@@ -59,7 +58,7 @@
return NumberUtils.SortableStr2doubleStr(indexedForm);
}
- public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+ public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
String sval = f.stringValue();
xmlWriter.writeDouble(name, NumberUtils.SortableStr2double(sval));
}
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/SortableFloatField.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/SortableFloatField.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/SortableFloatField.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/SortableFloatField.java Mon Nov 27 14:40:21 2006
@@ -22,7 +22,6 @@
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.FieldCacheSource;
import org.apache.solr.search.function.DocValues;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.util.NumberUtils;
@@ -51,7 +50,7 @@
return NumberUtils.float2sortableStr(val);
}
- public String toExternal(Field f) {
+ public String toExternal(Fieldable f) {
return indexedToReadable(f.stringValue());
}
@@ -59,7 +58,7 @@
return NumberUtils.SortableStr2floatStr(indexedForm);
}
- public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+ public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
String sval = f.stringValue();
xmlWriter.writeFloat(name, NumberUtils.SortableStr2float(sval));
}
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/SortableIntField.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/SortableIntField.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/SortableIntField.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/SortableIntField.java Mon Nov 27 14:40:21 2006
@@ -22,7 +22,6 @@
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.FieldCacheSource;
import org.apache.solr.search.function.DocValues;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.util.NumberUtils;
@@ -54,7 +53,7 @@
return NumberUtils.int2sortableStr(val);
}
- public String toExternal(Field f) {
+ public String toExternal(Fieldable f) {
return indexedToReadable(f.stringValue());
}
@@ -62,7 +61,7 @@
return NumberUtils.SortableStr2int(indexedForm);
}
- public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+ public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
String sval = f.stringValue();
// since writeInt an int instead of a String since that may be more efficient
// in the future (saves the construction of one String)
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/SortableLongField.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/SortableLongField.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/SortableLongField.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/SortableLongField.java Mon Nov 27 14:40:21 2006
@@ -22,7 +22,6 @@
import org.apache.solr.search.function.ValueSource;
import org.apache.solr.search.function.FieldCacheSource;
import org.apache.solr.search.function.DocValues;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.IndexReader;
import org.apache.solr.util.NumberUtils;
@@ -55,11 +54,11 @@
return NumberUtils.SortableStr2long(indexedForm);
}
- public String toExternal(Field f) {
+ public String toExternal(Fieldable f) {
return indexedToReadable(f.stringValue());
}
- public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+ public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
String sval = f.stringValue();
xmlWriter.writeLong(name, NumberUtils.SortableStr2long(sval,0,sval.length()));
}
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/StrField.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/StrField.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/StrField.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/StrField.java Mon Nov 27 14:40:21 2006
@@ -18,7 +18,6 @@
package org.apache.solr.schema;
import org.apache.lucene.search.SortField;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
@@ -38,7 +37,7 @@
return getStringSort(field,reverse);
}
- public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+ public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeStr(name, f.stringValue());
}
Modified: incubator/solr/trunk/src/java/org/apache/solr/schema/TextField.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/schema/TextField.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/schema/TextField.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/schema/TextField.java Mon Nov 27 14:40:21 2006
@@ -18,7 +18,6 @@
package org.apache.solr.schema;
import org.apache.lucene.search.SortField;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.solr.request.XMLWriter;
import org.apache.solr.request.TextResponseWriter;
@@ -41,7 +40,7 @@
return new SortField(field.name,SortField.STRING, reverse);
}
- public void write(XMLWriter xmlWriter, String name, Field f) throws IOException {
+ public void write(XMLWriter xmlWriter, String name, Fieldable f) throws IOException {
xmlWriter.writeStr(name, f.stringValue());
}
Modified: incubator/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/search/SolrIndexSearcher.java Mon Nov 27 14:40:21 2006
@@ -17,7 +17,7 @@
package org.apache.solr.search;
-import org.apache.lucene.document.Document;
+import org.apache.lucene.document.*;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermDocs;
@@ -29,6 +29,7 @@
import org.apache.solr.core.SolrInfoMBean;
import org.apache.solr.core.SolrInfoRegistry;
import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
import org.apache.solr.util.NamedList;
import org.apache.solr.util.OpenBitSet;
@@ -299,22 +300,90 @@
return searcher.docFreq(term);
}
+ /* ********************** Document retrieval *************************/
+
+ /* Future optimizations (yonik)
+ *
+ * If no cache is present:
+ * - use NO_LOAD instead of LAZY_LOAD
+ * - use LOAD_AND_BREAK if a single field is begin retrieved
+ */
+
+ /**
+ * FieldSelector which loads the specified fields, and load all other
+ * field lazily.
+ */
+ class SetNonLazyFieldSelector implements FieldSelector {
+ private Set<String> fieldsToLoad;
+ SetNonLazyFieldSelector(Set<String> toLoad) {
+ fieldsToLoad = toLoad;
+ }
+ public FieldSelectorResult accept(String fieldName) {
+ if(fieldsToLoad.contains(fieldName))
+ return FieldSelectorResult.LOAD;
+ else
+ return FieldSelectorResult.LAZY_LOAD;
+ }
+ }
+
+ /* solrconfig lazyfields setting */
+ public static final boolean enableLazyFieldLoading = SolrConfig.config.getBool("query/enableLazyFieldLoading", false);
+
+ /**
+ * Retrieve the {@link Document} instance corresponding to the document id.
+ */
public Document doc(int i) throws IOException {
+ return doc(i, null);
+ }
+ /**
+ * Retrieve the {@link Document} instance corresponding to the document id.
+ *
+ * Note: The document will have all fields accessable, but if a field
+ * filter is provided, only the provided fields will be loaded (the
+ * remainder will be available lazily).
+ */
+ public Document doc(int i, Set<String> fields) throws IOException {
+
Document d;
if (documentCache != null) {
d = (Document)documentCache.get(i);
if (d!=null) return d;
}
- d = searcher.doc(i);
+ if(!enableLazyFieldLoading || fields == null) {
+ d = searcher.getIndexReader().document(i);
+ } else {
+ d = searcher.getIndexReader().document(i,
+ new SetNonLazyFieldSelector(fields));
+ }
if (documentCache != null) {
- documentCache.put(i,d);
+ documentCache.put(i, d);
}
return d;
}
+ /**
+ * Takes a list of docs (the doc ids actually), and reads them into an array
+ * of Documents.
+ */
+ public void readDocs(Document[] docs, DocList ids) throws IOException {
+ readDocs(docs, ids, null);
+ }
+ /**
+ * Takes a list of docs (the doc ids actually) and a set of fields to load,
+ * and reads them into an array of Documents.
+ */
+ public void readDocs(Document[] docs, DocList ids, Set<String> fields) throws IOException {
+ DocIterator iter = ids.iterator();
+ for (int i=0; i<docs.length; i++) {
+ docs[i] = doc(iter.nextDoc(), fields);
+ }
+ }
+
+ /* ********************** end document retrieval *************************/
+
public int maxDoc() throws IOException {
return searcher.maxDoc();
}
@@ -1234,17 +1303,6 @@
Document[] docs = new Document[ids.size()];
readDocs(docs,ids);
return docs;
- }
-
- /**
- * Takes a list of docs (the doc ids actually), and reads them into an array
- * of Documents.
- */
- public void readDocs(Document[] docs, DocList ids) throws IOException {
- DocIterator iter = ids.iterator();
- for (int i=0; i<docs.length; i++) {
- docs[i] = doc(iter.nextDoc());
- }
}
Modified: incubator/solr/trunk/src/java/org/apache/solr/util/HighlightingUtils.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/util/HighlightingUtils.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/util/HighlightingUtils.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/util/HighlightingUtils.java Mon Nov 27 14:40:21 2006
@@ -20,12 +20,15 @@
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import org.apache.solr.request.*;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.schema.SchemaField;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Document;
@@ -215,13 +218,24 @@
SolrIndexSearcher searcher = req.getSearcher();
NamedList fragments = new NamedList();
String[] fieldNames = getHighlightFields(query, req, defaultFields);
+ Document[] readDocs = new Document[docs.size()];
+ {
+ // pre-fetch documents using the Searcher's doc cache
+ Set<String> fset = new HashSet<String>();
+ for(String f : fieldNames) { fset.add(f); }
+ // fetch unique key if one exists.
+ SchemaField keyField = req.getSearcher().getSchema().getUniqueKeyField();
+ if(null != keyField)
+ fset.add(keyField.getName());
+ searcher.readDocs(readDocs, docs, fset);
+ }
+ // Highlight each document
DocIterator iterator = docs.iterator();
for (int i = 0; i < docs.size(); i++)
{
int docId = iterator.nextDoc();
- // use the Searcher's doc cache
- Document doc = searcher.doc(docId);
+ Document doc = readDocs[i];
NamedList docSummaries = new NamedList();
for (String fieldName : fieldNames)
{
Modified: incubator/solr/trunk/src/java/org/apache/solr/util/SolrPluginUtils.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/java/org/apache/solr/util/SolrPluginUtils.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/java/org/apache/solr/util/SolrPluginUtils.java (original)
+++ incubator/solr/trunk/src/java/org/apache/solr/util/SolrPluginUtils.java Mon Nov 27 14:40:21 2006
@@ -35,6 +35,7 @@
import org.apache.solr.request.DefaultSolrParams;
import org.apache.solr.request.AppendedSolrParams;
import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.*;
import java.io.IOException;
@@ -206,6 +207,57 @@
}
/**
+ * Pre-fetch documents into the index searcher's document cache.
+ *
+ * This is an entirely optional step which you might want to perform for
+ * the following reasons:
+ *
+ * <ul>
+ * <li>Locates the document-retrieval costs in one spot, which helps
+ * detailed performance measurement</li>
+ *
+ * <li>Determines a priori what fields will be needed to be fetched by
+ * various subtasks, like response writing and highlighting. This
+ * minimizes the chance that many needed fields will be loaded lazily.
+ * (it is more efficient to load all the field we require normally).</li>
+ * </ul>
+ *
+ * If lazy field loading is disabled, this method does nothing.
+ */
+ public static void optimizePreFetchDocs(DocList docs,
+ Query query,
+ SolrQueryRequest req,
+ SolrQueryResponse res) throws IOException {
+ SolrIndexSearcher searcher = req.getSearcher();
+ if(!searcher.enableLazyFieldLoading) {
+ // nothing to do
+ return;
+ }
+
+ Set<String> fieldFilter = null;
+ Set<String> returnFields = res.getReturnFields();
+ if(returnFields != null) {
+ // copy return fields list
+ fieldFilter = new HashSet<String>(returnFields);
+ // add highlight fields
+ if(HighlightingUtils.isHighlightingEnabled(req)) {
+ for(String field: HighlightingUtils.getHighlightFields(query, req, null))
+ fieldFilter.add(field);
+ }
+ // fetch unique key if one exists.
+ SchemaField keyField = req.getSearcher().getSchema().getUniqueKeyField();
+ if(null != keyField)
+ fieldFilter.add(keyField.getName());
+ }
+
+ // get documents
+ DocIterator iter = docs.iterator();
+ for (int i=0; i<docs.size(); i++) {
+ searcher.doc(iter.nextDoc(), fieldFilter);
+ }
+ }
+
+ /**
* <p>
* Returns a NamedList containing many "standard" pieces of debugging
* information.
@@ -794,6 +846,7 @@
}
}
+
}
/**
Modified: incubator/solr/trunk/src/test/org/apache/solr/BasicFunctionalityTest.java
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/test/org/apache/solr/BasicFunctionalityTest.java?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/test/org/apache/solr/BasicFunctionalityTest.java (original)
+++ incubator/solr/trunk/src/test/org/apache/solr/BasicFunctionalityTest.java Mon Nov 27 14:40:21 2006
@@ -17,9 +17,10 @@
package org.apache.solr;
-import org.apache.lucene.document.Field;
+import org.apache.lucene.document.*;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.BooleanQuery;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.search.*;
import org.apache.solr.request.*;
import org.apache.solr.util.*;
@@ -567,6 +568,48 @@
assertTrue(luf.isStored());
}
+
+ public void testNotLazyField() throws IOException {
+ for(int i = 0; i < 10; i++) {
+ assertU(adoc("id", new Integer(i).toString(),
+ "title", "keyword",
+ "test_hlt", mkstr(20000)));
+ }
+ assertU(commit());
+ SolrCore core = h.getCore();
+
+ SolrQueryRequest req = req("q", "title:keyword", "fl", "id,title,test_hlt");
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ core.execute(req, rsp);
+
+ DocList dl = (DocList) rsp.getValues().get(null);
+ org.apache.lucene.document.Document d = req.getSearcher().doc(dl.iterator().nextDoc());
+ // ensure field is not lazy
+ assertTrue( d.getFieldable("test_hlt") instanceof Field );
+ assertTrue( d.getFieldable("title") instanceof Field );
+ }
+
+ public void testLazyField() throws IOException {
+ for(int i = 0; i < 10; i++) {
+ assertU(adoc("id", new Integer(i).toString(),
+ "title", "keyword",
+ "test_hlt", mkstr(20000)));
+ }
+ assertU(commit());
+ SolrCore core = h.getCore();
+
+ SolrQueryRequest req = req("q", "title:keyword", "fl", "id,title");
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ core.execute(req, rsp);
+
+ DocList dl = (DocList) rsp.getValues().get(null);
+ DocIterator di = dl.iterator();
+ org.apache.lucene.document.Document d = req.getSearcher().doc(di.nextDoc());
+ // ensure field is lazy
+ assertTrue( !( d.getFieldable("test_hlt") instanceof Field ) );
+ assertTrue( d.getFieldable("title") instanceof Field );
+ }
+
/** @see org.apache.solr.util.DateMathParserTest */
public void testDateMath() {
Modified: incubator/solr/trunk/src/test/test-files/solr/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/incubator/solr/trunk/src/test/test-files/solr/conf/solrconfig.xml?view=diff&rev=479793&r1=479792&r2=479793
==============================================================================
--- incubator/solr/trunk/src/test/test-files/solr/conf/solrconfig.xml (original)
+++ incubator/solr/trunk/src/test/test-files/solr/conf/solrconfig.xml Mon Nov 27 14:40:21 2006
@@ -106,7 +106,12 @@
initialSize="512"
autowarmCount="0"/>
+ <!-- If true, stored fields that are not requested will be loaded lazily.
+ -->
+ <enableLazyFieldLoading>true</enableLazyFieldLoading>
+
<!--
+
<cache name="myUserCache"
class="solr.search.LRUCache"
size="4096"