You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/03/30 11:17:42 UTC
svn commit: r1086876 [16/18] - in /lucene/dev/branches/realtime_search: ./
dev-tools/eclipse/ dev-tools/idea/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/ dev-tools/idea/solr/
dev-tools/idea/solr/contrib/analysis-extras/ dev-tools/idea/solr/c...
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/CSVResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/CSVResponseWriter.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/CSVResponseWriter.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/CSVResponseWriter.java Wed Mar 30 09:17:25 2011
@@ -34,6 +34,7 @@ import org.apache.solr.schema.SchemaFiel
import org.apache.solr.schema.StrField;
import org.apache.solr.search.DocIterator;
import org.apache.solr.search.DocList;
+import org.apache.solr.search.ReturnFields;
import org.apache.solr.search.SolrIndexSearcher;
import java.io.CharArrayWriter;
@@ -159,7 +160,6 @@ class CSVWriter extends TextResponseWrit
ResettableFastWriter mvWriter = new ResettableFastWriter(); // writer used for multi-valued fields
String NullValue;
- boolean returnScore = false;
public CSVWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) {
@@ -236,12 +236,9 @@ class CSVWriter extends TextResponseWrit
// encapsulator will already be disabled if it wasn't specified
}
- returnScore = returnFields != null && returnFields.contains("score");
- boolean needListOfFields = returnFields==null || returnFields.size()==0 || (returnFields.size()==1 && returnScore) || returnFields.contains("*");
- Collection<String> fields = returnFields;
-
+ Collection<String> fields = returnFields.getLuceneFieldNames();
Object responseObj = rsp.getValues().get("response");
- if (needListOfFields) {
+ if (fields==null) {
if (responseObj instanceof SolrDocumentList) {
// get the list of fields from the SolrDocumentList
fields = new LinkedHashSet<String>();
@@ -252,7 +249,7 @@ class CSVWriter extends TextResponseWrit
// get the list of fields from the index
fields = req.getSearcher().getFieldNames();
}
- if (returnScore) {
+ if (returnFields.wantsScore()) {
fields.add("score");
} else {
fields.remove("score");
@@ -327,11 +324,15 @@ class CSVWriter extends TextResponseWrit
printer.println();
}
-
- if (responseObj instanceof DocList) {
- writeDocList(null, (DocList)responseObj, null, null);
+ if (responseObj instanceof ResultContext ) {
+ writeDocuments(null, (ResultContext)responseObj, returnFields );
+ }
+ else if (responseObj instanceof DocList) {
+ ResultContext ctx = new ResultContext();
+ ctx.docs = (DocList)responseObj;
+ writeDocuments(null, ctx, returnFields );
} else if (responseObj instanceof SolrDocumentList) {
- writeSolrDocumentList(null, (SolrDocumentList)responseObj, null, null);
+ writeSolrDocumentList(null, (SolrDocumentList)responseObj, returnFields );
}
}
@@ -346,56 +347,21 @@ class CSVWriter extends TextResponseWrit
public void writeNamedList(String name, NamedList val) throws IOException {
}
- @Override
- public void writeDoc(String name, Document doc, Set<String> returnFields, float score, boolean includeScore) throws IOException {
- pass++;
-
- for (Fieldable field: doc.getFields()) {
- CSVField csvField = csvFields.get(field.name());
- if (csvField == null) continue;
- if (csvField.tmp != pass) {
- csvField.tmp = pass;
- csvField.values.clear();
- }
- csvField.values.add(field);
- }
-
- for (CSVField csvField : csvFields.values()) {
- if (csvField.name.equals("score")) {
- writeFloat("score", score);
- continue;
- }
- if (csvField.tmp != pass) {
- writeNull(csvField.name);
- continue;
- }
-
- if (csvField.sf.multiValued() || csvField.values.size() > 1) {
- mvWriter.reset();
- csvField.mvPrinter.reset();
- // switch the printer to use the multi-valued one
- CSVPrinter tmp = printer;
- printer = csvField.mvPrinter;
- for (Fieldable fval : csvField.values) {
- csvField.sf.getType().write(this, csvField.name, fval);
- }
- printer = tmp; // restore the original printer
-
- mvWriter.freeze();
- printer.print(mvWriter.getFrozenBuf(), 0, mvWriter.getFrozenSize(), true);
- } else {
- assert csvField.values.size() == 1;
- csvField.sf.getType().write(this,csvField.name,csvField.values.get(0));
- }
- }
+ public void writeStartDocumentList(String name,
+ long start, int size, long numFound, Float maxScore) throws IOException
+ {
+ // nothing
+ }
- printer.println();
+ public void writeEndDocumentList() throws IOException
+ {
+ // nothing
}
//NOTE: a document cannot currently contain another document
List tmpList;
@Override
- public void writeSolrDocument(String name, SolrDocument doc, Set<String> returnFields, Map pseudoFields) throws IOException {
+ public void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx ) throws IOException {
if (tmpList == null) {
tmpList = new ArrayList(1);
tmpList.add(null);
@@ -446,26 +412,6 @@ class CSVWriter extends TextResponseWrit
}
@Override
- public void writeDocList(String name, DocList ids, Set<String> fields, Map otherFields) throws IOException {
- int sz=ids.size();
- SolrIndexSearcher searcher = req.getSearcher();
- DocIterator iterator = ids.iterator();
- for (int i=0; i<sz; i++) {
- int id = iterator.nextDoc();
- Document doc = searcher.doc(id, fields);
- writeDoc(null, doc, fields, (returnScore ? iterator.score() : 0.0f), returnScore);
- }
- }
-
- Map scoreMap = new HashMap(1);
- @Override
- public void writeSolrDocumentList(String name, SolrDocumentList docs, Set<String> fields, Map otherFields) throws IOException {
- for (SolrDocument doc : docs) {
- writeSolrDocument(name, doc, fields, otherFields);
- }
- }
-
- @Override
public void writeStr(String name, String val, boolean needsEscaping) throws IOException {
printer.print(val, needsEscaping);
}
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/JSONResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/JSONResponseWriter.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/JSONResponseWriter.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/JSONResponseWriter.java Wed Mar 30 09:17:25 2011
@@ -17,24 +17,24 @@
package org.apache.solr.response;
-import org.apache.lucene.document.Document;
+import java.io.IOException;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.util.StringHelper;
import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrDocumentList;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.SchemaField;
-import org.apache.solr.schema.TextField;
-import org.apache.solr.search.DocIterator;
-import org.apache.solr.search.DocList;
-import org.apache.solr.search.SolrIndexSearcher;
-
-import java.io.IOException;
-import java.io.Writer;
-import java.util.*;
+import org.apache.solr.search.ReturnFields;
/**
* @version $Id$
@@ -73,7 +73,6 @@ class JSONWriter extends TextResponseWri
private static final String JSON_NL_ARROFMAP="arrmap";
private static final String JSON_WRAPPER_FUNCTION="json.wrf";
-
public JSONWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) {
super(writer, req, rsp);
namedListStyle = StringHelper.intern(req.getParams().get(JSON_NL_STYLE, JSON_NL_FLAT));
@@ -312,94 +311,21 @@ class JSONWriter extends TextResponseWri
}
}
- public void writeDoc(String name, Collection<Fieldable> fields, Set<String> returnFields, Map pseudoFields) throws IOException {
- writeMapOpener(-1); // no trivial way to determine map size
- incLevel();
-
- HashMap<String, MultiValueField> multi = new HashMap<String, MultiValueField>();
-
- boolean first=true;
-
- for (Fieldable ff : fields) {
- String fname = ff.name();
- if (returnFields!=null && !returnFields.contains(fname)) {
- continue;
- }
-
- // if the field is multivalued, it may have other values further on... so
- // build up a list for each multi-valued field.
- SchemaField sf = schema.getField(fname);
- if (sf.multiValued()) {
- MultiValueField mf = multi.get(fname);
- if (mf==null) {
- mf = new MultiValueField(sf, ff);
- multi.put(fname, mf);
- } else {
- mf.fields.add(ff);
- }
- } else {
- // not multi-valued, so write it immediately.
- if (first) {
- first=false;
- } else {
- writeMapSeparator();
- }
- indent();
- writeKey(fname,true);
- sf.write(this, fname, ff);
- }
- }
-
- for(MultiValueField mvf : multi.values()) {
- if (first) {
- first=false;
- } else {
- writeMapSeparator();
- }
-
- indent();
- writeKey(mvf.sfield.getName(), true);
-
- boolean indentArrElems=false;
- if (doIndent) {
- // heuristic... TextField is probably the only field type likely to be long enough
- // to warrant indenting individual values.
- indentArrElems = (mvf.sfield.getType() instanceof TextField);
- }
-
- writeArrayOpener(-1); // no trivial way to determine array size
- boolean firstArrElem=true;
- incLevel();
-
- for (Fieldable ff : mvf.fields) {
- if (firstArrElem) {
- firstArrElem=false;
- } else {
- writeArraySeparator();
- }
- if (indentArrElems) indent();
- mvf.sfield.write(this, null, ff);
- }
- writeArrayCloser();
- decLevel();
- }
-
- if (pseudoFields !=null && pseudoFields.size()>0) {
- writeMap(null,pseudoFields,true,first);
- }
-
- decLevel();
- writeMapCloser();
- }
-
@Override
- public void writeSolrDocument(String name, SolrDocument doc, Set<String> returnFields, Map pseudoFields) throws IOException {
- writeMapOpener(-1); // no trivial way to determine map size
- // TODO: could easily figure out size for SolrDocument if needed...
+ public void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx) throws IOException {
+ if( idx > 0 ) {
+ writeArraySeparator();
+ }
+
+ writeMapOpener(doc.size());
incLevel();
boolean first=true;
for (String fname : doc.getFieldNames()) {
+ if (!returnFields.wantsField(fname)) {
+ continue;
+ }
+
if (first) {
first=false;
}
@@ -424,144 +350,43 @@ class JSONWriter extends TextResponseWri
writeVal(fname, val);
}
}
-
- if (pseudoFields !=null && pseudoFields.size()>0) {
- writeMap(null,pseudoFields,true,first);
- }
}
-
+
decLevel();
writeMapCloser();
}
-
- // reusable map to store the "score" pseudo-field.
- // if a Doc can ever contain another doc, this optimization would have to go.
- private final HashMap scoreMap = new HashMap(1);
-
- @Override
- public void writeDoc(String name, Document doc, Set<String> returnFields, float score, boolean includeScore) throws IOException {
- Map other = null;
- if (includeScore) {
- other = scoreMap;
- scoreMap.put("score",score);
- }
- writeDoc(name, doc.getFields(), returnFields, other);
- }
-
@Override
- public void writeDocList(String name, DocList ids, Set<String> fields, Map otherFields) throws IOException {
- boolean includeScore=false;
- if (fields!=null) {
- includeScore = fields.contains("score");
- if (fields.size()==0 || (fields.size()==1 && includeScore) || fields.contains("*")) {
- fields=null; // null means return all stored fields
- }
- }
-
- int sz=ids.size();
-
- writeMapOpener(includeScore ? 4 : 3);
+ public void writeStartDocumentList(String name,
+ long start, int size, long numFound, Float maxScore) throws IOException
+ {
+ writeMapOpener((maxScore==null) ? 3 : 4);
incLevel();
writeKey("numFound",false);
- writeInt(null,ids.matches());
+ writeLong(null,numFound);
writeMapSeparator();
writeKey("start",false);
- writeInt(null,ids.offset());
+ writeLong(null,start);
- if (includeScore) {
+ if (maxScore!=null) {
writeMapSeparator();
writeKey("maxScore",false);
- writeFloat(null,ids.maxScore());
+ writeFloat(null,maxScore);
}
writeMapSeparator();
// indent();
writeKey("docs",false);
- writeArrayOpener(sz);
+ writeArrayOpener(size);
incLevel();
- boolean first=true;
-
- SolrIndexSearcher searcher = req.getSearcher();
- // be defensive... write out the doc even if we don't have the scores like we should
- includeScore = includeScore && ids.hasScores();
- DocIterator iterator = ids.iterator();
- for (int i=0; i<sz; i++) {
- int id = iterator.nextDoc();
- Document doc = searcher.doc(id, fields);
-
- if (first) {
- first=false;
- } else {
- writeArraySeparator();
- }
- indent();
- writeDoc(null, doc, fields, (includeScore ? iterator.score() : 0.0f), includeScore);
- }
- decLevel();
- writeArrayCloser();
-
- if (otherFields !=null) {
- writeMap(null, otherFields, true, false);
- }
-
- decLevel();
- indent();
- writeMapCloser();
}
-
@Override
- public void writeSolrDocumentList(String name, SolrDocumentList docs, Set<String> fields, Map otherFields) throws IOException {
- boolean includeScore=false;
- if (fields!=null) {
- includeScore = fields.contains("score");
- if (fields.size()==0 || (fields.size()==1 && includeScore) || fields.contains("*")) {
- fields=null; // null means return all stored fields
- }
- }
-
- int sz=docs.size();
-
- writeMapOpener(includeScore ? 4 : 3);
- incLevel();
- writeKey("numFound",false);
- writeLong(null,docs.getNumFound());
- writeMapSeparator();
- writeKey("start",false);
- writeLong(null,docs.getStart());
-
- if (includeScore && docs.getMaxScore() != null) {
- writeMapSeparator();
- writeKey("maxScore",false);
- writeFloat(null,docs.getMaxScore());
- }
- writeMapSeparator();
- // indent();
- writeKey("docs",false);
- writeArrayOpener(sz);
-
- incLevel();
- boolean first=true;
-
- SolrIndexSearcher searcher = req.getSearcher();
- for (SolrDocument doc : docs) {
-
- if (first) {
- first=false;
- } else {
- writeArraySeparator();
- }
- indent();
- writeSolrDocument(null, doc, fields, otherFields);
- }
+ public void writeEndDocumentList() throws IOException
+ {
decLevel();
writeArrayCloser();
- if (otherFields !=null) {
- writeMap(null, otherFields, true, false);
- }
-
decLevel();
indent();
writeMapCloser();
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/PHPSerializedResponseWriter.java Wed Mar 30 09:17:25 2011
@@ -17,57 +17,35 @@
package org.apache.solr.response;
-import java.io.Writer;
import java.io.IOException;
-import java.util.*;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
-import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Fieldable;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.UnicodeUtil;
+import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.schema.SchemaField;
-import org.apache.solr.search.DocIterator;
-import org.apache.solr.search.DocList;
-import org.apache.solr.search.SolrIndexSearcher;
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.search.ReturnFields;
+
+
/**
* A description of the PHP serialization format can be found here:
* http://www.hurring.com/scott/code/perl/serialize/
- *
- * <p>
- * In order to support PHP Serialized strings with a proper byte count, This ResponseWriter
- * must know if the Writers passed to it will result in an output of CESU-8 (UTF-8 w/o support
- * for large code points outside of the BMP)
- * <p>
- * Currently Solr assumes that all Jetty servlet containers (detected using the "jetty.home"
- * system property) use CESU-8 instead of UTF-8 (verified to the current release of 6.1.20).
- * <p>
- * In installations where Solr auto-detects incorrectly, the Solr Administrator should set the
- * "solr.phps.cesu8" system property to either "true" or "false" accordingly.
*/
public class PHPSerializedResponseWriter implements QueryResponseWriter {
static String CONTENT_TYPE_PHP_UTF8="text/x-php-serialized;charset=UTF-8";
- // Is this servlet container's UTF-8 encoding actually CESU-8 (i.e. lacks support for
- // large characters outside the BMP).
- boolean CESU8 = false;
public void init(NamedList n) {
- String cesu8Setting = System.getProperty("solr.phps.cesu8");
- if (cesu8Setting != null) {
- CESU8="true".equals(cesu8Setting);
- } else {
- // guess at the setting.
- // Jetty up until 6.1.20 at least (and probably versions after) uses CESU8
- CESU8 = System.getProperty("jetty.home") != null;
- }
}
public void write(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
- PHPSerializedWriter w = new PHPSerializedWriter(writer, req, rsp, CESU8);
+ PHPSerializedWriter w = new PHPSerializedWriter(writer, req, rsp);
try {
w.writeResponse();
} finally {
@@ -81,13 +59,11 @@ public class PHPSerializedResponseWriter
}
class PHPSerializedWriter extends JSONWriter {
- final private boolean CESU8;
final BytesRef utf8;
- public PHPSerializedWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp, boolean CESU8) {
+ public PHPSerializedWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) {
super(writer, req, rsp);
- this.CESU8 = CESU8;
- this.utf8 = CESU8 ? null : new BytesRef();
+ this.utf8 = new BytesRef();
// never indent serialized PHP data
doIndent = false;
}
@@ -104,126 +80,53 @@ class PHPSerializedWriter extends JSONWr
writeNamedListAsMapMangled(name,val);
}
- @Override
- public void writeDoc(String name, Collection<Fieldable> fields, Set<String> returnFields, Map pseudoFields) throws IOException {
- ArrayList<Fieldable> single = new ArrayList<Fieldable>();
- LinkedHashMap<String, MultiValueField> multi
- = new LinkedHashMap<String, MultiValueField>();
-
- for (Fieldable ff : fields) {
- String fname = ff.name();
- if (returnFields!=null && !returnFields.contains(fname)) {
- continue;
- }
- // if the field is multivalued, it may have other values further on... so
- // build up a list for each multi-valued field.
- SchemaField sf = schema.getField(fname);
- if (sf.multiValued()) {
- MultiValueField mf = multi.get(fname);
- if (mf==null) {
- mf = new MultiValueField(sf, ff);
- multi.put(fname, mf);
- } else {
- mf.fields.add(ff);
- }
- } else {
- single.add(ff);
- }
- }
-
- // obtain number of fields in doc
- writeArrayOpener(single.size() + multi.size() + ((pseudoFields!=null) ? pseudoFields.size() : 0));
-
- // output single value fields
- for(Fieldable ff : single) {
- SchemaField sf = schema.getField(ff.name());
- writeKey(ff.name(),true);
- sf.write(this, ff.name(), ff);
- }
-
- // output multi value fields
- for(MultiValueField mvf : multi.values()) {
- writeKey(mvf.sfield.getName(), true);
- writeArrayOpener(mvf.fields.size());
- int i = 0;
- for (Fieldable ff : mvf.fields) {
- writeKey(i++, false);
- mvf.sfield.write(this, null, ff);
- }
- writeArrayCloser();
- }
-
- // output pseudo fields
- if (pseudoFields !=null && pseudoFields.size()>0) {
- writeMap(null,pseudoFields,true,false);
- }
- writeArrayCloser();
- }
- @Override
- public void writeDocList(String name, DocList ids, Set<String> fields, Map otherFields) throws IOException {
- boolean includeScore=false;
-
- if (fields!=null) {
- includeScore = fields.contains("score");
- if (fields.size()==0 || (fields.size()==1 && includeScore) || fields.contains("*")) {
- fields=null; // null means return all stored fields
- }
- }
-
- int sz=ids.size();
- writeMapOpener(includeScore ? 4 : 3);
+ public void writeStartDocumentList(String name,
+ long start, int size, long numFound, Float maxScore) throws IOException
+ {
+ writeMapOpener((maxScore==null) ? 3 : 4);
writeKey("numFound",false);
- writeInt(null,ids.matches());
+ writeLong(null,numFound);
writeKey("start",false);
- writeInt(null,ids.offset());
+ writeLong(null,start);
- if (includeScore) {
+ if (maxScore!=null) {
writeKey("maxScore",false);
- writeFloat(null,ids.maxScore());
+ writeFloat(null,maxScore);
}
writeKey("docs",false);
- writeArrayOpener(sz);
-
- SolrIndexSearcher searcher = req.getSearcher();
- DocIterator iterator = ids.iterator();
- for (int i=0; i<sz; i++) {
- int id = iterator.nextDoc();
- Document doc = searcher.doc(id, fields);
- writeKey(i, false);
- writeDoc(null, doc, fields, (includeScore ? iterator.score() : 0.0f), includeScore);
- }
- writeMapCloser();
-
- if (otherFields !=null) {
- writeMap(null, otherFields, true, false);
- }
+ writeArrayOpener(size);
+ }
+ public void writeEndDocumentList() throws IOException
+ {
+ writeArrayCloser(); // doc list
writeMapCloser();
}
@Override
- public void writeSolrDocument(String name, SolrDocument doc, Set<String> returnFields, Map pseudoFields) throws IOException {
+ public void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx) throws IOException
+ {
+ writeKey(idx, false);
+
LinkedHashMap <String,Object> single = new LinkedHashMap<String, Object>();
LinkedHashMap <String,Object> multi = new LinkedHashMap<String, Object>();
- int pseudoSize = pseudoFields != null ? pseudoFields.size() : 0;
for (String fname : doc.getFieldNames()) {
- if(returnFields != null && !returnFields.contains(fname)){
+ if(!returnFields.wantsField(fname)){
continue;
}
Object val = doc.getFieldValue(fname);
- SchemaField sf = schema.getFieldOrNull(fname);
- if (sf != null && sf.multiValued()) {
+ if (val instanceof Collection) {
multi.put(fname, val);
}else{
single.put(fname, val);
}
}
- writeMapOpener(single.size() + multi.size() + pseudoSize);
+ writeMapOpener(single.size() + multi.size());
for(String fname: single.keySet()){
Object val = single.get(fname);
writeKey(fname, true);
@@ -244,51 +147,7 @@ class PHPSerializedWriter extends JSONWr
writeVal(fname, val);
}
}
-
- if (pseudoSize > 0) {
- writeMap(null,pseudoFields,true, false);
- }
- writeMapCloser();
- }
-
-
- @Override
- public void writeSolrDocumentList(String name, SolrDocumentList docs, Set<String> fields, Map otherFields) throws IOException {
- boolean includeScore=false;
- if (fields!=null) {
- includeScore = fields.contains("score");
- if (fields.size()==0 || (fields.size()==1 && includeScore) || fields.contains("*")) {
- fields=null; // null means return all stored fields
- }
- }
-
- int sz = docs.size();
-
- writeMapOpener(includeScore ? 4 : 3);
-
- writeKey("numFound",false);
- writeLong(null,docs.getNumFound());
-
- writeKey("start",false);
- writeLong(null,docs.getStart());
-
- if (includeScore && docs.getMaxScore() != null) {
- writeKey("maxScore",false);
- writeFloat(null,docs.getMaxScore());
- }
-
- writeKey("docs",false);
-
- writeArrayOpener(sz);
- for (int i=0; i<sz; i++) {
- writeKey(i, false);
- writeSolrDocument(null, docs.get(i), fields, otherFields);
- }
- writeArrayCloser();
-
- if (otherFields !=null) {
- writeMap(null, otherFields, true, false);
- }
+
writeMapCloser();
}
@@ -397,23 +256,8 @@ class PHPSerializedWriter extends JSONWr
public void writeStr(String name, String val, boolean needsEscaping) throws IOException {
// serialized PHP strings don't need to be escaped at all, however the
// string size reported needs be the number of bytes rather than chars.
- int nBytes;
- if (CESU8) {
- nBytes = 0;
- for (int i=0; i<val.length(); i++) {
- char ch = val.charAt(i);
- if (ch<='\u007f') {
- nBytes += 1;
- } else if (ch<='\u07ff') {
- nBytes += 2;
- } else {
- nBytes += 3;
- }
- }
- } else {
- UnicodeUtil.UTF16toUTF8(val, 0, val.length(), utf8);
- nBytes = utf8.length;
- }
+ UnicodeUtil.UTF16toUTF8(val, 0, val.length(), utf8);
+ int nBytes = utf8.length;
writer.write("s:");
writer.write(Integer.toString(nBytes));
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/QueryResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/QueryResponseWriter.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/QueryResponseWriter.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/QueryResponseWriter.java Wed Mar 30 09:17:25 2011
@@ -43,7 +43,7 @@ import org.apache.solr.util.plugin.Named
* @version $Id$
*/
public interface QueryResponseWriter extends NamedListInitializedPlugin {
- public static String CONTENT_TYPE_XML_UTF8="text/xml; charset=UTF-8";
+ public static String CONTENT_TYPE_XML_UTF8="application/xml; charset=UTF-8";
public static String CONTENT_TYPE_TEXT_UTF8="text/plain; charset=UTF-8";
public static String CONTENT_TYPE_TEXT_ASCII="text/plain; charset=US-ASCII";
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/SolrQueryResponse.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/SolrQueryResponse.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/SolrQueryResponse.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/SolrQueryResponse.java Wed Mar 30 09:17:25 2011
@@ -19,6 +19,7 @@ package org.apache.solr.response;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.search.ReturnFields;
import java.util.*;
@@ -67,13 +68,14 @@ public class SolrQueryResponse {
* @see <a href="#returnable_data">Note on Returnable Data</a>
*/
protected NamedList<Object> values = new SimpleOrderedMap<Object>();
-
- /**
+
+
+/**
* Container for storing information that should be logged by Solr before returning.
*/
protected NamedList<Object> toLog = new SimpleOrderedMap<Object>();
- protected Set<String> defaultReturnFields;
+ protected ReturnFields returnFields;
// error if this is set...
protected Exception err;
@@ -111,18 +113,19 @@ public class SolrQueryResponse {
* Sets the document field names of fields to return by default when
* returning DocLists
*/
- public void setReturnFields(Set<String> fields) {
- defaultReturnFields=fields;
+ public void setReturnFields(ReturnFields fields) {
+ returnFields=fields;
}
- // TODO: should this be represented as a String[] such
- // that order can be maintained if needed?
/**
* Gets the document field names of fields to return by default when
* returning DocLists
*/
- public Set<String> getReturnFields() {
- return defaultReturnFields;
+ public ReturnFields getReturnFields() {
+ if( returnFields == null ) {
+ returnFields = new ReturnFields(); // by default return everything
+ }
+ return returnFields;
}
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/TextResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/TextResponseWriter.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/TextResponseWriter.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/TextResponseWriter.java Wed Mar 30 09:17:25 2011
@@ -17,17 +17,23 @@
package org.apache.solr.response;
+import java.io.IOException;
+import java.io.Writer;
+import java.util.*;
+
import org.apache.lucene.document.Document;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.FastWriter;
+import org.apache.lucene.document.Fieldable;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.util.FastWriter;
+import org.apache.solr.common.util.NamedList;
import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.transform.DocTransformer;
+import org.apache.solr.response.transform.TransformContext;
import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
import org.apache.solr.search.DocList;
-import java.io.IOException;
-import java.io.Writer;
-import java.util.*;
+import org.apache.solr.search.ReturnFields;
/** Base class for text-oriented response writers.
*
@@ -49,7 +55,7 @@ public abstract class TextResponseWriter
protected final SolrQueryResponse rsp;
// the default set of fields to return for each document
- protected Set<String> returnFields;
+ protected ReturnFields returnFields;
protected int level;
protected boolean doIndent;
@@ -103,7 +109,7 @@ public abstract class TextResponseWriter
public abstract void writeNamedList(String name, NamedList val) throws IOException;
- public void writeVal(String name, Object val) throws IOException {
+ public final void writeVal(String name, Object val) throws IOException {
// if there get to be enough types, perhaps hashing on the type
// to get a handler might be faster (but types must be exact to do that...)
@@ -114,6 +120,15 @@ public abstract class TextResponseWriter
} else if (val instanceof String) {
writeStr(name, val.toString(), true);
// micro-optimization... using toString() avoids a cast first
+ } else if (val instanceof Fieldable) {
+ Fieldable f = (Fieldable)val;
+ SchemaField sf = schema.getFieldOrNull( f.name() );
+ if( sf != null ) {
+ sf.getType().write(this, name, f);
+ }
+ else {
+ writeStr(name, f.stringValue(), true);
+ }
} else if (val instanceof Integer) {
writeInt(name, val.toString());
} else if (val instanceof Boolean) {
@@ -129,19 +144,25 @@ public abstract class TextResponseWriter
} else if (val instanceof Double) {
writeDouble(name, ((Double)val).doubleValue());
} else if (val instanceof Document) {
- writeDoc(name, (Document)val, returnFields, 0.0f, false);
+ SolrDocument doc = toSolrDocument( (Document)val );
+ writeSolrDocument(name, doc, returnFields, 0 );
} else if (val instanceof SolrDocument) {
- writeSolrDocument(name, (SolrDocument)val, returnFields, null);
- } else if (val instanceof DocList) {
+ writeSolrDocument(name, (SolrDocument)val, returnFields, 0);
+ } else if (val instanceof ResultContext) {
// requires access to IndexReader
- writeDocList(name, (DocList)val, returnFields,null);
+ writeDocuments(name, (ResultContext)val, returnFields);
+ } else if (val instanceof DocList) {
+ // Should not happen normally
+ ResultContext ctx = new ResultContext();
+ ctx.docs = (DocList)val;
+ writeDocuments(name, ctx, returnFields);
// }
// else if (val instanceof DocSet) {
// how do we know what fields to read?
// todo: have a DocList/DocSet wrapper that
// restricts the fields to write...?
} else if (val instanceof SolrDocumentList) {
- writeSolrDocumentList(name, (SolrDocumentList)val, returnFields, null);
+ writeSolrDocumentList(name, (SolrDocumentList)val, returnFields);
} else if (val instanceof Map) {
writeMap(name, (Map)val, false, true);
} else if (val instanceof NamedList) {
@@ -162,20 +183,82 @@ public abstract class TextResponseWriter
// types of formats, including those where the name may come after the value (like
// some XML formats).
- public abstract void writeDoc(String name, Document doc, Set<String> returnFields, float score, boolean includeScore) throws IOException;
+ public abstract void writeStartDocumentList(String name, long start, int size, long numFound, Float maxScore) throws IOException;
- /**
- * @since solr 1.3
- */
- public abstract void writeSolrDocument(String name, SolrDocument doc, Set<String> returnFields, Map pseudoFields) throws IOException;
-
- public abstract void writeDocList(String name, DocList ids, Set<String> fields, Map otherFields) throws IOException;
-
- /**
- * @since solr 1.3
- */
- public abstract void writeSolrDocumentList(String name, SolrDocumentList docs, Set<String> fields, Map otherFields) throws IOException;
+ public abstract void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx) throws IOException;
+
+ public abstract void writeEndDocumentList() throws IOException;
+
+ // Assume each SolrDocument is already transformed
+ public final void writeSolrDocumentList(String name, SolrDocumentList docs, ReturnFields returnFields) throws IOException
+ {
+ writeStartDocumentList(name, docs.getStart(), docs.size(), docs.getNumFound(), docs.getMaxScore() );
+ for( int i=0; i<docs.size(); i++ ) {
+ writeSolrDocument( null, docs.get(i), returnFields, i );
+ }
+ writeEndDocumentList();
+ }
+ public final SolrDocument toSolrDocument( Document doc )
+ {
+ SolrDocument out = new SolrDocument();
+ for( Fieldable f : doc.getFields() ) {
+ if( "gack_i".equals( f.name() ) ) {
+ System.out.println( f );
+ }
+
+ // Make sure multivalued fields are represented as lists
+ Object existing = out.get(f.name());
+ if (existing == null) {
+ SchemaField sf = schema.getFieldOrNull(f.name());
+ if (sf != null && sf.multiValued()) {
+ List<Object> vals = new ArrayList<Object>();
+ vals.add( f );
+ out.setField( f.name(), vals );
+ }
+ else{
+ out.setField( f.name(), f );
+ }
+ }
+ else {
+ out.addField( f.name(), f );
+ }
+ }
+ return out;
+ }
+
+ public final void writeDocuments(String name, ResultContext res, ReturnFields fields ) throws IOException {
+ DocList ids = res.docs;
+ TransformContext context = new TransformContext();
+ context.query = res.query;
+ context.wantsScores = fields.wantsScore() && ids.hasScores();
+ writeStartDocumentList(name, ids.offset(), ids.size(), ids.matches(),
+ context.wantsScores ? new Float(ids.maxScore()) : null );
+
+ DocTransformer transformer = fields.getTransformer();
+ context.searcher = req.getSearcher();
+ context.iterator = ids.iterator();
+ if( transformer != null ) {
+ transformer.setContext( context );
+ }
+ int sz = ids.size();
+ Set<String> fnames = fields.getLuceneFieldNames();
+ for (int i=0; i<sz; i++) {
+ int id = context.iterator.nextDoc();
+ Document doc = context.searcher.doc(id, fnames);
+ SolrDocument sdoc = toSolrDocument( doc );
+ if( transformer != null ) {
+ transformer.transform( sdoc, id );
+ }
+ writeSolrDocument( null, sdoc, returnFields, i );
+ }
+ if( transformer != null ) {
+ transformer.setContext( null );
+ }
+ writeEndDocumentList();
+ }
+
+
public abstract void writeStr(String name, String val, boolean needsEscaping) throws IOException;
public abstract void writeMap(String name, Map val, boolean excludeOuter, boolean isFirstVal) throws IOException;
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/XMLWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/XMLWriter.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/XMLWriter.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/XMLWriter.java Wed Mar 30 09:17:25 2011
@@ -17,25 +17,23 @@
package org.apache.solr.response;
+import java.io.IOException;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.XML;
import org.apache.solr.request.SolrQueryRequest;
-import org.apache.solr.search.SolrIndexSearcher;
-import org.apache.solr.search.DocList;
-import org.apache.solr.search.DocIterator;
-import org.apache.solr.search.DocSet;
import org.apache.solr.schema.SchemaField;
-import org.apache.solr.schema.TextField;
-
-import java.io.Writer;
-import java.io.IOException;
-import java.util.*;
-
-import org.apache.lucene.document.Fieldable;
-import org.apache.lucene.document.Document;
+import org.apache.solr.search.ReturnFields;
public final class XMLWriter extends TextResponseWriter {
@@ -44,8 +42,8 @@ public final class XMLWriter extends Tex
private static final char[] XML_START1="<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n".toCharArray();
- private static final char[] XML_STYLESHEET="<?xml-stylesheet type=\"text/xsl\" href=\"/admin/".toCharArray();
- private static final char[] XML_STYLESHEET_END=".xsl\"?>\n".toCharArray();
+ private static final char[] XML_STYLESHEET="<?xml-stylesheet type=\"text/xsl\" href=\"".toCharArray();
+ private static final char[] XML_STYLESHEET_END="\"?>\n".toCharArray();
/***
private static final char[] XML_START2_SCHEMA=(
@@ -76,9 +74,13 @@ public final class XMLWriter extends Tex
public XMLWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) {
super(writer, req, rsp);
- String version = req.getParams().get("version");
+ String version = req.getParams().get(CommonParams.VERSION);
float ver = version==null? CURRENT_VERSION : Float.parseFloat(version);
this.version = (int)(ver*1000);
+ if( this.version < 2200 ) {
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,
+ "XMLWriter does not support version: "+version );
+ }
}
@@ -89,7 +91,7 @@ public final class XMLWriter extends Tex
String stylesheet = req.getParams().get("stylesheet");
if (stylesheet != null && stylesheet.length() > 0) {
writer.write(XML_STYLESHEET);
- writer.write(stylesheet);
+ XML.escapeAttributeValue(stylesheet, writer);
writer.write(XML_STYLESHEET_END);
}
@@ -110,28 +112,6 @@ public final class XMLWriter extends Tex
int sz = lst.size();
int start=0;
- // special case the response header if the version is 2.1 or less
- if (version<=2100 && sz>0) {
- Object header = lst.getVal(0);
- if (header instanceof NamedList && "responseHeader".equals(lst.getName(0))) {
- writer.write("<responseHeader>");
- incLevel();
- NamedList nl = (NamedList)header;
- for (int i=0; i<nl.size(); i++) {
- String name = nl.getName(i);
- Object val = nl.getVal(i);
- if ("status".equals(name) || "QTime".equals(name)) {
- writePrim(name,null,val.toString(),false);
- } else {
- writeVal(name,val);
- }
- }
- decLevel();
- writer.write("</responseHeader>");
- start=1;
- }
- }
-
for (int i=start; i<sz; i++) {
writeVal(lst.getName(i),lst.getVal(i));
}
@@ -183,313 +163,60 @@ public final class XMLWriter extends Tex
}
}
- private static final Comparator fieldnameComparator = new Comparator() {
- public int compare(Object o, Object o1) {
- Fieldable f1 = (Fieldable)o; Fieldable f2 = (Fieldable)o1;
- int cmp = f1.name().compareTo(f2.name());
- return cmp;
- // note - the sort is stable, so this should not have affected the ordering
- // of fields with the same name w.r.t eachother.
- }
- };
@Override
- public final void writeDoc(String name, Document doc, Set<String> returnFields, float score, boolean includeScore) throws IOException {
- startTag("doc", name, false);
- incLevel();
-
- if (includeScore) {
- writeFloat("score", score);
- }
-
-
- // Lucene Documents have multivalued types as multiple fields
- // with the same name.
- // The XML needs to represent these as
- // an array. The fastest way to detect multiple fields
- // with the same name is to sort them first.
-
-
- // using global tlst here, so we shouldn't call any other
- // function that uses it until we are done.
- tlst.clear();
- for (Object obj : doc.getFields()) {
- Fieldable ff = (Fieldable)obj;
- // skip this field if it is not a field to be returned.
- if (returnFields!=null && !returnFields.contains(ff.name())) {
- continue;
- }
- tlst.add(ff);
- }
- Collections.sort(tlst, fieldnameComparator);
-
- int sz = tlst.size();
- int fidx1 = 0, fidx2 = 0;
- while (fidx1 < sz) {
- Fieldable f1 = (Fieldable)tlst.get(fidx1);
- String fname = f1.name();
-
- // find the end of fields with this name
- fidx2 = fidx1+1;
- while (fidx2 < sz && fname.equals(((Fieldable)tlst.get(fidx2)).name()) ) {
- fidx2++;
- }
-
- /***
- // more efficient to use getFieldType instead of
- // getField since that way dynamic fields won't have
- // to create a SchemaField on the fly.
- FieldType ft = schema.getFieldType(fname);
- ***/
-
- SchemaField sf = schema.getFieldOrNull(fname);
- if( sf == null ) {
- sf = new SchemaField( fname, new TextField() );
- }
- if (fidx1+1 == fidx2) {
- // single field value
- if (version>=2100 && sf.multiValued()) {
- startTag("arr",fname,false);
- doIndent=false;
- sf.write(this, null, f1);
- writer.write("</arr>");
- doIndent=defaultIndent;
- } else {
- sf.write(this, f1.name(), f1);
- }
- } else {
- // multiple fields with same name detected
+ public void writeStartDocumentList(String name,
+ long start, int size, long numFound, Float maxScore) throws IOException
+ {
+ if (doIndent) indent();
- startTag("arr",fname,false);
- incLevel();
- doIndent=false;
- int cnt=0;
- for (int i=fidx1; i<fidx2; i++) {
- if (defaultIndent && ++cnt==4) { // only indent every 4th item
- indent();
- cnt=0;
- }
- sf.write(this, null, (Fieldable)tlst.get(i));
- }
- decLevel();
- // if (doIndent) indent();
- writer.write("</arr>");
- // doIndent=true;
- doIndent=defaultIndent;
- }
- fidx1 = fidx2;
+ writer.write("<result");
+ writeAttr("name",name);
+ writeAttr("numFound",Long.toString(numFound));
+ writeAttr("start",Long.toString(start));
+ if(maxScore!=null) {
+ writeAttr("maxScore",Float.toString(maxScore));
}
-
- decLevel();
- if (doIndent) indent();
- writer.write("</doc>");
+ writer.write(">");
+
+ incLevel();
}
+
+ /**
+ * The SolrDocument should already have multivalued fields implemented as
+ * Collections -- this will not rewrite to <arr>
+ */
@Override
- public void writeSolrDocument(String name, SolrDocument doc, Set<String> returnFields, Map pseudoFields) throws IOException {
+ public void writeSolrDocument(String name, SolrDocument doc, ReturnFields returnFields, int idx ) throws IOException {
startTag("doc", name, false);
incLevel();
for (String fname : doc.getFieldNames()) {
- if (returnFields!=null && !returnFields.contains(fname)) {
+ if (!returnFields.wantsField(fname)) {
continue;
}
+
Object val = doc.getFieldValue(fname);
-
- if (val instanceof Collection) {
- writeVal(fname, val);
- } else {
- // single valued... figure out if we should put <arr> tags around it anyway
- SchemaField sf = schema.getFieldOrNull(fname);
- if (version>=2100 && sf!=null && sf.multiValued()) {
- startTag("arr",fname,false);
- doIndent=false;
- writeVal(fname, val);
- writer.write("</arr>");
- doIndent=defaultIndent;
- } else {
- writeVal(fname, val);
- }
- }
- }
-
- if (pseudoFields != null) {
- for (Object fname : pseudoFields.keySet()) {
- writeVal(fname.toString(), pseudoFields.get(fname));
+ if( "_explain_".equals( fname ) ) {
+ System.out.println( val );
}
+ writeVal(fname, val);
}
-
+
decLevel();
- if (doIndent) indent();
writer.write("</doc>");
}
-
-
- private static interface DocumentListInfo {
- Float getMaxScore();
- int getCount();
- long getNumFound();
- long getStart();
- void writeDocs( boolean includeScore, Set<String> fields ) throws IOException;
- }
-
- private final void writeDocuments(
- String name,
- DocumentListInfo docs,
- Set<String> fields) throws IOException
+
+ @Override
+ public void writeEndDocumentList() throws IOException
{
- boolean includeScore=false;
- if (fields!=null) {
- includeScore = fields.contains("score");
- if (fields.size()==0 || (fields.size()==1 && includeScore) || fields.contains("*")) {
- fields=null; // null means return all stored fields
- }
- }
-
- int sz=docs.getCount();
- if (doIndent) indent();
-
- writer.write("<result");
- writeAttr("name",name);
- writeAttr("numFound",Long.toString(docs.getNumFound()));
- writeAttr("start",Long.toString(docs.getStart()));
- if (includeScore && docs.getMaxScore()!=null) {
- writeAttr("maxScore",Float.toString(docs.getMaxScore()));
- }
- if (sz==0) {
- writer.write("/>");
- return;
- } else {
- writer.write('>');
- }
-
- incLevel();
- docs.writeDocs(includeScore, fields);
decLevel();
-
if (doIndent) indent();
writer.write("</result>");
}
- @Override
- public final void writeSolrDocumentList(String name, final SolrDocumentList docs, Set<String> fields, Map otherFields) throws IOException
- {
- this.writeDocuments( name, new DocumentListInfo()
- {
- public int getCount() {
- return docs.size();
- }
-
- public Float getMaxScore() {
- return docs.getMaxScore();
- }
-
- public long getNumFound() {
- return docs.getNumFound();
- }
- public long getStart() {
- return docs.getStart();
- }
-
- public void writeDocs(boolean includeScore, Set<String> fields) throws IOException {
- for( SolrDocument doc : docs ) {
- writeSolrDocument(null, doc, fields, null);
- }
- }
- }, fields );
- }
-
- @Override
- public void writeDocList(String name, final DocList ids, Set<String> fields, Map otherFields) throws IOException
- {
- this.writeDocuments( name, new DocumentListInfo()
- {
- public int getCount() {
- return ids.size();
- }
-
- public Float getMaxScore() {
- return ids.maxScore();
- }
-
- public long getNumFound() {
- return ids.matches();
- }
-
- public long getStart() {
- return ids.offset();
- }
-
- public void writeDocs(boolean includeScore, Set<String> fields) throws IOException {
- SolrIndexSearcher searcher = req.getSearcher();
- DocIterator iterator = ids.iterator();
- int sz = ids.size();
- includeScore = includeScore && ids.hasScores();
- for (int i=0; i<sz; i++) {
- int id = iterator.nextDoc();
- Document doc = searcher.doc(id, fields);
- writeDoc(null, doc, fields, (includeScore ? iterator.score() : 0.0f), includeScore);
- }
- }
- }, fields );
- }
-
-
- @Override
- public void writeVal(String name, Object val) throws IOException {
-
- // if there get to be enough types, perhaps hashing on the type
- // to get a handler might be faster (but types must be exact to do that...)
-
- // go in order of most common to least common
- if (val==null) {
- writeNull(name);
- } else if (val instanceof String) {
- writeStr(name, (String)val, true);
- } else if (val instanceof Integer) {
- // it would be slower to pass the int ((Integer)val).intValue()
- writeInt(name, val.toString());
- } else if (val instanceof Boolean) {
- // could be optimized... only two vals
- writeBool(name, val.toString());
- } else if (val instanceof Long) {
- writeLong(name, val.toString());
- } else if (val instanceof Date) {
- writeDate(name,(Date)val);
- } else if (val instanceof Float) {
- // we pass the float instead of using toString() because
- // it may need special formatting. same for double.
- writeFloat(name, ((Float)val).floatValue());
- } else if (val instanceof Double) {
- writeDouble(name, ((Double)val).doubleValue());
- } else if (val instanceof Document) {
- writeDoc(name, (Document)val, returnFields, 0.0f, false);
- } else if (val instanceof DocList) {
- // requires access to IndexReader
- writeDocList(name, (DocList)val, returnFields, null);
- }else if (val instanceof SolrDocumentList) {
- // requires access to IndexReader
- writeSolrDocumentList(name, (SolrDocumentList)val, returnFields, null);
- }else if (val instanceof DocSet) {
- // how do we know what fields to read?
- // todo: have a DocList/DocSet wrapper that
- // restricts the fields to write...?
- } else if (val instanceof Map) {
- writeMap(name, (Map)val, false, true);
- } else if (val instanceof NamedList) {
- writeNamedList(name, (NamedList)val);
- } else if (val instanceof Iterable) {
- writeArray(name,((Iterable)val).iterator());
- } else if (val instanceof Object[]) {
- writeArray(name,(Object[])val);
- } else if (val instanceof Iterator) {
- writeArray(name,(Iterator)val);
- } else {
- // default...
- writeStr(name, val.getClass().getName() + ':' + val.toString(), true);
- }
- }
//
// Generic compound types
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/XSLTResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/XSLTResponseWriter.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/XSLTResponseWriter.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/XSLTResponseWriter.java Wed Mar 30 09:17:25 2011
@@ -44,7 +44,7 @@ import org.apache.solr.util.xslt.Transfo
*/
public class XSLTResponseWriter implements QueryResponseWriter {
- public static final String DEFAULT_CONTENT_TYPE = "text/xml";
+ public static final String DEFAULT_CONTENT_TYPE = "application/xml";
public static final String TRANSFORM_PARAM = "tr";
public static final String CONTEXT_TRANSFORMER_KEY = "xsltwriter.transformer";
@@ -70,14 +70,23 @@ public class XSLTResponseWriter implemen
throw new RuntimeException("getTransformer fails in getContentType",e);
}
- final String mediaTypeFromXslt = t.getOutputProperty("media-type");
- if(mediaTypeFromXslt == null || mediaTypeFromXslt.length()==0) {
+ String mediaType = t.getOutputProperty("media-type");
+ if (mediaType == null || mediaType.length()==0) {
// This did not happen in my tests, mediaTypeFromXslt is set to "text/xml"
// if the XSLT transform does not contain an xsl:output element. Not sure
// if this is standard behavior or if it's just my JVM/libraries
- return DEFAULT_CONTENT_TYPE;
+ mediaType = DEFAULT_CONTENT_TYPE;
}
- return mediaTypeFromXslt;
+
+ if (!mediaType.contains("charset")) {
+ String encoding = t.getOutputProperty("encoding");
+ if (encoding == null || encoding.length()==0) {
+ encoding = "UTF-8";
+ }
+ mediaType = mediaType + "; charset=" + encoding;
+ }
+
+ return mediaType;
}
public void write(Writer writer, SolrQueryRequest request, SolrQueryResponse response) throws IOException {
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/BinaryField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/BinaryField.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/BinaryField.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/BinaryField.java Wed Mar 30 09:17:25 2011
@@ -54,6 +54,7 @@ public class BinaryField extends FieldTy
return ByteBuffer.wrap(f.getBinaryValue(), f.getBinaryOffset(), f.getBinaryLength() ) ;
}
+ @Override
public Fieldable createField(SchemaField field, Object val, float boost) {
if (val == null) return null;
if (!field.stored()) {
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/DateField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/DateField.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/DateField.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/DateField.java Wed Mar 30 09:17:25 2011
@@ -410,7 +410,7 @@ public class DateField extends FieldType
/** DateField specific range query */
public Query getRangeQuery(QParser parser, SchemaField sf, Date part1, Date part2, boolean minInclusive, boolean maxInclusive) {
- return new TermRangeQuery(
+ return TermRangeQuery.newStringRange(
sf.getName(),
part1 == null ? null : toInternal(part1),
part2 == null ? null : toInternal(part2),
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/FieldType.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/FieldType.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/FieldType.java Wed Mar 30 09:17:25 2011
@@ -33,6 +33,7 @@ import org.apache.lucene.util.UnicodeUti
import org.apache.noggit.CharArr;
import org.apache.solr.analysis.SolrAnalyzer;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.response.TextResponseWriter;
import org.apache.solr.search.QParser;
import org.apache.solr.search.Sorting;
@@ -57,7 +58,7 @@ public abstract class FieldType extends
/**
* The default poly field separator.
*
- * @see #createFields(SchemaField, String, float)
+ * @see #createFields(SchemaField, Object, float)
* @see #isPolyField()
*/
public static final String POLY_FIELD_SEPARATOR = "___";
@@ -84,9 +85,9 @@ public abstract class FieldType extends
}
/**
- * A "polyField" is a FieldType that can produce more than one Fieldable instance for a single value, via the {@link #createFields(org.apache.solr.schema.SchemaField, String, float)} method. This is useful
+ * A "polyField" is a FieldType that can produce more than one Fieldable instance for a single value, via the {@link #createFields(org.apache.solr.schema.SchemaField, Object, float)} method. This is useful
* when hiding the implementation details of a field from the Solr end user. For instance, a spatial point may be represented by multiple different fields.
- * @return true if the {@link #createFields(org.apache.solr.schema.SchemaField, String, float)} method may return more than one field
+ * @return true if the {@link #createFields(org.apache.solr.schema.SchemaField, Object, float)} method may return more than one field
*/
public boolean isPolyField(){
return false;
@@ -222,17 +223,18 @@ public abstract class FieldType extends
*
*
*/
- public Fieldable createField(SchemaField field, String externalVal, float boost) {
+ public Fieldable createField(SchemaField field, Object value, float boost) {
if (!field.indexed() && !field.stored()) {
if (log.isTraceEnabled())
log.trace("Ignoring unindexed/unstored field: " + field);
return null;
}
+
String val;
try {
- val = toInternal(externalVal);
+ val = toInternal(value.toString());
} catch (RuntimeException e) {
- throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Error while creating field '" + field + "' from value '" + externalVal + "'", e, false);
+ throw new SolrException( SolrException.ErrorCode.SERVER_ERROR, "Error while creating field '" + field + "' from value '" + value + "'", e, false);
}
if (val==null) return null;
@@ -271,15 +273,15 @@ public abstract class FieldType extends
/**
* Given a {@link org.apache.solr.schema.SchemaField}, create one or more {@link org.apache.lucene.document.Fieldable} instances
* @param field the {@link org.apache.solr.schema.SchemaField}
- * @param externalVal The value to add to the field
+ * @param value The value to add to the field
* @param boost The boost to apply
* @return An array of {@link org.apache.lucene.document.Fieldable}
*
- * @see #createField(SchemaField, String, float)
+ * @see #createField(SchemaField, Object, float)
* @see #isPolyField()
*/
- public Fieldable[] createFields(SchemaField field, String externalVal, float boost) {
- Fieldable f = createField( field, externalVal, boost);
+ public Fieldable[] createFields(SchemaField field, Object value, float boost) {
+ Fieldable f = createField( field, value, boost);
return f==null ? new Fieldable[]{} : new Fieldable[]{f};
}
@@ -418,6 +420,7 @@ public abstract class FieldType extends
* of this type, subclasses can set analyzer themselves or override
* getAnalyzer()
* @see #getAnalyzer
+ * @see #setAnalyzer
*/
protected Analyzer analyzer=new DefaultAnalyzer(256);
@@ -426,6 +429,7 @@ public abstract class FieldType extends
* of this type, subclasses can set analyzer themselves or override
* getAnalyzer()
* @see #getQueryAnalyzer
+ * @see #setQueryAnalyzer
*/
protected Analyzer queryAnalyzer=analyzer;
@@ -451,22 +455,52 @@ public abstract class FieldType extends
return queryAnalyzer;
}
+ private final String analyzerError =
+ "FieldType: " + this.getClass().getSimpleName() +
+ " (" + typeName + ") does not support specifying an analyzer";
+
/**
* Sets the Analyzer to be used when indexing fields of this type.
+ *
+ * <p>
+ * The default implementation throws a SolrException.
+ * Subclasses that override this method need to ensure the behavior
+ * of the analyzer is consistent with the implementation of toInternal.
+ * </p>
+ *
+ * @see #toInternal
+ * @see #setQueryAnalyzer
* @see #getAnalyzer
*/
public void setAnalyzer(Analyzer analyzer) {
- this.analyzer = analyzer;
- log.trace("FieldType: " + typeName + ".setAnalyzer(" + analyzer.getClass().getName() + ")" );
+ SolrException e = new SolrException
+ (ErrorCode.SERVER_ERROR,
+ "FieldType: " + this.getClass().getSimpleName() +
+ " (" + typeName + ") does not support specifying an analyzer");
+ SolrException.logOnce(log,null,e);
+ throw e;
}
/**
* Sets the Analyzer to be used when querying fields of this type.
+ *
+ * <p>
+ * The default implementation throws a SolrException.
+ * Subclasses that override this method need to ensure the behavior
+ * of the analyzer is consistent with the implementation of toInternal.
+ * </p>
+ *
+ * @see #toInternal
+ * @see #setAnalyzer
* @see #getQueryAnalyzer
*/
public void setQueryAnalyzer(Analyzer analyzer) {
- this.queryAnalyzer = analyzer;
- log.trace("FieldType: " + typeName + ".setQueryAnalyzer(" + analyzer.getClass().getName() + ")" );
+ SolrException e = new SolrException
+ (ErrorCode.SERVER_ERROR,
+ "FieldType: " + this.getClass().getSimpleName() +
+ " (" + typeName + ") does not support specifying an analyzer");
+ SolrException.logOnce(log,null,e);
+ throw e;
}
/**
@@ -521,7 +555,7 @@ public abstract class FieldType extends
*/
public Query getRangeQuery(QParser parser, SchemaField field, String part1, String part2, boolean minInclusive, boolean maxInclusive) {
// constant score mode is now enabled per default
- return new TermRangeQuery(
+ return TermRangeQuery.newStringRange(
field.getName(),
part1 == null ? null : toInternal(part1),
part2 == null ? null : toInternal(part2),
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/IndexSchema.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/IndexSchema.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/IndexSchema.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/IndexSchema.java Wed Mar 30 09:17:25 2011
@@ -28,6 +28,7 @@ import org.apache.solr.common.ResourceLo
import org.apache.solr.common.SolrException;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.DOMUtil;
+import org.apache.solr.common.util.SystemIdResolver;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.Config;
import org.apache.solr.core.SolrResourceLoader;
@@ -39,6 +40,7 @@ import org.apache.solr.search.SolrQueryP
import org.apache.solr.util.plugin.AbstractPluginLoader;
import org.apache.solr.util.plugin.SolrCoreAware;
import org.w3c.dom.*;
+import org.xml.sax.InputSource;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
@@ -101,22 +103,17 @@ public final class IndexSchema {
* By default, this follows the normal config path directory searching rules.
* @see SolrResourceLoader#openResource
*/
- public IndexSchema(SolrConfig solrConfig, String name, InputStream is) {
+ public IndexSchema(SolrConfig solrConfig, String name, InputSource is) {
this.solrConfig = solrConfig;
if (name == null)
name = DEFAULT_SCHEMA_FILE;
this.resourceName = name;
loader = solrConfig.getResourceLoader();
- InputStream lis = is;
- if (lis == null)
- lis = loader.openSchema(name);
- readSchema(lis);
- if (lis != is) {
- try {
- lis.close();
- }
- catch(IOException xio) {} // ignore
+ if (is == null) {
+ is = new InputSource(loader.openSchema(name));
+ is.setSystemId(SystemIdResolver.createSystemIdFromResourceName(name));
}
+ readSchema(is);
loader.inform( loader );
}
@@ -353,7 +350,7 @@ public final class IndexSchema {
}
}
- private void readSchema(InputStream is) {
+ private void readSchema(InputSource is) {
log.info("Reading Solr Schema");
try {
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/LatLonType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/LatLonType.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/LatLonType.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/LatLonType.java Wed Mar 30 09:17:25 2011
@@ -54,7 +54,8 @@ public class LatLonType extends Abstract
}
@Override
- public Fieldable[] createFields(SchemaField field, String externalVal, float boost) {
+ public Fieldable[] createFields(SchemaField field, Object value, float boost) {
+ String externalVal = value.toString();
//we could have tileDiff + 3 fields (two for the lat/lon, one for storage)
Fieldable[] f = new Fieldable[(field.indexed() ? 2 : 0) + (field.stored() ? 1 : 0)];
if (field.indexed()) {
@@ -280,7 +281,7 @@ public class LatLonType extends Abstract
//It never makes sense to create a single field, so make it impossible to happen
@Override
- public Fieldable createField(SchemaField field, String externalVal, float boost) {
+ public Fieldable createField(SchemaField field, Object value, float boost) {
throw new UnsupportedOperationException("LatLonType uses multiple fields. field=" + field.getName());
}
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/PointType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/PointType.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/PointType.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/PointType.java Wed Mar 30 09:17:25 2011
@@ -68,7 +68,8 @@ public class PointType extends Coordinat
}
@Override
- public Fieldable[] createFields(SchemaField field, String externalVal, float boost) {
+ public Fieldable[] createFields(SchemaField field, Object value, float boost) {
+ String externalVal = value.toString();
String[] point = new String[0];
try {
point = DistanceUtils.parsePoint(null, externalVal, dimension);
@@ -112,7 +113,7 @@ public class PointType extends Coordinat
*
*/
@Override
- public Fieldable createField(SchemaField field, String externalVal, float boost) {
+ public Fieldable createField(SchemaField field, Object value, float boost) {
throw new UnsupportedOperationException("PointType uses multiple fields. field=" + field.getName());
}
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/SchemaField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/SchemaField.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/SchemaField.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/SchemaField.java Wed Mar 30 09:17:25 2011
@@ -93,16 +93,16 @@ public final class SchemaField extends F
boolean isBinary() { return (properties & BINARY)!=0; }
- public Fieldable createField(String val, float boost) {
+ public Fieldable createField(Object val, float boost) {
return type.createField(this,val,boost);
}
- public Fieldable[] createFields(String val, float boost) {
+ public Fieldable[] createFields(Object val, float boost) {
return type.createFields(this,val,boost);
}
/**
- * If true, then use {@link #createFields(String, float)}, else use {@link #createField} to save an extra allocation
+ * If true, then use {@link #createFields(Object, float)}, else use {@link #createField} to save an extra allocation
* @return true if this field is a poly field
*/
public boolean isPolyField(){
@@ -150,7 +150,6 @@ public final class SchemaField extends F
"can not sort on multivalued field: "
+ getName());
}
-
}
/**
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/TextField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/TextField.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/TextField.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/TextField.java Wed Mar 30 09:17:25 2011
@@ -84,6 +84,15 @@ public class TextField extends FieldType
return ByteUtils.UTF8toUTF16(term);
}
+ @Override
+ public void setAnalyzer(Analyzer analyzer) {
+ this.analyzer = analyzer;
+ }
+
+ @Override
+ public void setQueryAnalyzer(Analyzer analyzer) {
+ this.queryAnalyzer = analyzer;
+ }
static Query parseFieldQuery(QParser parser, Analyzer analyzer, String field, String queryText) {
int phraseSlop = 0;
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/TrieDateField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/TrieDateField.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/TrieDateField.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/TrieDateField.java Wed Mar 30 09:17:25 2011
@@ -79,7 +79,18 @@ public class TrieDateField extends DateF
@Override
public SortField getSortField(SchemaField field, boolean top) {
field.checkSortability();
- return new SortField(new LongValuesCreator( field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER, CachedArrayCreator.CACHE_VALUES_AND_BITS ), top);
+
+ int flags = CachedArrayCreator.CACHE_VALUES_AND_BITS;
+ boolean sortMissingLast = field.sortMissingLast();
+ boolean sortMissingFirst = field.sortMissingFirst();
+
+ Object missingValue = null;
+ if( sortMissingLast ) {
+ missingValue = top ? Long.MIN_VALUE : Long.MAX_VALUE;
+ } else if( sortMissingFirst ) {
+ missingValue = top ? Long.MAX_VALUE : Long.MIN_VALUE;
+ }
+ return new SortField(new LongValuesCreator(field.getName(), FieldCache.NUMERIC_UTILS_LONG_PARSER, flags), top).setMissingValue(missingValue);
}
@Override
@@ -157,7 +168,7 @@ public class TrieDateField extends DateF
}
@Override
- public Fieldable createField(SchemaField field, String externalVal, float boost) {
+ public Fieldable createField(SchemaField field, Object value, float boost) {
boolean indexed = field.indexed();
boolean stored = field.stored();
@@ -172,7 +183,10 @@ public class TrieDateField extends DateF
byte[] arr=null;
TokenStream ts=null;
- long time = super.parseMath(null, externalVal).getTime();
+ long time = (value instanceof Date)
+ ? ((Date)value).getTime()
+ : super.parseMath(null, value.toString()).getTime();
+
if (stored) arr = TrieField.toArr(time);
if (indexed) ts = new NumericTokenStream(ps).setLongValue(time);
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/TrieField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/TrieField.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/TrieField.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/TrieField.java Wed Mar 30 09:17:25 2011
@@ -127,8 +127,8 @@ public class TrieField extends FieldType
int flags = CachedArrayCreator.CACHE_VALUES_AND_BITS;
Object missingValue = null;
- boolean sortMissingLast = on( SORT_MISSING_LAST, properties );
- boolean sortMissingFirst = on( SORT_MISSING_FIRST, properties );
+ boolean sortMissingLast = field.sortMissingLast();
+ boolean sortMissingFirst = field.sortMissingFirst();
switch (type) {
case INTEGER:
@@ -482,7 +482,7 @@ public class TrieField extends FieldType
}
@Override
- public Fieldable createField(SchemaField field, String externalVal, float boost) {
+ public Fieldable createField(SchemaField field, Object value, float boost) {
boolean indexed = field.indexed();
boolean stored = field.stored();
@@ -500,27 +500,37 @@ public class TrieField extends FieldType
switch (type) {
case INTEGER:
- int i = Integer.parseInt(externalVal);
+ int i = (value instanceof Number)
+ ? ((Number)value).intValue()
+ : Integer.parseInt(value.toString());
if (stored) arr = toArr(i);
if (indexed) ts = new NumericTokenStream(ps).setIntValue(i);
break;
case FLOAT:
- float f = Float.parseFloat(externalVal);
+ float f = (value instanceof Number)
+ ? ((Number)value).floatValue()
+ : Float.parseFloat(value.toString());
if (stored) arr = toArr(f);
if (indexed) ts = new NumericTokenStream(ps).setFloatValue(f);
break;
case LONG:
- long l = Long.parseLong(externalVal);
+ long l = (value instanceof Number)
+ ? ((Number)value).longValue()
+ : Long.parseLong(value.toString());
if (stored) arr = toArr(l);
if (indexed) ts = new NumericTokenStream(ps).setLongValue(l);
break;
case DOUBLE:
- double d = Double.parseDouble(externalVal);
+ double d = (value instanceof Number)
+ ? ((Number)value).doubleValue()
+ : Double.parseDouble(value.toString());
if (stored) arr = toArr(d);
if (indexed) ts = new NumericTokenStream(ps).setDoubleValue(d);
break;
case DATE:
- long time = dateField.parseMath(null, externalVal).getTime();
+ long time = (value instanceof Date)
+ ? ((Date)value).getTime()
+ : dateField.parseMath(null, value.toString()).getTime();
if (stored) arr = toArr(time);
if (indexed) ts = new NumericTokenStream(ps).setLongValue(time);
break;
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/search/QueryParsing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/search/QueryParsing.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/search/QueryParsing.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/search/QueryParsing.java Wed Mar 30 09:17:25 2011
@@ -398,8 +398,8 @@ public class QueryParsing {
String fname = q.getField();
FieldType ft = writeFieldName(fname, schema, out, flags);
out.append(q.includesLower() ? '[' : '{');
- String lt = q.getLowerTerm();
- String ut = q.getUpperTerm();
+ String lt = q.getLowerTerm().utf8ToString();
+ String ut = q.getUpperTerm().utf8ToString();
if (lt == null) {
out.append('*');
} else {
@@ -582,7 +582,7 @@ public class QueryParsing {
boolean opt(char ch) {
eatws();
- if (val.charAt(pos) == ch) {
+ if (pos < end && val.charAt(pos) == ch) {
pos++;
return true;
}
@@ -693,7 +693,29 @@ public class QueryParsing {
pos++;
while (pos < end) {
ch = val.charAt(pos);
- if (!Character.isJavaIdentifierPart(ch) && ch != '.') {
+ if (!Character.isJavaIdentifierPart(ch) && ch != '.' && ch != ':') {
+ break;
+ }
+ pos++;
+ }
+ return val.substring(id_start, pos);
+ }
+
+ if (errMessage != null) {
+ throw new ParseException(errMessage + " at pos " + pos + " str='" + val + "'");
+ }
+ return null;
+ }
+
+ public String getGlobbedId(String errMessage) throws ParseException {
+ eatws();
+ int id_start = pos;
+ char ch;
+ if (pos < end && (ch = val.charAt(pos)) != '$' && (Character.isJavaIdentifierStart(ch) || ch=='?' || ch=='*')) {
+ pos++;
+ while (pos < end) {
+ ch = val.charAt(pos);
+ if (!(Character.isJavaIdentifierPart(ch) || ch=='?' || ch=='*') && ch != '.') {
break;
}
pos++;
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/search/ValueSourceParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/search/ValueSourceParser.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/search/ValueSourceParser.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/search/ValueSourceParser.java Wed Mar 30 09:17:25 2011
@@ -583,7 +583,7 @@ public abstract class ValueSourceParser
if (ft == null) ft = new StrField();
if (ft instanceof TextField) {
- // need to do analyisis on the term
+ // need to do analysis on the term
String indexedVal = tinfo.val;
Query q = ft.getFieldQuery(fp, fp.getReq().getSchema().getFieldOrNull(tinfo.field), tinfo.val);
if (q instanceof TermQuery) {
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/spelling/suggest/Lookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/spelling/suggest/Lookup.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/spelling/suggest/Lookup.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/spelling/suggest/Lookup.java Wed Mar 30 09:17:25 2011
@@ -39,7 +39,7 @@ public abstract class Lookup {
public static final class LookupPriorityQueue extends PriorityQueue<LookupResult> {
public LookupPriorityQueue(int size) {
- initialize(size);
+ super(size);
}
@Override
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/spelling/suggest/Suggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/spelling/suggest/Suggester.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/spelling/suggest/Suggester.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/spelling/suggest/Suggester.java Wed Mar 30 09:17:25 2011
@@ -83,6 +83,8 @@ public class Suggester extends SolrSpell
if (lookupImpl == null) {
lookupImpl = JaspellLookup.class.getName();
}
+ lookup = (Lookup) core.getResourceLoader().newInstance(lookupImpl);
+ lookup.init(config, core);
String store = (String)config.get(STORE_DIR);
if (store != null) {
storeDir = new File(store);
@@ -91,6 +93,13 @@ public class Suggester extends SolrSpell
}
if (!storeDir.exists()) {
storeDir.mkdirs();
+ } else {
+ // attempt reload of the stored lookup
+ try {
+ lookup.load(storeDir);
+ } catch (IOException e) {
+ LOG.warn("Loading stored lookup data failed", e);
+ }
}
}
return name;
@@ -107,17 +116,17 @@ public class Suggester extends SolrSpell
dictionary = new FileDictionary(new InputStreamReader(
core.getResourceLoader().openResource(sourceLocation), "UTF-8"));
} catch (UnsupportedEncodingException e) {
- e.printStackTrace();
+ // should not happen
+ LOG.error("should not happen", e);
}
}
- lookup = (Lookup) core.getResourceLoader().newInstance(lookupImpl);
try {
lookup.build(dictionary);
if (storeDir != null) {
lookup.store(storeDir);
}
} catch (Exception e) {
- e.printStackTrace();
+ LOG.error("Error while building or storing Suggester data", e);
}
}
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellLookup.java?rev=1086876&r1=1086875&r2=1086876&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellLookup.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/spelling/suggest/jaspell/JaspellLookup.java Wed Mar 30 09:17:25 2011
@@ -1,6 +1,10 @@
package org.apache.solr.spelling.suggest.jaspell;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
@@ -9,6 +13,7 @@ import org.apache.solr.common.util.Named
import org.apache.solr.core.SolrCore;
import org.apache.solr.spelling.suggest.Lookup;
import org.apache.solr.spelling.suggest.UnsortedTermFreqIteratorWrapper;
+import org.apache.solr.spelling.suggest.jaspell.JaspellTernarySearchTrie.TSTNode;
import org.apache.solr.util.SortedIterator;
import org.apache.solr.util.TermFreqIterator;
import org.slf4j.Logger;
@@ -16,7 +21,7 @@ import org.slf4j.LoggerFactory;
public class JaspellLookup extends Lookup {
private static final Logger LOG = LoggerFactory.getLogger(JaspellLookup.class);
- JaspellTernarySearchTrie trie;
+ JaspellTernarySearchTrie trie = new JaspellTernarySearchTrie();
private boolean usePrefix = true;
private int editDistance = 2;
@@ -89,14 +94,89 @@ public class JaspellLookup extends Looku
return res;
}
+ public static final String FILENAME = "jaspell.dat";
+ private static final byte LO_KID = 0x01;
+ private static final byte EQ_KID = 0x02;
+ private static final byte HI_KID = 0x04;
+ private static final byte HAS_VALUE = 0x08;
+
+
@Override
public boolean load(File storeDir) throws IOException {
- return false;
+ File data = new File(storeDir, FILENAME);
+ if (!data.exists() || !data.canRead()) {
+ return false;
+ }
+ DataInputStream in = new DataInputStream(new FileInputStream(data));
+ TSTNode root = trie.new TSTNode('\0', null);
+ try {
+ readRecursively(in, root);
+ trie.setRoot(root);
+ } finally {
+ in.close();
+ }
+ return true;
+ }
+
+ private void readRecursively(DataInputStream in, TSTNode node) throws IOException {
+ node.splitchar = in.readChar();
+ byte mask = in.readByte();
+ if ((mask & HAS_VALUE) != 0) {
+ node.data = new Float(in.readFloat());
+ }
+ if ((mask & LO_KID) != 0) {
+ TSTNode kid = trie.new TSTNode('\0', node);
+ node.relatives[TSTNode.LOKID] = kid;
+ readRecursively(in, kid);
+ }
+ if ((mask & EQ_KID) != 0) {
+ TSTNode kid = trie.new TSTNode('\0', node);
+ node.relatives[TSTNode.EQKID] = kid;
+ readRecursively(in, kid);
+ }
+ if ((mask & HI_KID) != 0) {
+ TSTNode kid = trie.new TSTNode('\0', node);
+ node.relatives[TSTNode.HIKID] = kid;
+ readRecursively(in, kid);
+ }
}
@Override
public boolean store(File storeDir) throws IOException {
- return false;
+ if (!storeDir.exists() || !storeDir.isDirectory() || !storeDir.canWrite()) {
+ return false;
+ }
+ TSTNode root = trie.getRoot();
+ if (root == null) { // empty tree
+ return false;
+ }
+ File data = new File(storeDir, FILENAME);
+ DataOutputStream out = new DataOutputStream(new FileOutputStream(data));
+ try {
+ writeRecursively(out, root);
+ out.flush();
+ } finally {
+ out.close();
+ }
+ return true;
+ }
+
+ private void writeRecursively(DataOutputStream out, TSTNode node) throws IOException {
+ if (node == null) {
+ return;
+ }
+ out.writeChar(node.splitchar);
+ byte mask = 0;
+ if (node.relatives[TSTNode.LOKID] != null) mask |= LO_KID;
+ if (node.relatives[TSTNode.EQKID] != null) mask |= EQ_KID;
+ if (node.relatives[TSTNode.HIKID] != null) mask |= HI_KID;
+ if (node.data != null) mask |= HAS_VALUE;
+ out.writeByte(mask);
+ if (node.data != null) {
+ out.writeFloat((Float)node.data);
+ }
+ writeRecursively(out, node.relatives[TSTNode.LOKID]);
+ writeRecursively(out, node.relatives[TSTNode.EQKID]);
+ writeRecursively(out, node.relatives[TSTNode.HIKID]);
}
-
}