You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by yo...@apache.org on 2010/07/27 21:06:41 UTC
svn commit: r979807 - in /lucene/dev/trunk/solr: ./ example/solr/conf/ lib/
src/common/org/apache/solr/common/util/ src/java/org/apache/solr/core/
src/java/org/apache/solr/handler/ src/java/org/apache/solr/response/
src/java/org/apache/solr/util/ src/t...
Author: yonik
Date: Tue Jul 27 19:06:39 2010
New Revision: 979807
URL: http://svn.apache.org/viewvc?rev=979807&view=rev
Log:
SOLR-1925: CSVResponseWriter and commons-csv update
Added:
lucene/dev/trunk/solr/lib/commons-csv-1.0-SNAPSHOT-r966014.jar (with props)
lucene/dev/trunk/solr/src/java/org/apache/solr/response/CSVResponseWriter.java (with props)
lucene/dev/trunk/solr/src/test/org/apache/solr/response/
lucene/dev/trunk/solr/src/test/org/apache/solr/response/TestCSVResponseWriter.java
Removed:
lucene/dev/trunk/solr/lib/commons-csv-1.0-SNAPSHOT-r609327.jar
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/example/solr/conf/solrconfig.xml
lucene/dev/trunk/solr/src/common/org/apache/solr/common/util/DateUtil.java
lucene/dev/trunk/solr/src/common/org/apache/solr/common/util/FastWriter.java
lucene/dev/trunk/solr/src/java/org/apache/solr/core/SolrCore.java
lucene/dev/trunk/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java
lucene/dev/trunk/solr/src/java/org/apache/solr/util/SolrPluginUtils.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=979807&r1=979806&r2=979807&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Tue Jul 27 19:06:39 2010
@@ -210,7 +210,10 @@ New Features
will cause the parser to generate text:"pdp 11" rather than (text:PDP OR text:11).
Note that autoGeneratePhraseQueries="true" tends to not work well for non whitespace
delimited languages. (yonik)
-
+
+* SOLR-1925: Add CSVResponseWriter (use wt=csv) that returns the list of documents
+ in CSV format. (Chris Mattmann, yonik)
+
Optimizations
----------------------
Modified: lucene/dev/trunk/solr/example/solr/conf/solrconfig.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/example/solr/conf/solrconfig.xml?rev=979807&r1=979806&r2=979807&view=diff
==============================================================================
--- lucene/dev/trunk/solr/example/solr/conf/solrconfig.xml (original)
+++ lucene/dev/trunk/solr/example/solr/conf/solrconfig.xml Tue Jul 27 19:06:39 2010
@@ -1062,6 +1062,7 @@
<queryResponseWriter name="php" class="solr.PHPResponseWriter"/>
<queryResponseWriter name="phps" class="solr.PHPSerializedResponseWriter"/>
<queryResponseWriter name="velocity" class="solr.VelocityResponseWriter"/>
+ <queryResponseWriter name="csv" class="solr.CSVResponseWriter"/>
Custom response writers can be declared as needed...
Added: lucene/dev/trunk/solr/lib/commons-csv-1.0-SNAPSHOT-r966014.jar
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/lib/commons-csv-1.0-SNAPSHOT-r966014.jar?rev=979807&view=auto
==============================================================================
Binary file - no diff available.
Propchange: lucene/dev/trunk/solr/lib/commons-csv-1.0-SNAPSHOT-r966014.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: lucene/dev/trunk/solr/src/common/org/apache/solr/common/util/DateUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/common/org/apache/solr/common/util/DateUtil.java?rev=979807&r1=979806&r2=979807&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/common/org/apache/solr/common/util/DateUtil.java (original)
+++ lucene/dev/trunk/solr/src/common/org/apache/solr/common/util/DateUtil.java Tue Jul 27 19:06:39 2010
@@ -16,6 +16,8 @@ package org.apache.solr.common.util;
* limitations under the License.
*/
+import java.io.IOException;
+import java.io.Writer;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
@@ -196,5 +198,66 @@ public class DateUtil {
}
}
+ /** Formats the date and returns the calendar instance that was used (which may be reused) */
+ public static Calendar formatDate(Date date, Calendar cal, Appendable out) throws IOException {
+ // using a stringBuilder for numbers can be nice since
+ // a temporary string isn't used (it's added directly to the
+ // builder's buffer.
+
+ StringBuilder sb = out instanceof StringBuilder ? (StringBuilder)out : new StringBuilder();
+ if (cal==null) cal = Calendar.getInstance(TimeZone.getTimeZone("GMT"), Locale.US);
+ cal.setTime(date);
+
+ int i = cal.get(Calendar.YEAR);
+ sb.append(i);
+ sb.append('-');
+ i = cal.get(Calendar.MONTH) + 1; // 0 based, so add 1
+ if (i<10) sb.append('0');
+ sb.append(i);
+ sb.append('-');
+ i=cal.get(Calendar.DAY_OF_MONTH);
+ if (i<10) sb.append('0');
+ sb.append(i);
+ sb.append('T');
+ i=cal.get(Calendar.HOUR_OF_DAY); // 24 hour time format
+ if (i<10) sb.append('0');
+ sb.append(i);
+ sb.append(':');
+ i=cal.get(Calendar.MINUTE);
+ if (i<10) sb.append('0');
+ sb.append(i);
+ sb.append(':');
+ i=cal.get(Calendar.SECOND);
+ if (i<10) sb.append('0');
+ sb.append(i);
+ i=cal.get(Calendar.MILLISECOND);
+ if (i != 0) {
+ sb.append('.');
+ if (i<100) sb.append('0');
+ if (i<10) sb.append('0');
+ sb.append(i);
+
+ // handle canonical format specifying fractional
+ // seconds shall not end in '0'. Given the slowness of
+ // integer div/mod, simply checking the last character
+ // is probably the fastest way to check.
+ int lastIdx = sb.length()-1;
+ if (sb.charAt(lastIdx)=='0') {
+ lastIdx--;
+ if (sb.charAt(lastIdx)=='0') {
+ lastIdx--;
+ }
+ sb.setLength(lastIdx+1);
+ }
+
+ }
+ sb.append('Z');
+
+ if (out != sb)
+ out.append(sb);
+
+ return cal;
+ }
+
}
\ No newline at end of file
Modified: lucene/dev/trunk/solr/src/common/org/apache/solr/common/util/FastWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/common/org/apache/solr/common/util/FastWriter.java?rev=979807&r1=979806&r2=979807&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/common/org/apache/solr/common/util/FastWriter.java (original)
+++ lucene/dev/trunk/solr/src/common/org/apache/solr/common/util/FastWriter.java Tue Jul 27 19:06:39 2010
@@ -27,9 +27,9 @@ public class FastWriter extends Writer {
// use default BUFSIZE of BufferedWriter so if we wrap that
// it won't cause double buffering.
private static final int BUFSIZE = 8192;
- private final Writer sink;
- private final char[] buf;
- private int pos;
+ protected final Writer sink;
+ protected final char[] buf;
+ protected int pos;
public FastWriter(Writer w) {
this(w, new char[BUFSIZE], 0);
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/core/SolrCore.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/core/SolrCore.java?rev=979807&r1=979806&r2=979807&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/core/SolrCore.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/core/SolrCore.java Tue Jul 27 19:06:39 2010
@@ -34,6 +34,7 @@ import org.apache.solr.handler.component
import org.apache.solr.highlight.DefaultSolrHighlighter;
import org.apache.solr.highlight.SolrHighlighter;
import org.apache.solr.request.*;
+import org.apache.solr.response.*;
import org.apache.solr.response.BinaryResponseWriter;
import org.apache.solr.response.JSONResponseWriter;
import org.apache.solr.response.PHPResponseWriter;
@@ -43,7 +44,6 @@ import org.apache.solr.response.QueryRes
import org.apache.solr.response.RawResponseWriter;
import org.apache.solr.response.RubyResponseWriter;
import org.apache.solr.response.SolrQueryResponse;
-import org.apache.solr.response.VelocityResponseWriter;
import org.apache.solr.response.XMLResponseWriter;
import org.apache.solr.schema.IndexSchema;
import org.apache.solr.search.QParserPlugin;
@@ -1408,6 +1408,7 @@ public final class SolrCore implements S
m.put("raw", new RawResponseWriter());
m.put("javabin", new BinaryResponseWriter());
m.put("velocity", new VelocityResponseWriter());
+ m.put("csv", new CSVResponseWriter());
DEFAULT_RESPONSE_WRITERS = Collections.unmodifiableMap(m);
}
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java?rev=979807&r1=979806&r2=979807&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java Tue Jul 27 19:06:39 2010
@@ -70,18 +70,18 @@ public class CSVRequestHandler extends C
abstract class CSVLoader extends ContentStreamLoader {
- static String SEPARATOR="separator";
- static String FIELDNAMES="fieldnames";
- static String HEADER="header";
- static String SKIP="skip";
- static String SKIPLINES="skipLines";
- static String MAP="map";
- static String TRIM="trim";
- static String EMPTY="keepEmpty";
- static String SPLIT="split";
- static String ENCAPSULATOR="encapsulator";
- static String ESCAPE="escape";
- static String OVERWRITE="overwrite";
+ public static final String SEPARATOR="separator";
+ public static final String FIELDNAMES="fieldnames";
+ public static final String HEADER="header";
+ public static final String SKIP="skip";
+ public static final String SKIPLINES="skipLines";
+ public static final String MAP="map";
+ public static final String TRIM="trim";
+ public static final String EMPTY="keepEmpty";
+ public static final String SPLIT="split";
+ public static final String ENCAPSULATOR="encapsulator";
+ public static final String ESCAPE="escape";
+ public static final String OVERWRITE="overwrite";
private static Pattern colonSplit = Pattern.compile(":");
private static Pattern commaSplit = Pattern.compile(",");
@@ -219,7 +219,7 @@ abstract class CSVLoader extends Content
// if only encapsulator or escape is set, disable the other escaping mechanism
if (encapsulator == null && escape != null) {
- strategy.setEncapsulator((char)-2); // TODO: add CSVStrategy.ENCAPSULATOR_DISABLED
+ strategy.setEncapsulator( CSVStrategy.ENCAPSULATOR_DISABLED);
strategy.setEscape(escape.charAt(0));
} else {
if (encapsulator != null) {
Added: lucene/dev/trunk/solr/src/java/org/apache/solr/response/CSVResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/response/CSVResponseWriter.java?rev=979807&view=auto
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/response/CSVResponseWriter.java (added)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/response/CSVResponseWriter.java Tue Jul 27 19:06:39 2010
@@ -0,0 +1,536 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.response;
+
+import org.apache.commons.csv.CSVPrinter;
+import org.apache.commons.csv.CSVStrategy;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Fieldable;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.DateUtil;
+import org.apache.solr.common.util.FastWriter;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.schema.StrField;
+import org.apache.solr.search.DocIterator;
+import org.apache.solr.search.DocList;
+import org.apache.solr.search.SolrIndexSearcher;
+
+import java.io.CharArrayWriter;
+import java.io.IOException;
+import java.io.Writer;
+import java.util.*;
+
+/**
+ * @version $Id$
+ */
+
+public class CSVResponseWriter implements QueryResponseWriter {
+
+ public void init(NamedList n) {
+ }
+
+ public void write(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
+ CSVWriter w = new CSVWriter(writer, req, rsp);
+ try {
+ w.writeResponse();
+ } finally {
+ w.close();
+ }
+ }
+
+ public String getContentType(SolrQueryRequest request, SolrQueryResponse response) {
+ // using the text/plain allows this to be viewed in the browser easily
+ return CONTENT_TYPE_TEXT_UTF8;
+ }
+}
+
+
+class CSVWriter extends TextResponseWriter {
+ static String SEPARATOR = "separator";
+ static String ENCAPSULATOR = "encapsulator";
+ static String ESCAPE = "escape";
+
+ static String CSV = "csv.";
+ static String CSV_SEPARATOR = CSV + SEPARATOR;
+ static String CSV_ENCAPSULATOR = CSV + ENCAPSULATOR;
+ static String CSV_ESCAPE = CSV + ESCAPE;
+
+ static String MV = CSV+"mv.";
+ static String MV_SEPARATOR = MV + SEPARATOR;
+ static String MV_ENCAPSULATOR = MV + ENCAPSULATOR;
+ static String MV_ESCAPE = MV + ESCAPE;
+
+ static String CSV_NULL = CSV + "null";
+ static String CSV_HEADER = CSV + "header";
+ static String CSV_NEWLINE = CSV + "newline";
+
+ char[] sharedCSVBuf = new char[8192];
+
+ // prevent each instance from creating it's own buffer
+ class CSVSharedBufPrinter extends CSVPrinter {
+ public CSVSharedBufPrinter(Writer out, CSVStrategy strategy) {
+ super(out, strategy);
+ super.buf = sharedCSVBuf;
+ }
+
+ public void reset() {
+ super.newLine = true;
+ // update our shared buf in case a new bigger one was allocated
+ sharedCSVBuf = super.buf;
+ }
+ }
+
+ // allows access to internal buf w/o copying it
+ static class OpenCharArrayWriter extends CharArrayWriter {
+ public char[] getInternalBuf() { return buf; }
+ }
+
+ // Writes all data to a char array,
+ // allows access to internal buffer, and allows fast resetting.
+ static class ResettableFastWriter extends FastWriter {
+ OpenCharArrayWriter cw = new OpenCharArrayWriter();
+ char[] result;
+ int resultLen;
+
+ public ResettableFastWriter() {
+ super(new OpenCharArrayWriter());
+ cw = (OpenCharArrayWriter)sink;
+ }
+
+ public void reset() {
+ cw.reset();
+ pos=0;
+ }
+
+ public void freeze() throws IOException {
+ if (cw.size() > 0) {
+ flush();
+ result = cw.getInternalBuf();
+ resultLen = cw.size();
+ } else {
+ result = buf;
+ resultLen = pos;
+ }
+ }
+
+ public int getFrozenSize() { return resultLen; }
+ public char[] getFrozenBuf() { return result; }
+ }
+
+
+ static class CSVField {
+ String name;
+ SchemaField sf;
+ CSVSharedBufPrinter mvPrinter; // printer used to encode multiple values in a single CSV value
+
+ // used to collect values
+ List<Fieldable> values = new ArrayList<Fieldable>(1); // low starting amount in case there are many fields
+ int tmp;
+ }
+
+ int pass;
+ Map<String,CSVField> csvFields = new LinkedHashMap<String,CSVField>();
+
+ Calendar cal; // for formatting date objects
+
+ CSVStrategy strategy; // strategy for encoding the fields of documents
+ CSVPrinter printer;
+ ResettableFastWriter mvWriter = new ResettableFastWriter(); // writer used for multi-valued fields
+
+ String NullValue;
+ boolean returnScore = false;
+
+
+ public CSVWriter(Writer writer, SolrQueryRequest req, SolrQueryResponse rsp) {
+ super(writer, req, rsp);
+ }
+
+ public void writeResponse() throws IOException {
+ SolrParams params = req.getParams();
+
+ strategy = new CSVStrategy(',', '"', CSVStrategy.COMMENTS_DISABLED, CSVStrategy.ESCAPE_DISABLED, false, false, false, true);
+ CSVStrategy strat = strategy;
+
+ String sep = params.get(CSV_SEPARATOR);
+ if (sep!=null) {
+ if (sep.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid separator:'"+sep+"'");
+ strat.setDelimiter(sep.charAt(0));
+ }
+
+ String nl = params.get(CSV_NEWLINE);
+ if (nl!=null) {
+ if (nl.length()==0) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid newline:'"+nl+"'");
+ strat.setPrinterNewline(nl);
+ }
+
+ String encapsulator = params.get(CSV_ENCAPSULATOR);
+ String escape = params.get(CSV_ESCAPE);
+ if (encapsulator!=null) {
+ if (encapsulator.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid encapsulator:'"+encapsulator+"'");
+ strat.setEncapsulator(encapsulator.charAt(0));
+ }
+
+ if (escape!=null) {
+ if (escape.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid escape:'"+escape+"'");
+ strat.setEscape(escape.charAt(0));
+ if (encapsulator == null) {
+ strat.setEncapsulator( CSVStrategy.ENCAPSULATOR_DISABLED);
+ }
+ }
+
+ if (strat.getEscape() == '\\') {
+ // If the escape is the standard backslash, then also enable
+ // unicode escapes (it's harmless since 'u' would not otherwise
+ // be escaped.
+ strat.setUnicodeEscapeInterpretation(true);
+ }
+
+ printer = new CSVPrinter(writer, strategy);
+
+
+ CSVStrategy mvStrategy = new CSVStrategy(strategy.getDelimiter(), CSVStrategy.ENCAPSULATOR_DISABLED, CSVStrategy.COMMENTS_DISABLED, '\\', false, false, false, false);
+ strat = mvStrategy;
+
+ sep = params.get(MV_SEPARATOR);
+ if (sep!=null) {
+ if (sep.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv separator:'"+sep+"'");
+ strat.setDelimiter(sep.charAt(0));
+ }
+
+ encapsulator = params.get(MV_ENCAPSULATOR);
+ escape = params.get(MV_ESCAPE);
+
+ if (encapsulator!=null) {
+ if (encapsulator.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv encapsulator:'"+encapsulator+"'");
+ strat.setEncapsulator(encapsulator.charAt(0));
+ if (escape == null) {
+ strat.setEscape(CSVStrategy.ESCAPE_DISABLED);
+ }
+ }
+
+ escape = params.get(MV_ESCAPE);
+ if (escape!=null) {
+ if (escape.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv escape:'"+escape+"'");
+ strat.setEscape(escape.charAt(0));
+ // encapsulator will already be disabled if it wasn't specified
+ }
+
+ returnScore = returnFields != null && returnFields.contains("score");
+ boolean needListOfFields = returnFields==null || returnFields.size()==0 || (returnFields.size()==1 && returnScore) || returnFields.contains("*");
+ Collection<String> fields = returnFields;
+
+ Object responseObj = rsp.getValues().get("response");
+ if (needListOfFields) {
+ if (responseObj instanceof SolrDocumentList) {
+ // get the list of fields from the SolrDocumentList
+ fields = new LinkedHashSet<String>();
+ for (SolrDocument sdoc: (SolrDocumentList)responseObj) {
+ fields.addAll(sdoc.getFieldNames());
+ }
+ } else {
+ // get the list of fields from the index
+ fields = req.getSearcher().getFieldNames();
+ }
+ if (returnScore) {
+ fields.add("score");
+ } else {
+ fields.remove("score");
+ }
+ }
+
+ CSVSharedBufPrinter csvPrinterMV = new CSVSharedBufPrinter(mvWriter, mvStrategy);
+
+ for (String field : fields) {
+ if (field.equals("score")) {
+ CSVField csvField = new CSVField();
+ csvField.name = "score";
+ csvFields.put("score", csvField);
+ continue;
+ }
+
+ SchemaField sf = schema.getFieldOrNull(field);
+ if (sf == null) {
+ FieldType ft = new StrField();
+ sf = new SchemaField(field, ft);
+ }
+
+ // if we got the list of fields from the index, only list stored fields
+ if (returnFields==null && sf != null && !sf.stored()) {
+ continue;
+ }
+
+ // check for per-field overrides
+ sep = params.get("f." + field + '.' + CSV_SEPARATOR);
+ encapsulator = params.get("f." + field + '.' + CSV_ENCAPSULATOR);
+ escape = params.get("f." + field + '.' + CSV_ESCAPE);
+
+ CSVSharedBufPrinter csvPrinter = csvPrinterMV;
+ if (sep != null || encapsulator != null || escape != null) {
+ // create a new strategy + printer if there were any per-field overrides
+ strat = (CSVStrategy)mvStrategy.clone();
+ if (sep!=null) {
+ if (sep.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv separator:'"+sep+"'");
+ strat.setDelimiter(sep.charAt(0));
+ }
+ if (encapsulator!=null) {
+ if (encapsulator.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv encapsulator:'"+encapsulator+"'");
+ strat.setEncapsulator(encapsulator.charAt(0));
+ if (escape == null) {
+ strat.setEscape(CSVStrategy.ESCAPE_DISABLED);
+ }
+ }
+ if (escape!=null) {
+ if (escape.length()!=1) throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid mv escape:'"+escape+"'");
+ strat.setEscape(escape.charAt(0));
+ if (encapsulator == null) {
+ strat.setEncapsulator(CSVStrategy.ENCAPSULATOR_DISABLED);
+ }
+ }
+ csvPrinter = new CSVSharedBufPrinter(mvWriter, strat);
+ }
+
+
+ CSVField csvField = new CSVField();
+ csvField.name = field;
+ csvField.sf = sf;
+ csvField.mvPrinter = csvPrinter;
+ csvFields.put(field, csvField);
+ }
+
+ NullValue = params.get(CSV_NULL, "");
+
+ if (params.getBool(CSV_HEADER, true)) {
+ for (CSVField csvField : csvFields.values()) {
+ printer.print(csvField.name);
+ }
+ printer.println();
+ }
+
+
+ if (responseObj instanceof DocList) {
+ writeDocList(null, (DocList)responseObj, null, null);
+ } else if (responseObj instanceof SolrDocumentList) {
+ writeSolrDocumentList(null, (SolrDocumentList)responseObj, null, null);
+ }
+
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (printer != null) printer.flush();
+ super.close();
+ }
+
+ @Override
+ public void writeNamedList(String name, NamedList val) throws IOException {
+ }
+
+ @Override
+ public void writeDoc(String name, Document doc, Set<String> returnFields, float score, boolean includeScore) throws IOException {
+ pass++;
+
+ for (Fieldable field: doc.getFields()) {
+ CSVField csvField = csvFields.get(field.name());
+ if (csvField == null) continue;
+ if (csvField.tmp != pass) {
+ csvField.tmp = pass;
+ csvField.values.clear();
+ }
+ csvField.values.add(field);
+ }
+
+ for (CSVField csvField : csvFields.values()) {
+ if (csvField.name.equals("score")) {
+ writeFloat("score", score);
+ continue;
+ }
+ if (csvField.tmp != pass) {
+ writeNull(csvField.name);
+ continue;
+ }
+
+ if (csvField.sf.multiValued() || csvField.values.size() > 1) {
+ mvWriter.reset();
+ csvField.mvPrinter.reset();
+ // switch the printer to use the multi-valued one
+ CSVPrinter tmp = printer;
+ printer = csvField.mvPrinter;
+ for (Fieldable fval : csvField.values) {
+ csvField.sf.getType().write(this, csvField.name, fval);
+ }
+ printer = tmp; // restore the original printer
+
+ mvWriter.freeze();
+ printer.print(mvWriter.getFrozenBuf(), 0, mvWriter.getFrozenSize(), true);
+ } else {
+ assert csvField.values.size() == 1;
+ csvField.sf.getType().write(this,csvField.name,csvField.values.get(0));
+ }
+ }
+
+ printer.println();
+ }
+
+ //NOTE: a document cannot currently contain another document
+ List tmpList;
+ @Override
+ public void writeSolrDocument(String name, SolrDocument doc, Set<String> returnFields, Map pseudoFields) throws IOException {
+ if (tmpList == null) {
+ tmpList = new ArrayList(1);
+ tmpList.add(null);
+ }
+
+ for (CSVField csvField : csvFields.values()) {
+ Object val = doc.getFieldValue(csvField.name);
+ int nVals = val instanceof Collection ? ((Collection)val).size() : (val==null ? 0 : 1);
+ if (nVals == 0) {
+ writeNull(csvField.name);
+ continue;
+ }
+
+ if ((csvField.sf != null && csvField.sf.multiValued()) || nVals > 1) {
+ Collection values;
+ // normalize to a collection
+ if (val instanceof Collection) {
+ values = (Collection)val;
+ } else {
+ tmpList.set(0, val);
+ values = tmpList;
+ }
+
+ mvWriter.reset();
+ csvField.mvPrinter.reset();
+ // switch the printer to use the multi-valued one
+ CSVPrinter tmp = printer;
+ printer = csvField.mvPrinter;
+ for (Object fval : values) {
+ writeVal(csvField.name, fval);
+ }
+ printer = tmp; // restore the original printer
+
+ mvWriter.freeze();
+ printer.print(mvWriter.getFrozenBuf(), 0, mvWriter.getFrozenSize(), true);
+
+ } else {
+ // normalize to first value
+ if (val instanceof Collection) {
+ Collection values = (Collection)val;
+ val = values.iterator().next();
+ }
+ writeVal(csvField.name, val);
+ }
+ }
+
+ printer.println();
+ }
+
+ @Override
+ public void writeDocList(String name, DocList ids, Set<String> fields, Map otherFields) throws IOException {
+ int sz=ids.size();
+ SolrIndexSearcher searcher = req.getSearcher();
+ DocIterator iterator = ids.iterator();
+ for (int i=0; i<sz; i++) {
+ int id = iterator.nextDoc();
+ Document doc = searcher.doc(id, fields);
+ writeDoc(null, doc, fields, (returnScore ? iterator.score() : 0.0f), returnScore);
+ }
+ }
+
+ Map scoreMap = new HashMap(1);
+ @Override
+ public void writeSolrDocumentList(String name, SolrDocumentList docs, Set<String> fields, Map otherFields) throws IOException {
+ for (SolrDocument doc : docs) {
+ writeSolrDocument(name, doc, fields, otherFields);
+ }
+ }
+
+ @Override
+ public void writeStr(String name, String val, boolean needsEscaping) throws IOException {
+ printer.print(val, needsEscaping);
+ }
+
+ @Override
+ public void writeMap(String name, Map val, boolean excludeOuter, boolean isFirstVal) throws IOException {
+ }
+
+ @Override
+ public void writeArray(String name, Object[] val) throws IOException {
+ }
+
+ @Override
+ public void writeArray(String name, Iterator val) throws IOException {
+ }
+
+ @Override
+ public void writeNull(String name) throws IOException {
+ printer.print(NullValue);
+ }
+
+ @Override
+ public void writeInt(String name, String val) throws IOException {
+ printer.print(val, false);
+ }
+
+ @Override
+ public void writeLong(String name, String val) throws IOException {
+ printer.print(val, false);
+ }
+
+ @Override
+ public void writeBool(String name, String val) throws IOException {
+ printer.print(val, false);
+ }
+
+ @Override
+ public void writeFloat(String name, String val) throws IOException {
+ printer.print(val, false);
+ }
+
+ @Override
+ public void writeDouble(String name, String val) throws IOException {
+ printer.print(val, false);
+ }
+
+ @Override
+ public void writeDate(String name, Date val) throws IOException {
+ StringBuilder sb = new StringBuilder(25);
+ cal = DateUtil.formatDate(val, cal, sb);
+ writeDate(name, sb.toString());
+ }
+
+ @Override
+ public void writeDate(String name, String val) throws IOException {
+ printer.print(val, false);
+ }
+
+ @Override
+ public void writeShort(String name, String val) throws IOException {
+ printer.print(val, false);
+ }
+
+ @Override
+ public void writeByte(String name, String val) throws IOException {
+ printer.print(val, false);
+ }
+}
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/response/CSVResponseWriter.java
------------------------------------------------------------------------------
svn:eol-style = native
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/response/CSVResponseWriter.java
------------------------------------------------------------------------------
svn:executable = *
Propchange: lucene/dev/trunk/solr/src/java/org/apache/solr/response/CSVResponseWriter.java
------------------------------------------------------------------------------
svn:keywords = Date Author Id Revision HeadURL
Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/util/SolrPluginUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/util/SolrPluginUtils.java?rev=979807&r1=979806&r2=979807&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/util/SolrPluginUtils.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/util/SolrPluginUtils.java Tue Jul 27 19:06:39 2010
@@ -205,7 +205,7 @@ public class SolrPluginUtils {
// TODO - should field order be maintained?
String[] flst = split(fl);
if (flst.length > 0 && !(flst.length==1 && flst[0].length()==0)) {
- Set<String> set = new HashSet<String>();
+ Set<String> set = new LinkedHashSet<String>();
for (String fname : flst) {
if("score".equalsIgnoreCase(fname))
flags |= SolrIndexSearcher.GET_SCORES;
Added: lucene/dev/trunk/solr/src/test/org/apache/solr/response/TestCSVResponseWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/response/TestCSVResponseWriter.java?rev=979807&view=auto
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/response/TestCSVResponseWriter.java (added)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/response/TestCSVResponseWriter.java Tue Jul 27 19:06:39 2010
@@ -0,0 +1,165 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.response;
+
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.util.DateUtil;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.util.SolrPluginUtils;
+import org.junit.*;
+
+import java.io.StringWriter;
+
+import static org.junit.Assert.*;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestCSVResponseWriter extends SolrTestCaseJ4 {
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("solrconfig.xml","schema12.xml");
+ createIndex();
+ }
+
+ public static void createIndex() {
+ assertU(adoc("id","1", "foo_i","-1", "foo_s","hi", "foo_l","12345678987654321", "foo_b","false", "foo_f","1.414","foo_d","-1.0E300","foo_dt","2000-01-02T03:04:05Z"));
+ assertU(adoc("id","2", "v_ss","hi", "v_ss","there", "v2_ss","nice", "v2_ss","output"));
+ assertU(commit());
+ }
+
+
+ @Test
+ public void testCSVOutput() throws Exception {
+ // test our basic types,and that fields come back in the requested order
+ assertEquals("id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt\n1,hi,-1,12345678987654321,false,1.414,-1.0E300,2000-01-02T03:04:05Z\n"
+ , h.query(req("q","id:1", "wt","csv", "fl","id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt")));
+
+ // test retrieving score, csv.header
+ assertEquals("1,0.0,hi\n"
+ , h.query(req("q","id:1^0", "wt","csv", "csv.header","false", "fl","id,score,foo_s")));
+
+ // test multivalued
+ assertEquals("2,\"hi,there\"\n"
+ , h.query(req("q","id:2", "wt","csv", "csv.header","false", "fl","id,v_ss")));
+
+ // test separator change
+ assertEquals("2|\"hi|there\"\n"
+ , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.separator","|", "fl","id,v_ss")));
+
+ // test mv separator change
+ assertEquals("2,hi|there\n"
+ , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "fl","id,v_ss")));
+
+ // test mv separator change for a single field
+ assertEquals("2,hi|there,nice:output\n"
+ , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "f.v2_ss.csv.separator",":", "fl","id,v_ss,v2_ss")));
+
+ // test retrieving fields from index
+ String result = h.query(req("q","*:*", "wt","csv", "csv.header","true", "fl","*,score"));
+ for (String field : "id,foo_s,foo_i,foo_l,foo_b,foo_f,foo_d,foo_dt,v_ss,v2_ss,score".split(",")) {
+ assertTrue(result.indexOf(field) >= 0);
+ }
+
+ // test null values
+ assertEquals("2,,hi|there\n"
+ , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "fl","id,foo_s,v_ss")));
+
+ // test alternate null value
+ assertEquals("2,NULL,hi|there\n"
+ , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.mv.separator","|", "csv.null","NULL","fl","id,foo_s,v_ss")));
+
+ // test alternate newline
+ assertEquals("2,\"hi,there\"\r\n"
+ , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.newline","\r\n", "fl","id,v_ss")));
+
+ // test alternate encapsulator
+ assertEquals("2,'hi,there'\n"
+ , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.encapsulator","'", "fl","id,v_ss")));
+
+ // test using escape instead of encapsulator
+ assertEquals("2,hi\\,there\n"
+ , h.query(req("q","id:2", "wt","csv", "csv.header","false", "csv.escape","\\", "fl","id,v_ss")));
+
+ // test multiple lines
+ assertEquals("1,,hi\n2,\"hi,there\",\n"
+ , h.query(req("q","id:[1 TO 2]", "wt","csv", "csv.header","false", "fl","id,v_ss,foo_s")));
+
+
+ // now test SolrDocumentList
+ SolrDocument d = new SolrDocument();
+ SolrDocument d1 = d;
+ d.addField("id","1");
+ d.addField("foo_i",-1);
+ d.addField("foo_s","hi");
+ d.addField("foo_l","12345678987654321L");
+ d.addField("foo_b",false);
+ d.addField("foo_f",1.414f);
+ d.addField("foo_d",-1.0E300);
+ d.addField("foo_dt", DateUtil.parseDate("2000-01-02T03:04:05Z"));
+ d.addField("score", "2.718");
+
+ d = new SolrDocument();
+ SolrDocument d2 = d;
+ d.addField("id","2");
+ d.addField("v_ss","hi");
+ d.addField("v_ss","there");
+ d.addField("v2_ss","nice");
+ d.addField("v2_ss","output");
+ d.addField("score", "89.83");
+
+ SolrDocumentList sdl = new SolrDocumentList();
+ sdl.add(d1);
+ sdl.add(d2);
+
+ SolrQueryRequest req = req("q","*:*");
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ rsp.add("response", sdl);
+ QueryResponseWriter w = new CSVResponseWriter();
+
+ SolrPluginUtils.setReturnFields("id,foo_s", rsp);
+ StringWriter buf = new StringWriter();
+ w.write(buf, req, rsp);
+ assertEquals("id,foo_s\n1,hi\n2,\n", buf.toString());
+
+ // try scores
+ SolrPluginUtils.setReturnFields("id,score,foo_s", rsp);
+ buf = new StringWriter();
+ w.write(buf, req, rsp);
+ assertEquals("id,score,foo_s\n1,2.718,hi\n2,89.83,\n", buf.toString());
+
+ // get field values from docs... should be ordered and not include score unless requested
+ SolrPluginUtils.setReturnFields("*", rsp);
+ buf = new StringWriter();
+ w.write(buf, req, rsp);
+ assertEquals("id,foo_i,foo_s,foo_l,foo_b,foo_f,foo_d,foo_dt,v_ss,v2_ss\n" +
+ "1,-1,hi,12345678987654321L,false,1.414,-1.0E300,2000-01-02T03:04:05Z,,\n" +
+ "2,,,,,,,,\"hi,there\",\"nice,output\"\n",
+ buf.toString());
+
+
+ // get field values and scores - just check that the scores are there... we don't guarantee where
+ SolrPluginUtils.setReturnFields("*,score", rsp);
+ buf = new StringWriter();
+ w.write(buf, req, rsp);
+ String s = buf.toString();
+ assertTrue(s.indexOf("score") >=0 && s.indexOf("2.718") > 0 && s.indexOf("89.83") > 0 );
+ }
+
+}
\ No newline at end of file