You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by eh...@apache.org on 2011/08/02 17:33:53 UTC
svn commit: r1153166 - in /lucene/dev/trunk/solr: CHANGES.txt
core/src/java/org/apache/solr/handler/CSVRequestHandler.java
core/src/test/org/apache/solr/handler/TestCSVLoader.java
Author: ehatcher
Date: Tue Aug 2 15:33:52 2011
New Revision: 1153166
URL: http://svn.apache.org/viewvc?rev=1153166&view=rev
Log:
SOLR-1032: add literal.field_name=val support to CSV update handler
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1153166&r1=1153165&r2=1153166&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Tue Aug 2 15:33:52 2011
@@ -161,6 +161,9 @@ New Features
* SOLR-2399: Updated Solr Admin interface. New look and feel with per core administration
and many new options. (Stefan Matheis via ryan)
+* SOLR-1032: CSV handler now supports "literal.field_name=value" parameters.
+ (Simon Rosenthal, ehatcher)
+
Optimizations
----------------------
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java?rev=1153166&r1=1153165&r2=1153166&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/CSVRequestHandler.java Tue Aug 2 15:33:52 2011
@@ -34,13 +34,18 @@ import org.apache.commons.io.IOUtils;
import java.util.regex.Pattern;
import java.util.List;
+import java.util.HashMap;
+import java.util.Iterator;
import java.io.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
*
*/
public class CSVRequestHandler extends ContentStreamHandlerBase {
+ public static Logger log = LoggerFactory.getLogger(CSVRequestHandler.class);
@Override
protected ContentStreamLoader newLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
@@ -83,16 +88,20 @@ abstract class CSVLoader extends Content
public static final String ENCAPSULATOR="encapsulator";
public static final String ESCAPE="escape";
public static final String OVERWRITE="overwrite";
+ public static final String LITERALS_PREFIX = "literal.";
private static Pattern colonSplit = Pattern.compile(":");
private static Pattern commaSplit = Pattern.compile(",");
+
+ public static Logger log = LoggerFactory.getLogger(CSVRequestHandler.class);
final IndexSchema schema;
final SolrParams params;
final CSVStrategy strategy;
final UpdateRequestProcessor processor;
-
+ // hashmap to save any literal fields and their values
+ HashMap <SchemaField, String> literals;
String[] fieldnames;
SchemaField[] fields;
CSVLoader.FieldAdder[] adders;
@@ -189,6 +198,7 @@ abstract class CSVLoader extends Content
this.processor = processor;
this.params = req.getParams();
schema = req.getSchema();
+ this.literals = new HashMap<SchemaField, String>();
templateAdd = new AddUpdateCommand(req);
templateAdd.overwrite=params.getBool(OVERWRITE,true);
@@ -289,7 +299,7 @@ abstract class CSVLoader extends Content
adders[i] = new CSVLoader.FieldMapperSingle(mapArgs[0], mapArgs[1], adders[i]);
}
}
-
+
if (params.getFieldBool(fname,TRIM,false)) {
adders[i] = new CSVLoader.FieldTrimmer(adders[i]);
}
@@ -306,6 +316,18 @@ abstract class CSVLoader extends Content
adders[i] = new CSVLoader.FieldSplitter(fstrat, adders[i]);
}
}
+ // look for any literal fields - literal.foo=xyzzy
+ Iterator<String> paramNames = params.getParameterNamesIterator();
+ while (paramNames.hasNext()) {
+ String pname = paramNames.next();
+ if (!pname.startsWith(LITERALS_PREFIX)) continue;
+
+ String name = pname.substring(LITERALS_PREFIX.length());
+ SchemaField sf = schema.getFieldOrNull(name);
+ if(sf == null)
+ throw new SolrException( SolrException.ErrorCode.BAD_REQUEST,"Invalid field name for literal:'"+ name +"'");
+ literals.put(sf, params.get(pname));
+ }
}
private void input_err(String msg, String[] line, int lineno) {
@@ -397,6 +419,13 @@ abstract class CSVLoader extends Content
adders[i].add(doc, line, i, val);
}
+ // add any literals
+ for (SchemaField sf : literals.keySet()) {
+ String fn = sf.getName();
+ String val = literals.get(sf);
+ doc.addField(fn, val);
+ }
+
template.solrDoc = doc;
processor.processAdd(template);
}
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java?rev=1153166&r1=1153165&r2=1153166&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/handler/TestCSVLoader.java Tue Aug 2 15:33:52 2011
@@ -121,6 +121,14 @@ public class TestCSVLoader extends SolrT
}
@Test
+ public void testLiteral() throws Exception {
+ makeFile("id\n100");
+ loadLocal("stream.file",filename,"commit","true", "literal.name","LITERAL_VALUE");
+ assertQ(req("*:*"),"//doc/str[@name='name'][.='LITERAL_VALUE']");
+ }
+
+
+ @Test
public void testCSV() throws Exception {
lrf.args.put(CommonParams.VERSION,"2.2");