You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2014/08/11 16:35:40 UTC
svn commit: r1617296 - in /lucene/dev/branches/branch_4x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/handler/loader/ solr/solrj/
solr/solrj/src/java/org/apache/solr/common/util/
solr/solrj/src/test/org/apache/solr/common/util/
Author: noble
Date: Mon Aug 11 14:35:39 2014
New Revision: 1617296
URL: http://svn.apache.org/r1617296
Log:
SOLR-6304 JsonLoader should be able to flatten an input JSON to multiple docs
Added:
lucene/dev/branches/branch_4x/solr/solrj/src/java/org/apache/solr/common/util/JsonRecordReader.java
- copied unchanged from r1617287, lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/util/JsonRecordReader.java
lucene/dev/branches/branch_4x/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java
- copied unchanged from r1617287, lucene/dev/trunk/solr/solrj/src/test/org/apache/solr/common/util/TestJsonRecordReader.java
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/solr/ (props changed)
lucene/dev/branches/branch_4x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_4x/solr/core/ (props changed)
lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java
lucene/dev/branches/branch_4x/solr/solrj/ (props changed)
Modified: lucene/dev/branches/branch_4x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/CHANGES.txt?rev=1617296&r1=1617295&r2=1617296&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/solr/CHANGES.txt Mon Aug 11 14:35:39 2014
@@ -108,6 +108,9 @@ New Features
* SOLR-6283: Add support for Interval Faceting in SolrJ. (Tomás Fernández Löbbe)
+* SOLR-6304 : JsonLoader should be able to flatten an input JSON to multiple docs (Noble Paul)
+
+
Bug Fixes
----------------------
Modified: lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java?rev=1617296&r1=1617295&r2=1617296&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java (original)
+++ lucene/dev/branches/branch_4x/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java Mon Aug 11 14:35:39 2014
@@ -20,6 +20,7 @@ import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -28,6 +29,7 @@ import java.util.Map;
import org.apache.commons.io.IOUtils;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.params.UpdateParams;
+import org.apache.solr.common.util.JsonRecordReader;
import org.noggit.JSONParser;
import org.noggit.ObjectBuilder;
import org.apache.solr.common.SolrException;
@@ -63,7 +65,7 @@ public class JsonLoader extends ContentS
@Override
public void load(SolrQueryRequest req, SolrQueryResponse rsp,
ContentStream stream, UpdateRequestProcessor processor) throws Exception {
- new SingleThreadedJsonLoader(req,processor).load(req, rsp, stream, processor);
+ new SingleThreadedJsonLoader(req,rsp,processor).load(req, rsp, stream, processor);
}
@@ -71,14 +73,16 @@ public class JsonLoader extends ContentS
protected final UpdateRequestProcessor processor;
protected final SolrQueryRequest req;
+ protected SolrQueryResponse rsp;
protected JSONParser parser;
protected final int commitWithin;
protected final boolean overwrite;
- public SingleThreadedJsonLoader(SolrQueryRequest req, UpdateRequestProcessor processor) {
+ public SingleThreadedJsonLoader(SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor processor) {
this.processor = processor;
this.req = req;
-
+ this.rsp = rsp;
+
commitWithin = req.getParams().getInt(UpdateParams.COMMIT_WITHIN, -1);
overwrite = req.getParams().getBool(UpdateParams.OVERWRITE, true);
}
@@ -110,7 +114,13 @@ public class JsonLoader extends ContentS
void processUpdate() throws IOException
{
if("false".equals( req.getParams().get("json.command"))){
- handleStreamingSingleDocs();
+
+ String split = req.getParams().get("split");
+ if(split != null){
+ handleSplitMode(split);
+ } else {
+ handleStreamingSingleDocs();
+ }
return;
}
int ev = parser.nextEvent();
@@ -182,6 +192,39 @@ public class JsonLoader extends ContentS
}
}
+ private void handleSplitMode(String split) throws IOException {
+ String[] fields = req.getParams().getParams("f");
+ req.getCore().getLatestSchema().getDefaultSearchFieldName();
+ final boolean echo = "true".equals( req.getParams().get("echo"));
+ JsonRecordReader jsonRecordReader = JsonRecordReader.getInst(split, Arrays.asList(fields));
+ jsonRecordReader.streamRecords(parser,new JsonRecordReader.Handler() {
+ ArrayList docs =null;
+ @Override
+ public void handle(Map<String, Object> record, String path) {
+ if(echo){
+ if(docs ==null) {
+ docs = new ArrayList();
+ rsp.add("docs",docs);
+ }
+ docs.add(record);
+ } else {
+ AddUpdateCommand cmd = new AddUpdateCommand(req);
+ cmd.commitWithin = commitWithin;
+ cmd.overwrite = overwrite;
+ cmd.solrDoc = new SolrInputDocument();
+ for (Map.Entry<String, Object> entry : record.entrySet()) {
+ cmd.solrDoc.setField(entry.getKey(),entry.getValue());
+ }
+ try {
+ processor.processAdd(cmd);
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "error inserting doc",e);
+ }
+ }
+ }
+ });
+ }
+
private void handleStreamingSingleDocs() throws IOException
{
while( true ) {