You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mv...@apache.org on 2012/03/06 20:12:31 UTC
svn commit: r1297654 - in
/lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler: CHANGES.txt
src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java
src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java
Author: mvg
Date: Tue Mar 6 19:12:31 2012
New Revision: 1297654
URL: http://svn.apache.org/viewvc?rev=1297654&view=rev
Log:
SOLR-3190: Minor improvements to SolrEntityProcessor.
Modified:
lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.txt
lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java
lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java
Modified: lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.txt?rev=1297654&r1=1297653&r2=1297654&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/CHANGES.txt Tue Mar 6 19:12:31 2012
@@ -15,6 +15,10 @@ New Features
----------------------
* SOLR-1499: Added SolrEntityProcessor that imports data from another Solr core or instance based on a specified query.
(Lance Norskog, Erik Hatcher, Pulkit Singhal, Ahmet Arslan, Luca Cavanna, Martijn van Groningen)
+ Additional Work:
+ SOLR-3190: Minor improvements to SolrEntityProcessor. Add more consistency between solr parameters
+ and parameters used in SolrEntityProcessor and ability to specify a custom HttpClient instance.
+ (Luca Cavanna via Martijn van Groningen)
Changes in Runtime Behavior
----------------------
Modified: lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java?rev=1297654&r1=1297653&r2=1297654&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java (original)
+++ lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrEntityProcessor.java Tue Mar 6 19:12:31 2012
@@ -16,16 +16,6 @@
*/
package org.apache.solr.handler.dataimport;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
-import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
-
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.Map;
-
import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
import org.apache.solr.client.solrj.SolrQuery;
@@ -36,9 +26,20 @@ import org.apache.solr.client.solrj.impl
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.params.CommonParams;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.SEVERE;
+import static org.apache.solr.handler.dataimport.DataImportHandlerException.wrapAndThrow;
+
/**
* <p>
* An implementation of {@link EntityProcessor} which fetches values from a
@@ -57,15 +58,8 @@ public class SolrEntityProcessor extends
public static final String SOLR_SERVER = "url";
public static final String QUERY = "query";
- /**
- * (format="javabin|xml") default is javabin
- */
- public static final String FORMAT = "format";
- public static final String ROWS = "rows";
- public static final String FIELDS = "fields";
- public static final String FQ = "fq";
public static final String TIMEOUT = "timeout";
-
+
public static final int TIMEOUT_SECS = 5 * 60; // 5 minutes
public static final int ROWS_DEFAULT = 50;
@@ -74,10 +68,22 @@ public class SolrEntityProcessor extends
private int rows = ROWS_DEFAULT;
private String[] filterQueries;
private String[] fields;
+ private String queryType;
private int timeout = TIMEOUT_SECS;
private boolean initDone = false;
-
+
+ /**
+ * Factory method that returns a {@link HttpClient} instance used for interfacing with a source Solr service.
+ * One can override this method to return a differently configured {@link HttpClient} instance.
+ * For example configure https and http authentication.
+ *
+ * @return a {@link HttpClient} instance used for interfacing with a source Solr service
+ */
+ protected HttpClient getHttpClient() {
+ return new HttpClient(new MultiThreadedHttpConnectionManager());
+ }
+
@Override
protected void firstInit(Context context) {
super.firstInit(context);
@@ -88,23 +94,21 @@ public class SolrEntityProcessor extends
throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
"SolrEntityProcessor: parameter 'url' is required");
}
- HttpClient client = new HttpClient(
- new MultiThreadedHttpConnectionManager());
+
+ HttpClient client = getHttpClient();
URL url = new URL(serverPath);
-
- if ("xml".equals(context.getResolvedEntityAttribute(FORMAT))) {
- solrServer = new CommonsHttpSolrServer(url, client,
- new XMLResponseParser(), false);
+ // (wt="javabin|xml") default is javabin
+ if ("xml".equals(context.getResolvedEntityAttribute(CommonParams.WT))) {
+ solrServer = new CommonsHttpSolrServer(url, client, new XMLResponseParser(), false);
LOG.info("using XMLResponseParser");
} else {
solrServer = new CommonsHttpSolrServer(url, client);
LOG.info("using BinaryResponseParser");
}
-
} catch (MalformedURLException e) {
throw new DataImportHandlerException(DataImportHandlerException.SEVERE, e);
}
-
+
this.queryString = context.getResolvedEntityAttribute(QUERY);
if (this.queryString == null) {
throw new DataImportHandlerException(
@@ -113,21 +117,21 @@ public class SolrEntityProcessor extends
);
}
- String rowsP = context.getResolvedEntityAttribute(ROWS);
+ String rowsP = context.getResolvedEntityAttribute(CommonParams.ROWS);
if (rowsP != null) {
rows = Integer.parseInt(rowsP);
}
- String fqAsString = context.getResolvedEntityAttribute(FQ);
+ String fqAsString = context.getResolvedEntityAttribute(CommonParams.FQ);
if (fqAsString != null) {
this.filterQueries = fqAsString.split(",");
}
- String fieldsAsString = context.getResolvedEntityAttribute(FIELDS);
+ String fieldsAsString = context.getResolvedEntityAttribute(CommonParams.FL);
if (fieldsAsString != null) {
this.fields = fieldsAsString.split(",");
}
-
+ this.queryType = context.getResolvedEntityAttribute(CommonParams.QT);
String timeoutAsString = context.getResolvedEntityAttribute(TIMEOUT);
if (timeoutAsString != null) {
this.timeout = Integer.parseInt(timeoutAsString);
@@ -180,6 +184,7 @@ public class SolrEntityProcessor extends
solrQuery.addField(field);
}
}
+ solrQuery.setQueryType(queryType);
solrQuery.setFilterQueries(filterQueries);
solrQuery.setTimeAllowed(timeout * 1000);
Modified: lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java?rev=1297654&r1=1297653&r2=1297654&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java (original)
+++ lucene/dev/branches/branch_3x/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestSolrEntityProcessorEndToEnd.java Tue Mar 6 19:12:31 2012
@@ -152,7 +152,7 @@ public class TestSolrEntityProcessorEndT
try {
addDocumentsToSolr(generateSolrDocuments(7));
- runFullImport(generateDIHConfig("query='*:*' fields='id' rows='2'"));
+ runFullImport(generateDIHConfig("query='*:*' fl='id' rows='2'"));
} catch (Exception e) {
LOG.error(e.getMessage(), e);
fail(e.getMessage());
@@ -178,6 +178,7 @@ public class TestSolrEntityProcessorEndT
try {
MockDataSource.setIterator("select * from x", DB_DOCS.iterator());
addDocumentsToSolr(SOLR_DOCS);
+ runFullImport(generateDIHConfig("query='*:*' rows='2' fl='id,desc' onError='skip'"));
runFullImport(DIH_CONFIG_TAGS_INNER_ENTITY);
} catch (Exception e) {
LOG.error(e.getMessage(), e);
@@ -218,6 +219,7 @@ public class TestSolrEntityProcessorEndT
assertQ(req("*:*"), "//result[@numFound='0']");
try {
+ runFullImport(generateDIHConfig("query='bogus:3' rows='2' fl='id,desc' onError='abort'"));
runFullImport(generateDIHConfig("query='bogus:3' rows='2' fields='id,desc' onError='abort'"));
} catch (Exception e) {
LOG.error(e.getMessage(), e);
@@ -233,6 +235,7 @@ public class TestSolrEntityProcessorEndT
List<Map<String,Object>> docList = generateSolrDocuments(numDocs);
try {
+ runFullImport(generateDIHConfig("query='*:*' rows='2' fl='id,desc' onError='skip'"));
addDocumentsToSolr(docList);
Map<String,String> map = new HashMap<String,String>();
map.put("rows", "50");