You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jd...@apache.org on 2014/05/05 19:40:32 UTC
svn commit: r1592583 - in /lucene/dev/trunk/solr: ./
contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/
contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/
Author: jdyer
Date: Mon May 5 17:40:32 2014
New Revision: 1592583
URL: http://svn.apache.org/r1592583
Log:
SOLR-3671: DIHWriter fix
Added:
lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestWriterImpl.java (with props)
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java
lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java
lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1592583&r1=1592582&r2=1592583&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Mon May 5 17:40:32 2014
@@ -158,6 +158,8 @@ Other Changes
* SOLR-6022: Deprecate getAnalyzer() in IndexField and FieldType, and add getIndexAnalyzer().
(Ryan Ernst)
+* SOLR_3671: Fix DIHWriter interface usage so users may implement writers that output
+ documents to a location external to Solr (ex. a NoSql db). (Roman Chyla via James Dyer)
Build
---------------------
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java?rev=1592583&r1=1592582&r2=1592583&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImportHandler.java Mon May 5 17:40:32 2014
@@ -17,6 +17,7 @@
package org.apache.solr.handler.dataimport;
import static org.apache.solr.handler.dataimport.DataImporter.IMPORT_CMD;
+
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.CommonParams;
@@ -43,6 +44,8 @@ import org.apache.solr.util.plugin.SolrC
import java.util.*;
import java.io.StringReader;
+import java.lang.reflect.Constructor;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.InputSource;
@@ -76,6 +79,9 @@ public class DataImportHandler extends R
private String myName = "dataimport";
+ private static final String PARAM_WRITER_IMPL = "writerImpl";
+ private static final String DEFAULT_WRITER_NAME = "SolrWriter";
+
@Override
@SuppressWarnings("unchecked")
public void init(NamedList args) {
@@ -171,7 +177,7 @@ public class DataImportHandler extends R
req.getCore().getUpdateProcessingChain(params.get(UpdateParams.UPDATE_CHAIN));
UpdateRequestProcessor processor = processorChain.createProcessor(req, rsp);
SolrResourceLoader loader = req.getCore().getResourceLoader();
- SolrWriter sw = getSolrWriter(processor, loader, requestParams, req);
+ DIHWriter sw = getSolrWriter(processor, loader, requestParams, req);
if (requestParams.isDebug()) {
if (debugEnabled) {
@@ -224,23 +230,44 @@ public class DataImportHandler extends R
return result;
}
- private SolrWriter getSolrWriter(final UpdateRequestProcessor processor,
- final SolrResourceLoader loader, final RequestInfo requestParams, SolrQueryRequest req) {
-
- return new SolrWriter(processor, req) {
-
- @Override
- public boolean upload(SolrInputDocument document) {
- try {
- return super.upload(document);
- } catch (RuntimeException e) {
- LOG.error( "Exception while adding: " + document, e);
- return false;
- }
+ private DIHWriter getSolrWriter(final UpdateRequestProcessor processor,
+ final SolrResourceLoader loader, final RequestInfo requestParams,
+ SolrQueryRequest req) {
+ SolrParams reqParams = req.getParams();
+ String writerClassStr = null;
+ if (reqParams != null && reqParams.get(PARAM_WRITER_IMPL) != null) {
+ writerClassStr = (String) reqParams.get(PARAM_WRITER_IMPL);
+ }
+ DIHWriter writer;
+ if (writerClassStr != null
+ && !writerClassStr.equals(DEFAULT_WRITER_NAME)
+ && !writerClassStr.equals(DocBuilder.class.getPackage().getName() + "."
+ + DEFAULT_WRITER_NAME)) {
+ try {
+ @SuppressWarnings("unchecked")
+ Class<DIHWriter> writerClass = DocBuilder.loadClass(writerClassStr, req.getCore());
+ Constructor<DIHWriter> cnstr = writerClass.getConstructor(new Class[] {
+ UpdateRequestProcessor.class, SolrQueryRequest.class});
+ return cnstr.newInstance((Object) processor, (Object) req);
+ } catch (Exception e) {
+ throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
+ "Unable to load Writer implementation:" + writerClassStr, e);
}
- };
+ } else {
+ return new SolrWriter(processor, req) {
+ @Override
+ public boolean upload(SolrInputDocument document) {
+ try {
+ return super.upload(document);
+ } catch (RuntimeException e) {
+ LOG.error("Exception while adding: " + document, e);
+ return false;
+ }
+ }
+ };
+ }
}
-
+
@Override
@SuppressWarnings("unchecked")
public NamedList getStatistics() {
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=1592583&r1=1592582&r2=1592583&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java Mon May 5 17:40:32 2014
@@ -399,7 +399,7 @@ public class DataImporter {
return importLock.isLocked();
}
- public void doFullImport(SolrWriter writer, RequestInfo requestParams) {
+ public void doFullImport(DIHWriter writer, RequestInfo requestParams) {
LOG.info("Starting Full Import");
setStatus(Status.RUNNING_FULL_DUMP);
boolean success = false;
@@ -425,14 +425,14 @@ public class DataImporter {
}
- private void checkWritablePersistFile(SolrWriter writer, DIHProperties dihPropWriter) {
+ private void checkWritablePersistFile(DIHWriter writer, DIHProperties dihPropWriter) {
if (isDeltaImportSupported && !dihPropWriter.isWritable()) {
throw new DataImportHandlerException(SEVERE,
"Properties is not writable. Delta imports are supported by data config but will not work.");
}
}
- public void doDeltaImport(SolrWriter writer, RequestInfo requestParams) {
+ public void doDeltaImport(DIHWriter writer, RequestInfo requestParams) {
LOG.info("Starting Delta Import");
setStatus(Status.RUNNING_DELTA_DUMP);
boolean success = false;
@@ -457,7 +457,7 @@ public class DataImporter {
}
- public void runAsync(final RequestInfo reqParams, final SolrWriter sw) {
+ public void runAsync(final RequestInfo reqParams, final DIHWriter sw) {
new Thread() {
@Override
public void run() {
@@ -466,7 +466,7 @@ public class DataImporter {
}.start();
}
- void runCmd(RequestInfo reqParams, SolrWriter sw) {
+ void runCmd(RequestInfo reqParams, DIHWriter sw) {
String command = reqParams.getCommand();
if (command.equals(ABORT_CMD)) {
if (docBuilder != null) {
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=1592583&r1=1592582&r2=1592583&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java Mon May 5 17:40:32 2014
@@ -80,13 +80,10 @@ public class DocBuilder {
private Map<String, Object> persistedProperties;
private DIHProperties propWriter;
- private static final String PARAM_WRITER_IMPL = "writerImpl";
- private static final String DEFAULT_WRITER_NAME = "SolrWriter";
private DebugLogger debugLogger;
private final RequestInfo reqParams;
- @SuppressWarnings("unchecked")
- public DocBuilder(DataImporter dataImporter, SolrWriter solrWriter, DIHProperties propWriter, RequestInfo reqParams) {
+ public DocBuilder(DataImporter dataImporter, DIHWriter solrWriter, DIHProperties propWriter, RequestInfo reqParams) {
INSTANCE.set(this);
this.dataImporter = dataImporter;
this.reqParams = reqParams;
@@ -95,20 +92,7 @@ public class DocBuilder {
verboseDebug = reqParams.isDebug() && reqParams.getDebugInfo().verbose;
persistedProperties = propWriter.readIndexerProperties();
- String writerClassStr = null;
- if(reqParams!=null && reqParams.getRawParams() != null) {
- writerClassStr = (String) reqParams.getRawParams().get(PARAM_WRITER_IMPL);
- }
- if(writerClassStr != null && !writerClassStr.equals(DEFAULT_WRITER_NAME) && !writerClassStr.equals(DocBuilder.class.getPackage().getName() + "." + DEFAULT_WRITER_NAME)) {
- try {
- Class<DIHWriter> writerClass = loadClass(writerClassStr, dataImporter.getCore());
- this.writer = writerClass.newInstance();
- } catch (Exception e) {
- throw new DataImportHandlerException(DataImportHandlerException.SEVERE, "Unable to load Writer implementation:" + writerClassStr, e);
- }
- } else {
- writer = solrWriter;
- }
+ writer = solrWriter;
ContextImpl ctx = new ContextImpl(null, null, null, null, reqParams.getRawParams(), null, this);
writer.init(ctx);
}
@@ -905,9 +889,6 @@ public class DocBuilder {
return reqParams;
}
-
-
-
@SuppressWarnings("unchecked")
static Class loadClass(String name, SolrCore core) throws ClassNotFoundException {
try {
Added: lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestWriterImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestWriterImpl.java?rev=1592583&view=auto
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestWriterImpl.java (added)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestWriterImpl.java Mon May 5 17:40:32 2014
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.dataimport;
+
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.util.*;
+
+/**
+ * <p>
+ * Test for writerImpl paramater (to provide own SolrWriter)
+ * </p>
+ *
+ *
+ * @since solr 4.0
+ */
+public class TestWriterImpl extends AbstractDataImportHandlerTestCase {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ initCore("dataimport-nodatasource-solrconfig.xml", "dataimport-schema.xml");
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testDataConfigWithDataSource() throws Exception {
+ List rows = new ArrayList();
+ rows.add(createMap("id", "1", "desc", "one"));
+ rows.add(createMap("id", "2", "desc", "two"));
+ rows.add(createMap("id", "3", "desc", "break"));
+ rows.add(createMap("id", "4", "desc", "four"));
+
+ MockDataSource.setIterator("select * from x", rows.iterator());
+
+ Map extraParams = createMap("writerImpl", TestSolrWriter.class.getName(),
+ "commit", "true");
+ runFullImport(loadDataConfig("data-config-with-datasource.xml"),
+ extraParams);
+
+ assertQ(req("id:1"), "//*[@numFound='1']");
+ assertQ(req("id:2"), "//*[@numFound='1']");
+ assertQ(req("id:3"), "//*[@numFound='0']");
+ assertQ(req("id:4"), "//*[@numFound='1']");
+ }
+
+ public static class TestSolrWriter extends SolrWriter {
+
+ public TestSolrWriter(UpdateRequestProcessor processor, SolrQueryRequest req) {
+ super(processor, req);
+ }
+
+ @Override
+ public boolean upload(SolrInputDocument doc) {
+ if (doc.getField("desc").getFirstValue().equals("break")) {
+ return false;
+ }
+ return super.upload(doc);
+ }
+
+ }
+
+}