You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by eh...@apache.org on 2014/07/25 14:19:49 UTC
svn commit: r1613406 - in /lucene/dev/trunk/solr: ./
contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/
contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/
contrib/dataimporthandler/src/test/org/apache/solr...
Author: ehatcher
Date: Fri Jul 25 12:19:48 2014
New Revision: 1613406
URL: http://svn.apache.org/r1613406
Log:
SOLR-3622, SOLR-5847, SOLR-6194, SOLR-6269: Several DIH fixes/improvements
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java
lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java
lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java
lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java
lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java
lucene/dev/trunk/solr/webapp/web/js/scripts/dataimport.js
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1613406&r1=1613405&r2=1613406&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Fri Jul 25 12:19:48 2014
@@ -222,6 +222,8 @@ Bug Fixes
datatypes of options people can specify, additional error handling of duplicated/unidentified
options has also been added. (Maciej Zasada, hossman)
+* SOLR-5847: Fixed data import abort button in admin UI. (ehatcher)
+
Optimizations
---------------------
@@ -289,6 +291,15 @@ Other Changes
* SOLR-6274: UpdateShardHandler should log the params used to configure it's
HttpClient. (Ramkumar Aiyengar via Mark Miller)
+* SOLR-6194: Opened up "public" access to DataSource, DocBuilder, and EntityProcessorWrapper
+ in DIH. (Aaron LaBella via ehatcher)
+
+* SOLR-6269: Renamed "rollback" to "error" in DIH internals, including renaming onRollback
+ to onError introduced in SOLR-6258. (ehatcher)
+
+* SOLR-3622: When using DIH in SolrCloud-mode, rollback will no longer be called when
+ an error occurs. (ehatcher)
+
================== 4.9.0 ==================
Versions of Major Components
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java?rev=1613406&r1=1613405&r2=1613406&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/ContextImpl.java Fri Jul 25 12:19:48 2014
@@ -54,6 +54,8 @@ public class ContextImpl extends Context
DocBuilder docBuilder;
+ Exception lastException = null;
+
public ContextImpl(EntityProcessorWrapper epw, VariableResolver resolver,
DataSource ds, String currProcess,
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java?rev=1613406&r1=1613405&r2=1613406&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DataImporter.java Fri Jul 25 12:19:48 2014
@@ -179,7 +179,11 @@ public class DataImporter {
* Used by tests
*/
public void loadAndInit(String configStr) {
- config = loadDataConfig(new InputSource(new StringReader(configStr)));
+ config = loadDataConfig(new InputSource(new StringReader(configStr)));
+ }
+
+ public void loadAndInit(InputSource configFile) {
+ config = loadDataConfig(configFile);
}
public DIHConfiguration loadDataConfig(InputSource configFile) {
@@ -349,7 +353,7 @@ public class DataImporter {
return store.get(key);
}
- DataSource getDataSourceInstance(Entity key, String name, Context ctx) {
+ public DataSource getDataSourceInstance(Entity key, String name, Context ctx) {
Map<String,String> p = requestLevelDataSourceProps.get(name);
if (p == null)
p = config.getDataSources().get(name);
@@ -404,7 +408,6 @@ public class DataImporter {
public void doFullImport(DIHWriter writer, RequestInfo requestParams) {
LOG.info("Starting Full Import");
setStatus(Status.RUNNING_FULL_DUMP);
- boolean success = false;
try {
DIHProperties dihPropWriter = createPropertyWriter();
setIndexStartTime(dihPropWriter.getCurrentTimestamp());
@@ -413,14 +416,10 @@ public class DataImporter {
docBuilder.execute();
if (!requestParams.isDebug())
cumulativeStatistics.add(docBuilder.importStatistics);
- success = true;
} catch (Exception e) {
SolrException.log(LOG, "Full Import failed", e);
+ docBuilder.handleError("Full Import failed", e);
} finally {
- if (!success) {
- docBuilder.rollback();
- }
-
setStatus(Status.IDLE);
DocBuilder.INSTANCE.set(null);
}
@@ -437,7 +436,6 @@ public class DataImporter {
public void doDeltaImport(DIHWriter writer, RequestInfo requestParams) {
LOG.info("Starting Delta Import");
setStatus(Status.RUNNING_DELTA_DUMP);
- boolean success = false;
try {
DIHProperties dihPropWriter = createPropertyWriter();
setIndexStartTime(dihPropWriter.getCurrentTimestamp());
@@ -446,13 +444,10 @@ public class DataImporter {
docBuilder.execute();
if (!requestParams.isDebug())
cumulativeStatistics.add(docBuilder.importStatistics);
- success = true;
} catch (Exception e) {
LOG.error("Delta Import Failed", e);
+ docBuilder.handleError("Delta Import Failed", e);
} finally {
- if (!success) {
- docBuilder.rollback();
- }
setStatus(Status.IDLE);
DocBuilder.INSTANCE.set(null);
}
@@ -510,10 +505,15 @@ public class DataImporter {
}
- DocBuilder getDocBuilder() {
+ public DocBuilder getDocBuilder() {
return docBuilder;
}
-
+
+ public DocBuilder getDocBuilder(DIHWriter writer, RequestInfo requestParams) {
+ DIHProperties dihPropWriter = createPropertyWriter();
+ return new DocBuilder(this, writer, dihPropWriter, requestParams);
+ }
+
Map<String, Evaluator> getEvaluators() {
return getEvaluators(config.getFunctions());
}
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=1613406&r1=1613405&r2=1613406&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/DocBuilder.java Fri Jul 25 12:19:48 2014
@@ -94,7 +94,9 @@ public class DocBuilder {
writer = solrWriter;
ContextImpl ctx = new ContextImpl(null, null, null, null, reqParams.getRawParams(), null, this);
- writer.init(ctx);
+ if (writer != null) {
+ writer.init(ctx);
+ }
}
@@ -146,25 +148,31 @@ public class DocBuilder {
return null;
}
}
-
private void invokeEventListener(String className) {
+ invokeEventListener(className, null);
+ }
+
+
+ private void invokeEventListener(String className, Exception lastException) {
try {
EventListener listener = (EventListener) loadClass(className, dataImporter.getCore()).newInstance();
- notifyListener(listener);
+ notifyListener(listener, lastException);
} catch (Exception e) {
wrapAndThrow(SEVERE, e, "Unable to load class : " + className);
}
}
- private void notifyListener(EventListener listener) {
+ private void notifyListener(EventListener listener, Exception lastException) {
String currentProcess;
if (dataImporter.getStatus() == DataImporter.Status.RUNNING_DELTA_DUMP) {
currentProcess = Context.DELTA_DUMP;
} else {
currentProcess = Context.FULL_DUMP;
}
- listener.onEvent(new ContextImpl(null, getVariableResolver(), null, currentProcess, session, null, this));
+ ContextImpl ctx = new ContextImpl(null, getVariableResolver(), null, currentProcess, session, null, this);
+ ctx.lastException = lastException;
+ listener.onEvent(ctx);
}
@SuppressWarnings("unchecked")
@@ -234,7 +242,7 @@ public class DocBuilder {
if (stop.get()) {
// Dont commit if aborted using command=abort
statusMessages.put("Aborted", new SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).format(new Date()));
- rollback();
+ handleError("Aborted", null);
} else {
// Do not commit unnecessarily if this is a delta-import and no documents were created or deleted
if (!reqParams.isClean()) {
@@ -305,12 +313,15 @@ public class DocBuilder {
}
}
- void rollback() {
- writer.rollback();
- statusMessages.put("", "Indexing failed. Rolled back all changes.");
- addStatusMessage("Rolledback");
- if ((config != null) && (config.getOnRollback() != null)) {
- invokeEventListener(config.getOnRollback());
+ void handleError(String message, Exception e) {
+ if (!dataImporter.getCore().getCoreDescriptor().getCoreContainer().isZooKeeperAware()) {
+ writer.rollback();
+ }
+
+ statusMessages.put(message, "Indexing error");
+ addStatusMessage(message);
+ if ((config != null) && (config.getOnError() != null)) {
+ invokeEventListener(config.getOnError(), e);
}
}
@@ -688,7 +699,7 @@ public class DocBuilder {
}
}
- private EntityProcessorWrapper getEntityProcessorWrapper(Entity entity) {
+ public EntityProcessorWrapper getEntityProcessorWrapper(Entity entity) {
EntityProcessor entityProcessor = null;
if (entity.getProcessorName() == null) {
entityProcessor = new SqlEntityProcessor();
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java?rev=1613406&r1=1613405&r2=1613406&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/SolrWriter.java Fri Jul 25 12:19:48 2014
@@ -117,7 +117,7 @@ public class SolrWriter extends DIHWrite
RollbackUpdateCommand rollback = new RollbackUpdateCommand(req);
processor.processRollback(rollback);
} catch (Exception e) {
- log.error("Exception while solr rollback.", e);
+ log.error("Exception during rollback command.", e);
}
}
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java?rev=1613406&r1=1613405&r2=1613406&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/java/org/apache/solr/handler/dataimport/config/DIHConfiguration.java Fri Jul 25 12:19:48 2014
@@ -56,7 +56,7 @@ public class DIHConfiguration {
private final List<Entity> entities;
private final String onImportStart;
private final String onImportEnd;
- private final String onRollback;
+ private final String onError;
private final List<Map<String, String>> functions;
private final Script script;
private final Map<String, Map<String,String>> dataSources;
@@ -72,7 +72,7 @@ public class DIHConfiguration {
this.deleteQuery = ConfigParseUtil.getStringAttribute(element, "deleteQuery", null);
this.onImportStart = ConfigParseUtil.getStringAttribute(element, "onImportStart", null);
this.onImportEnd = ConfigParseUtil.getStringAttribute(element, "onImportEnd", null);
- this.onRollback = ConfigParseUtil.getStringAttribute(element, "onRollback", null);
+ this.onError = ConfigParseUtil.getStringAttribute(element, "onError", null);
List<Entity> modEntities = new ArrayList<>();
List<Element> l = ConfigParseUtil.getChildNodes(element, "entity");
boolean docRootFound = false;
@@ -165,8 +165,8 @@ public class DIHConfiguration {
public String getOnImportEnd() {
return onImportEnd;
}
- public String getOnRollback() {
- return onRollback;
+ public String getOnError() {
+ return onError;
}
public List<Map<String,String>> getFunctions() {
return functions;
Modified: lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java?rev=1613406&r1=1613405&r2=1613406&view=diff
==============================================================================
--- lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java (original)
+++ lucene/dev/trunk/solr/contrib/dataimporthandler/src/test/org/apache/solr/handler/dataimport/TestDocBuilder2.java Fri Jul 25 12:19:48 2014
@@ -76,14 +76,15 @@ public class TestDocBuilder2 extends Abs
}
@Test
- public void testRollbackHandler() throws Exception {
+ public void testErrorHandler() throws Exception {
List rows = new ArrayList();
- rows.add(createMap("id", "1", "FORCE_ROLLBACK", "true"));
+ rows.add(createMap("id", "1", "FORCE_ERROR", "true"));
MockDataSource.setIterator("select * from x", rows.iterator());
- runFullImport(dataConfigWithRollbackHandler);
+ runFullImport(dataConfigWithErrorHandler);
- assertTrue("Rollback event listener was not called", RollbackEventListener.executed);
+ assertTrue("Error event listener was not called", ErrorEventListener.executed);
+ assertTrue(ErrorEventListener.lastException.getMessage().contains("ForcedException"));
}
@Test
@@ -316,12 +317,14 @@ public class TestDocBuilder2 extends Abs
}
}
- public static class RollbackEventListener implements EventListener {
+ public static class ErrorEventListener implements EventListener {
public static boolean executed = false;
+ public static Exception lastException = null;
@Override
public void onEvent(Context ctx) {
executed = true;
+ lastException = ((ContextImpl) ctx).lastException;
}
}
@@ -377,11 +380,11 @@ public class TestDocBuilder2 extends Abs
" </document>\n" +
"</dataConfig>";
- private final String dataConfigWithRollbackHandler = "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
- " <document onRollback=\"TestDocBuilder2$RollbackEventListener\">\n" +
+ private final String dataConfigWithErrorHandler = "<dataConfig> <dataSource type=\"MockDataSource\"/>\n" +
+ " <document onError=\"TestDocBuilder2$ErrorEventListener\">\n" +
" <entity name=\"books\" query=\"select * from x\" transformer=\"TestDocBuilder2$ForcedExceptionTransformer\">\n" +
" <field column=\"id\" />\n" +
- " <field column=\"FORCE_ROLLBACK\" />\n" +
+ " <field column=\"FORCE_ERROR\" />\n" +
" </entity>\n" +
" </document>\n" +
"</dataConfig>";
Modified: lucene/dev/trunk/solr/webapp/web/js/scripts/dataimport.js
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/webapp/web/js/scripts/dataimport.js?rev=1613406&r1=1613405&r2=1613406&view=diff
==============================================================================
--- lucene/dev/trunk/solr/webapp/web/js/scripts/dataimport.js (original)
+++ lucene/dev/trunk/solr/webapp/web/js/scripts/dataimport.js Fri Jul 25 12:19:48 2014
@@ -350,7 +350,7 @@ sammy.get
{
url : handler_url + '?command=abort&wt=json',
dataType : 'json',
- type: 'POST',
+ type: 'GET',
context: $( this ),
beforeSend : function( xhr, settings )
{