You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by sh...@apache.org on 2009/01/27 08:49:22 UTC
svn commit: r738020 - in /lucene/solr/trunk/contrib/dataimporthandler:
CHANGES.txt src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java
Author: shalin
Date: Tue Jan 27 07:49:22 2009
New Revision: 738020
URL: http://svn.apache.org/viewvc?rev=738020&view=rev
Log:
SOLR-974 -- DataImportHandler skips commit if no data has been updated
Modified:
lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java
Modified: lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt?rev=738020&r1=738019&r2=738020&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt Tue Jan 27 07:49:22 2009
@@ -61,6 +61,9 @@
1. SOLR-846: Reduce memory consumption during delta import by removing keys when used
(Ricky Leung, Noble Paul via shalin)
+2. SOLR-974: DataImportHandler skips commit if no data has been updated.
+ (Wojtek Piaseczny, shalin)
+
Bug Fixes
----------------------
1. SOLR-800: Deep copy collections to avoid ConcurrentModificationException in XPathEntityprocessor while streaming
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=738020&r1=738019&r2=738020&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java Tue Jan 27 07:49:22 2009
@@ -175,8 +175,15 @@
}
}
} else {
- // Finished operation normally, commit now
- commit();
+ // Do not commit unnecessarily if this is a delta-import and no documents were created or deleted
+ if (!requestParameters.clean) {
+ if (importStatistics.docCount.get() > 0 || importStatistics.deletedDocCount.get() > 0) {
+ commit();
+ }
+ } else {
+ // Finished operation normally, commit now
+ commit();
+ }
if (document.onImportEnd != null) {
invokeEventListener(document.onImportEnd);
}
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java?rev=738020&r1=738019&r2=738020&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/test/java/org/apache/solr/handler/dataimport/TestDocBuilder.java Tue Jan 27 07:49:22 2009
@@ -21,10 +21,7 @@
import org.junit.Assert;
import org.junit.Test;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
/**
* <p>
@@ -71,6 +68,31 @@
}
@Test
+ public void testDeltaImportNoRows_MustNotCommit() {
+ try {
+ DataImporter di = new DataImporter();
+ di.loadDataConfig(dc_deltaConfig);
+ DataConfig cfg = di.getConfig();
+ DataConfig.Entity ent = cfg.document.entities.get(0);
+ MockDataSource.setIterator("select * from x", new ArrayList().iterator());
+ MockDataSource.setIterator("select id from x", new ArrayList().iterator());
+ ent.dataSrc = new MockDataSource();
+ ent.isDocRoot = true;
+ DataImporter.RequestParams rp = new DataImporter.RequestParams(createMap("command", "delta-import"));
+ SolrWriterImpl swi = new SolrWriterImpl();
+ di.runCmd(rp, swi);
+ Assert.assertEquals(Boolean.FALSE, swi.deleteAllCalled);
+ Assert.assertEquals(Boolean.FALSE, swi.commitCalled);
+ Assert.assertEquals(0, swi.docs.size());
+ Assert.assertEquals(1, di.getDocBuilder().importStatistics.queryCount.get());
+ Assert.assertEquals(0, di.getDocBuilder().importStatistics.docCount.get());
+ Assert.assertEquals(0, di.getDocBuilder().importStatistics.rowsCount.get());
+ } finally {
+ MockDataSource.clearCache();
+ }
+ }
+
+ @Test
public void singleEntityOneRow() {
try {
DataImporter di = new DataImporter();
@@ -153,9 +175,9 @@
static class SolrWriterImpl extends SolrWriter {
List<SolrInputDocument> docs = new ArrayList<SolrInputDocument>();
- Boolean deleteAllCalled;
+ Boolean deleteAllCalled = Boolean.FALSE;
- Boolean commitCalled;
+ Boolean commitCalled = Boolean.FALSE;
public SolrWriterImpl() {
super(null, ".");
@@ -186,4 +208,12 @@
+ " <field column=\"desc\" name=\"desc_s\" />" + " </entity>\n"
+ " </document>\n" + "</dataConfig>";
+ public static final String dc_deltaConfig = "<dataConfig>\n"
+ + " <document name=\"X\" >\n"
+ + " <entity name=\"x\" query=\"select * from x\" deltaQuery=\"select id from x\">\n"
+ + " <field column=\"id\"/>\n"
+ + " <field column=\"desc\"/>\n"
+ + " <field column=\"desc\" name=\"desc_s\" />" + " </entity>\n"
+ + " </document>\n" + "</dataConfig>";
+
}