You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by jn...@apache.org on 2010/03/05 11:09:08 UTC
svn commit: r919358 - in /lucene/nutch/trunk: CHANGES.txt
src/java/org/apache/nutch/indexer/solr/SolrIndexer.java
src/java/org/apache/nutch/indexer/solr/SolrWriter.java
Author: jnioche
Date: Fri Mar 5 10:09:08 2010
New Revision: 919358
URL: http://svn.apache.org/viewvc?rev=919358&view=rev
Log:
NUTCH-799 SOLRIndexer to commit once all reducers have finished
Modified:
lucene/nutch/trunk/CHANGES.txt
lucene/nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrIndexer.java
lucene/nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrWriter.java
Modified: lucene/nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/CHANGES.txt?rev=919358&r1=919357&r2=919358&view=diff
==============================================================================
--- lucene/nutch/trunk/CHANGES.txt (original)
+++ lucene/nutch/trunk/CHANGES.txt Fri Mar 5 10:09:08 2010
@@ -2,6 +2,8 @@
Unreleased Changes
+* NUTCH-799 SOLRIndexer to commit once all reducers have finished (jnioche)
+
* NUTCH-782 Ability to order htmlparsefilters (jnioche)
* NUTCH-719 fetchQueues.totalSize incorrect in Fetcher (Steven Denny via jnioche)
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrIndexer.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrIndexer.java?rev=919358&r1=919357&r2=919358&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrIndexer.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrIndexer.java Fri Mar 5 10:09:08 2010
@@ -37,6 +37,8 @@
import org.apache.nutch.indexer.NutchIndexWriterFactory;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
+import org.apache.solr.client.solrj.SolrServer;
+import org.apache.solr.client.solrj.impl.CommonsHttpSolrServer;
public class SolrIndexer extends Configured implements Tool {
@@ -71,6 +73,12 @@
FileOutputFormat.setOutputPath(job, tmp);
try {
JobClient.runJob(job);
+ // do the commits once and for all the reducers in one go
+ SolrServer solr = new CommonsHttpSolrServer(solrUrl);
+ solr.commit();
+ }
+ catch (Exception e){
+ LOG.error(e);
} finally {
FileSystem.get(job).delete(tmp, true);
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrWriter.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrWriter.java?rev=919358&r1=919357&r2=919358&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrWriter.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/indexer/solr/SolrWriter.java Fri Mar 5 10:09:08 2010
@@ -74,7 +74,7 @@
solr.add(inputDocs);
inputDocs.clear();
}
- solr.commit();
+ // solr.commit();
} catch (final SolrServerException e) {
throw makeIOException(e);
}