You are viewing a plain text version of this content. The canonical link for it is here.
Posted to solr-commits@lucene.apache.org by sh...@apache.org on 2008/12/11 10:05:39 UTC
svn commit: r725627 - in /lucene/solr/trunk/contrib/dataimporthandler:
CHANGES.txt src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
Author: shalin
Date: Thu Dec 11 01:05:39 2008
New Revision: 725627
URL: http://svn.apache.org/viewvc?rev=725627&view=rev
Log:
SOLR-846 -- Reduce memory consumption during delta import by removing keys when used
Modified:
lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
Modified: lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt?rev=725627&r1=725626&r2=725627&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/CHANGES.txt Thu Dec 11 01:05:39 2008
@@ -35,11 +35,13 @@
5. SOLR-887: A Transformer to strip HTML tags.
(Ahmed Hammad via shalin)
-6. SOLR-886: DataImportHandler should rollback when an import fails or it is aborted (shalin)
+6. SOLR-886: DataImportHandler should rollback when an import fails or it is aborted
+ (shalin)
Optimizations
----------------------
-
+1. SOLR-846: Reduce memory consumption during delta import by removing keys when used
+ (Ricky Leung, Noble Paul via shalin)
Bug Fixes
----------------------
Modified: lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java
URL: http://svn.apache.org/viewvc/lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java?rev=725627&r1=725626&r2=725627&view=diff
==============================================================================
--- lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java (original)
+++ lucene/solr/trunk/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/DocBuilder.java Thu Dec 11 01:05:39 2008
@@ -207,12 +207,16 @@
// Make sure that documents are not re-created
allPks.removeAll(deletedKeys);
}
+ deletedKeys = null;
statusMessages.put("Total Changed Documents", allPks.size());
- for (Map<String, Object> pk : allPks) {
- VariableResolverImpl vri = getVariableResolver(dataImporter);
- vri.addNamespace(DataConfig.IMPORTER_NS + ".delta", pk);
- buildDocument(vri, null, pk, root, true, null);
+ VariableResolverImpl vri = getVariableResolver(dataImporter);
+ Iterator<Map<String, Object>> pkIter = allPks.iterator();
+ while (pkIter.hasNext()) {
+ Map<String, Object> map = pkIter.next();
+ vri.addNamespace(DataConfig.IMPORTER_NS + ".delta", map);
+ buildDocument(vri, null, map, root, true, null);
+ pkIter.remove();
}
if (!stop.get()) {
@@ -223,8 +227,11 @@
private void deleteAll(Set<Map<String, Object>> deletedKeys) {
LOG.info("Deleting stale documents ");
- for (Map<String, Object> deletedKey : deletedKeys) {
- writer.deleteDoc(deletedKey.get(root.pk));
+ Iterator<Map<String, Object>> iter = deletedKeys.iterator();
+ while (iter.hasNext()) {
+ Map<String, Object> map = iter.next();
+ writer.deleteDoc(map.get(root.pk));
+ iter.remove();
}
}