svn commit: r637122 - in /lucene/nutch/trunk: CHANGES.txt src/java/org/apache/nutch/crawl/Crawl.java

[ab...@apache.org]

Author: ab
Date: Fri Mar 14 07:54:31 2008
New Revision: 637122

URL: http://svn.apache.org/viewvc?rev=637122&view=rev
Log:
NUTCH-601 Recrawling in existing crawl directory.

Modified:
    lucene/nutch/trunk/CHANGES.txt
    lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java

Modified: lucene/nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/CHANGES.txt?rev=637122&r1=637121&r2=637122&view=diff
==============================================================================
--- lucene/nutch/trunk/CHANGES.txt (original)
+++ lucene/nutch/trunk/CHANGES.txt Fri Mar 14 07:54:31 2008
@@ -223,6 +223,8 @@
 80. NUTCH-612 - URL filtering was disabled in Generator when invoked
     from Crawl (Susam Pal via ab)
 
+81. NUTCH-601 - Recrawling on existing crawl directory (Susam Pal via ab)
+
 
 Release 0.9 - 2007-04-02
 

Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java?rev=637122&r1=637121&r2=637122&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java Fri Mar 14 07:54:31 2008
@@ -82,9 +82,6 @@
     }
 
     FileSystem fs = FileSystem.get(job);
-    if (fs.exists(dir)) {
-      throw new RuntimeException(dir + " already exists.");
-    }
 
     if (LOG.isInfoEnabled()) {
       LOG.info("crawl started in: " + dir);
@@ -130,6 +127,18 @@
     }
     if (i > 0) {
       linkDbTool.invert(linkDb, segments, true, true, false); // invert links
+
+      // Delete old indexes
+      if (fs.exists(indexes)) {
+        LOG.info("Deleting old indexes: " + indexes);
+        fs.delete(indexes);
+      }
+
+      // Delete old index
+      if (fs.exists(index)) {
+        LOG.info("Deleting old merged index: " + index);
+        fs.delete(index);
+      }
 
       // index, dedup & merge
       indexer.index(indexes, crawlDb, linkDb, fs.listPaths(segments, HadoopFSUtil.getPassAllFilter()));