You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by ab...@apache.org on 2007/03/07 22:59:09 UTC
svn commit: r515791 - in /lucene/nutch/trunk: ./ lib/
lib/native/Linux-i386-32/ src/java/org/apache/nutch/crawl/
src/java/org/apache/nutch/fetcher/ src/java/org/apache/nutch/indexer/
src/java/org/apache/nutch/parse/ src/java/org/apache/nutch/protocol/ ...
Author: ab
Date: Wed Mar 7 13:59:07 2007
New Revision: 515791
URL: http://svn.apache.org/viewvc?view=rev&rev=515791
Log:
Upgrade to Hadoop 0.11.2 and Lucene 2.1.0 releases.
Added:
lucene/nutch/trunk/lib/hadoop-0.11.2-core.jar (with props)
lucene/nutch/trunk/lib/lucene-core-2.1.0.jar (with props)
lucene/nutch/trunk/lib/lucene-misc-2.1.0.jar (with props)
lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/lib/lucene-analyzers-2.1.0.jar (with props)
lucene/nutch/trunk/src/plugin/summary-lucene/lib/lucene-highlighter-2.1.0.jar (with props)
Removed:
lucene/nutch/trunk/lib/hadoop-0.10.1-core.jar
lucene/nutch/trunk/lib/lucene-core-2.0.0.jar
lucene/nutch/trunk/lib/lucene-misc-2.0.0.jar
lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/lib/lucene-analyzers-2.0.0.jar
lucene/nutch/trunk/src/plugin/summary-lucene/lib/lucene-highlighter-2.0.0.jar
Modified:
lucene/nutch/trunk/CHANGES.txt
lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.a
lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so
lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so.1
lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so.1.0.0
lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbMerger.java
lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java
lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Inlinks.java
lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbMerger.java
lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher2.java
lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java
lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexMerger.java
lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java
lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java
lucene/nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolStatus.java
lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java
lucene/nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java
lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/plugin.xml
lucene/nutch/trunk/src/plugin/summary-lucene/plugin.xml
lucene/nutch/trunk/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java
lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbMerger.java
lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestGenerator.java
lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java
Modified: lucene/nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/CHANGES.txt?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/CHANGES.txt (original)
+++ lucene/nutch/trunk/CHANGES.txt Wed Mar 7 13:59:07 2007
@@ -151,6 +151,8 @@
50. NUTCH-432 - Fix a bug where platform name with spaces would break the
bin/nutch script. (Brian Whitman via ab)
+51. Upgrade to Hadoop 0.11.2 and Lucene 2.1.0 release.
+
Release 0.8 - 2006-07-25
Added: lucene/nutch/trunk/lib/hadoop-0.11.2-core.jar
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/hadoop-0.11.2-core.jar?view=auto&rev=515791
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/lib/hadoop-0.11.2-core.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/nutch/trunk/lib/lucene-core-2.1.0.jar
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/lucene-core-2.1.0.jar?view=auto&rev=515791
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/lib/lucene-core-2.1.0.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: lucene/nutch/trunk/lib/lucene-misc-2.1.0.jar
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/lucene-misc-2.1.0.jar?view=auto&rev=515791
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/lib/lucene-misc-2.1.0.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.a
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.a?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
Binary files - no diff available.
Modified: lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
Binary files - no diff available.
Modified: lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so.1
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so.1?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
Binary files - no diff available.
Modified: lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so.1.0.0
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/lib/native/Linux-i386-32/libhadoop.so.1.0.0?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
Binary files - no diff available.
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbMerger.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbMerger.java?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbMerger.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbMerger.java Wed Mar 7 13:59:07 2007
@@ -25,7 +25,6 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
@@ -33,7 +32,6 @@
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.ToolBase;
-import org.apache.nutch.net.URLFilters;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java Wed Mar 7 13:59:07 2007
@@ -346,8 +346,6 @@
job.addInputPath(new Path(crawlDb, CrawlDb.CURRENT_NAME));
job.setInputFormat(SequenceFileInputFormat.class);
- job.setInputKeyClass(Text.class);
- job.setInputValueClass(CrawlDatum.class);
job.setOutputPath(outFolder);
job.setOutputFormat(TextOutputFormat.class);
@@ -396,8 +394,6 @@
job.addInputPath(tempDir);
job.setInputFormat(SequenceFileInputFormat.class);
- job.setInputKeyClass(FloatWritable.class);
- job.setInputValueClass(Text.class);
job.setMapperClass(IdentityMapper.class);
job.setReducerClass(CrawlDbTopNReducer.class);
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Inlinks.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Inlinks.java?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Inlinks.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Inlinks.java Wed Mar 7 13:59:07 2007
@@ -25,7 +25,7 @@
/** A list of {@link Inlink}s. */
public class Inlinks implements Writable {
- private HashSet inlinks = new HashSet(1);
+ private HashSet<Inlink> inlinks = new HashSet<Inlink>(1);
public void add(Inlink inlink) { inlinks.add(inlink); }
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbMerger.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbMerger.java?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbMerger.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbMerger.java Wed Mar 7 13:59:07 2007
@@ -21,7 +21,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobClient;
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher2.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher2.java?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher2.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher2.java Wed Mar 7 13:59:07 2007
@@ -343,7 +343,6 @@
}
public synchronized void dump() {
- Iterator it = queues.keySet().iterator();
for (String id : queues.keySet()) {
FetchItemQueue fiq = queues.get(id);
if (fiq.getQueueSize() == 0) continue;
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/FetcherOutputFormat.java Wed Mar 7 13:59:07 2007
@@ -56,7 +56,7 @@
new Path(new Path(job.getOutputPath(), Content.DIR_NAME), name);
final MapFile.Writer fetchOut =
- new MapFile.Writer(fs, fetch.toString(), Text.class, CrawlDatum.class);
+ new MapFile.Writer(job, fs, fetch.toString(), Text.class, CrawlDatum.class);
return new RecordWriter() {
private MapFile.Writer contentOut;
@@ -64,7 +64,7 @@
{
if (Fetcher.isStoringContent(job)) {
- contentOut = new MapFile.Writer(fs, content.toString(),
+ contentOut = new MapFile.Writer(job, fs, content.toString(),
Text.class, Content.class);
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexMerger.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexMerger.java?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexMerger.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexMerger.java Wed Mar 7 13:59:07 2007
@@ -61,7 +61,7 @@
if (LOG.isInfoEnabled()) {
LOG.info("merging indexes to: " + outputIndex);
}
- FileSystem localFs = FileSystem.getNamed("local", getConf());
+ FileSystem localFs = FileSystem.getLocal(getConf());
if (localWorkingDir == null) {
localWorkingDir = new Path("indexmerger-" + System.currentTimeMillis());
}
@@ -102,7 +102,7 @@
// Put target back
//
fs.completeLocalOutput(outputIndex, tmpLocalOutput);
- FileSystem.getNamed("local", conf).delete(localWorkingDir);
+ FileSystem.getLocal(conf).delete(localWorkingDir);
if (LOG.isInfoEnabled()) { LOG.info("done merging"); }
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java Wed Mar 7 13:59:07 2007
@@ -24,7 +24,6 @@
import org.apache.hadoop.io.*;
import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.nutch.crawl.CrawlDatum;
-import org.apache.nutch.fetcher.Fetcher;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.mapred.*;
import org.apache.nutch.scoring.ScoringFilterException;
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java Wed Mar 7 13:59:07 2007
@@ -21,7 +21,6 @@
import org.apache.commons.logging.LogFactory;
import org.apache.nutch.crawl.SignatureFactory;
-import org.apache.nutch.fetcher.Fetcher;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.conf.*;
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolStatus.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolStatus.java?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolStatus.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolStatus.java Wed Mar 7 13:59:07 2007
@@ -46,7 +46,7 @@
/** Resource has moved temporarily. New url should be found in args. */
public static final int TEMP_MOVED = 13;
/** Resource was not found. */
- public static final int NOTFOUND = 14;
+ public static final int NOTFOUND = 14;
/** Temporary failure. Application may retry immediately. */
public static final int RETRY = 15;
/** Unspecified exception occured. Further information may be provided in args. */
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java Wed Mar 7 13:59:07 2007
@@ -232,7 +232,7 @@
} else {
wname = new Path(new Path(new Path(job.getOutputPath(), segmentName + "-" + slice), dirName), name);
}
- res = new MapFile.Writer(fs, wname.toString(), Text.class, clazz);
+ res = new MapFile.Writer(job, fs, wname.toString(), Text.class, clazz);
sliceWriters.put(slice + dirName, res);
return res;
}
Modified: lucene/nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java (original)
+++ lucene/nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java Wed Mar 7 13:59:07 2007
@@ -20,7 +20,6 @@
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.lucene.document.DateField;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
Added: lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/lib/lucene-analyzers-2.1.0.jar
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/lib/lucene-analyzers-2.1.0.jar?view=auto&rev=515791
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/lib/lucene-analyzers-2.1.0.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/plugin.xml
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/plugin.xml?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/plugin.xml (original)
+++ lucene/nutch/trunk/src/plugin/lib-lucene-analyzers/plugin.xml Wed Mar 7 13:59:07 2007
@@ -29,7 +29,7 @@
provider-name="org.apache.lucene">
<runtime>
- <library name="lucene-analyzers-2.0.0.jar">
+ <library name="lucene-analyzers-2.1.0.jar">
<export name="*"/>
</library>
</runtime>
Added: lucene/nutch/trunk/src/plugin/summary-lucene/lib/lucene-highlighter-2.1.0.jar
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/summary-lucene/lib/lucene-highlighter-2.1.0.jar?view=auto&rev=515791
==============================================================================
Binary file - no diff available.
Propchange: lucene/nutch/trunk/src/plugin/summary-lucene/lib/lucene-highlighter-2.1.0.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Modified: lucene/nutch/trunk/src/plugin/summary-lucene/plugin.xml
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/summary-lucene/plugin.xml?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/src/plugin/summary-lucene/plugin.xml (original)
+++ lucene/nutch/trunk/src/plugin/summary-lucene/plugin.xml Wed Mar 7 13:59:07 2007
@@ -25,7 +25,7 @@
<library name="summary-lucene.jar">
<export name="*"/>
</library>
- <library name="lucene-highlighter-2.0.0.jar"/>
+ <library name="lucene-highlighter-2.1.0.jar"/>
</runtime>
<requires>
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java Wed Mar 7 13:59:07 2007
@@ -50,11 +50,11 @@
* urls to be inserted, objects are of type URLCrawlDatum
* @throws Exception
*/
- public static void createCrawlDb(FileSystem fs, Path crawldb, List<URLCrawlDatum> init)
+ public static void createCrawlDb(Configuration conf, FileSystem fs, Path crawldb, List<URLCrawlDatum> init)
throws Exception {
LOG.trace("* creating crawldb: " + crawldb);
Path dir = new Path(crawldb, CrawlDb.CURRENT_NAME);
- MapFile.Writer writer = new MapFile.Writer(fs, new Path(dir, "part-00000")
+ MapFile.Writer writer = new MapFile.Writer(conf, fs, new Path(dir, "part-00000")
.toString(), Text.class, CrawlDatum.class);
Iterator<URLCrawlDatum> it = init.iterator();
while (it.hasNext()) {
@@ -85,7 +85,7 @@
* @return
*/
public static Configuration createConfiguration(){
- Configuration conf=new Configuration();
+ Configuration conf = new Configuration();
conf.addDefaultResource("nutch-default.xml");
conf.addFinalResource("crawl-tests.xml");
return conf;
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbMerger.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbMerger.java?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbMerger.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestCrawlDbMerger.java Wed Mar 7 13:59:07 2007
@@ -97,8 +97,8 @@
Path crawldb1 = new Path(testDir, "crawldb1");
Path crawldb2 = new Path(testDir, "crawldb2");
Path output = new Path(testDir, "output");
- createCrawlDb(fs, crawldb1, init1, cd1);
- createCrawlDb(fs, crawldb2, init2, cd2);
+ createCrawlDb(conf, fs, crawldb1, init1, cd1);
+ createCrawlDb(conf, fs, crawldb2, init2, cd2);
CrawlDbMerger merger = new CrawlDbMerger(conf);
LOG.fine("* merging crawldbs to " + output);
merger.merge(output, new Path[]{crawldb1, crawldb2}, false, false);
@@ -123,10 +123,10 @@
fs.delete(testDir);
}
- private void createCrawlDb(FileSystem fs, Path crawldb, TreeSet init, CrawlDatum cd) throws Exception {
+ private void createCrawlDb(Configuration config, FileSystem fs, Path crawldb, TreeSet init, CrawlDatum cd) throws Exception {
LOG.fine("* creating crawldb: " + crawldb);
Path dir = new Path(crawldb, CrawlDb.CURRENT_NAME);
- MapFile.Writer writer = new MapFile.Writer(fs, new Path(dir, "part-00000").toString(), Text.class, CrawlDatum.class);
+ MapFile.Writer writer = new MapFile.Writer(config, fs, new Path(dir, "part-00000").toString(), Text.class, CrawlDatum.class);
Iterator it = init.iterator();
while (it.hasNext()) {
String key = (String)it.next();
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestGenerator.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestGenerator.java?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestGenerator.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestGenerator.java Wed Mar 7 13:59:07 2007
@@ -330,7 +330,7 @@
fs.mkdirs(segmentsDir);
// create crawldb
- CrawlDBTestUtil.createCrawlDb(fs, dbDir, list);
+ CrawlDBTestUtil.createCrawlDb(conf, fs, dbDir, list);
}
/**
Modified: lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java?view=diff&rev=515791&r1=515790&r2=515791
==============================================================================
--- lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java (original)
+++ lucene/nutch/trunk/src/test/org/apache/nutch/crawl/TestLinkDbMerger.java Wed Mar 7 13:59:07 2007
@@ -109,8 +109,8 @@
Path linkdb1 = new Path(testDir, "linkdb1");
Path linkdb2 = new Path(testDir, "linkdb2");
Path output = new Path(testDir, "output");
- createLinkDb(fs, linkdb1, init1);
- createLinkDb(fs, linkdb2, init2);
+ createLinkDb(conf, fs, linkdb1, init1);
+ createLinkDb(conf, fs, linkdb2, init2);
LinkDbMerger merger = new LinkDbMerger(conf);
LOG.fine("* merging linkdbs to " + output);
merger.merge(output, new Path[]{linkdb1, linkdb2}, false, false);
@@ -139,10 +139,10 @@
fs.delete(testDir);
}
- private void createLinkDb(FileSystem fs, Path linkdb, TreeMap init) throws Exception {
+ private void createLinkDb(Configuration config, FileSystem fs, Path linkdb, TreeMap init) throws Exception {
LOG.fine("* creating linkdb: " + linkdb);
Path dir = new Path(linkdb, LinkDb.CURRENT_NAME);
- MapFile.Writer writer = new MapFile.Writer(fs, new Path(dir, "part-00000").toString(), Text.class, Inlinks.class);
+ MapFile.Writer writer = new MapFile.Writer(config, fs, new Path(dir, "part-00000").toString(), Text.class, Inlinks.class);
Iterator it = init.keySet().iterator();
while (it.hasNext()) {
String key = (String)it.next();