You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by je...@apache.org on 2006/06/12 22:51:45 UTC
svn commit: r413742 [1/3] - in /lucene/nutch/trunk: ./ bin/ conf/
src/java/org/apache/nutch/analysis/ src/java/org/apache/nutch/clustering/
src/java/org/apache/nutch/crawl/ src/java/org/apache/nutch/fetcher/
src/java/org/apache/nutch/indexer/ src/java/...
Author: jerome
Date: Mon Jun 12 13:51:40 2006
New Revision: 413742
URL: http://svn.apache.org/viewvc?rev=413742&view=rev
Log:
NUTCH-303 : Make use of the Commons Logging API and use log4j as the default implementation
Added:
lucene/nutch/trunk/conf/log4j.properties (with props)
lucene/nutch/trunk/src/test/log4j.properties (with props)
lucene/nutch/trunk/src/web/log4j.properties (with props)
Modified:
lucene/nutch/trunk/bin/nutch
lucene/nutch/trunk/build.xml
lucene/nutch/trunk/src/java/org/apache/nutch/analysis/AnalyzerFactory.java
lucene/nutch/trunk/src/java/org/apache/nutch/analysis/CommonGrams.java
lucene/nutch/trunk/src/java/org/apache/nutch/clustering/OnlineClustererFactory.java
lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java
lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDb.java
lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbMerger.java
lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java
lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReducer.java
lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java
lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java
lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDb.java
lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java
lucene/nutch/trunk/src/java/org/apache/nutch/crawl/MapWritable.java
lucene/nutch/trunk/src/java/org/apache/nutch/crawl/SignatureFactory.java
lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java
lucene/nutch/trunk/src/java/org/apache/nutch/indexer/DeleteDuplicates.java
lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexMerger.java
lucene/nutch/trunk/src/java/org/apache/nutch/indexer/Indexer.java
lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexingFilters.java
lucene/nutch/trunk/src/java/org/apache/nutch/net/BasicUrlNormalizer.java
lucene/nutch/trunk/src/java/org/apache/nutch/net/RegexUrlNormalizer.java
lucene/nutch/trunk/src/java/org/apache/nutch/net/URLFilterChecker.java
lucene/nutch/trunk/src/java/org/apache/nutch/net/UrlNormalizerFactory.java
lucene/nutch/trunk/src/java/org/apache/nutch/ontology/OntologyFactory.java
lucene/nutch/trunk/src/java/org/apache/nutch/parse/OutlinkExtractor.java
lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java
lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParsePluginsReader.java
lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java
lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseUtil.java
lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java
lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParserFactory.java
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginDescriptor.java
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginManifestParser.java
lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java
lucene/nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolFactory.java
lucene/nutch/trunk/src/java/org/apache/nutch/searcher/DistributedSearch.java
lucene/nutch/trunk/src/java/org/apache/nutch/searcher/NutchBean.java
lucene/nutch/trunk/src/java/org/apache/nutch/searcher/OpenSearchServlet.java
lucene/nutch/trunk/src/java/org/apache/nutch/searcher/Query.java
lucene/nutch/trunk/src/java/org/apache/nutch/searcher/QueryFilters.java
lucene/nutch/trunk/src/java/org/apache/nutch/searcher/SummarizerFactory.java
lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java
lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentReader.java
lucene/nutch/trunk/src/java/org/apache/nutch/tools/DmozParser.java
lucene/nutch/trunk/src/java/org/apache/nutch/tools/PruneIndexTool.java
lucene/nutch/trunk/src/java/org/apache/nutch/util/DomUtil.java
lucene/nutch/trunk/src/java/org/apache/nutch/util/GZIPUtils.java
lucene/nutch/trunk/src/java/org/apache/nutch/util/ThreadPool.java
lucene/nutch/trunk/src/java/org/apache/nutch/util/mime/MimeTypes.java
lucene/nutch/trunk/src/java/org/apache/nutch/util/mime/MimeTypesReader.java
lucene/nutch/trunk/src/plugin/analysis-fr/plugin.xml
lucene/nutch/trunk/src/plugin/clustering-carrot2/src/java/org/apache/nutch/clustering/carrot2/Clusterer.java
lucene/nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCDeleteUnlicensedTool.java
lucene/nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCIndexingFilter.java
lucene/nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java
lucene/nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java
lucene/nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
lucene/nutch/trunk/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/HTMLLanguageParser.java
lucene/nutch/trunk/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/LanguageIdentifier.java
lucene/nutch/trunk/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/NGramProfile.java
lucene/nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
lucene/nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/RobotRulesParser.java
lucene/nutch/trunk/src/plugin/lib-parsems/src/java/org/apache/nutch/parse/ms/MSBaseParser.java
lucene/nutch/trunk/src/plugin/lib-parsems/src/java/org/apache/nutch/parse/ms/MSExtractor.java
lucene/nutch/trunk/src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexURLFilterBase.java
lucene/nutch/trunk/src/plugin/lib-regex-filter/src/test/org/apache/nutch/urlfilter/api/RegexURLFilterBaseTest.java
lucene/nutch/trunk/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java
lucene/nutch/trunk/src/plugin/ontology/src/java/org/apache/nutch/ontology/jena/OntologyImpl.java
lucene/nutch/trunk/src/plugin/ontology/src/java/org/apache/nutch/ontology/jena/OwlParser.java
lucene/nutch/trunk/src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java
lucene/nutch/trunk/src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java
lucene/nutch/trunk/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java
lucene/nutch/trunk/src/plugin/parse-mspowerpoint/src/java/org/apache/nutch/parse/mspowerpoint/ContentReaderListener.java
lucene/nutch/trunk/src/plugin/parse-mspowerpoint/src/java/org/apache/nutch/parse/mspowerpoint/PPTExtractor.java
lucene/nutch/trunk/src/plugin/parse-mspowerpoint/src/test/org/apache/nutch/parse/mspowerpoint/TestMSPowerPointParser.java
lucene/nutch/trunk/src/plugin/parse-oo/src/java/org/apache/nutch/parse/oo/OOParser.java
lucene/nutch/trunk/src/plugin/parse-pdf/src/java/org/apache/nutch/parse/pdf/PdfParser.java
lucene/nutch/trunk/src/plugin/parse-rss/src/java/org/apache/nutch/parse/rss/RSSParser.java
lucene/nutch/trunk/src/plugin/parse-swf/src/java/org/apache/nutch/parse/swf/SWFParser.java
lucene/nutch/trunk/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java
lucene/nutch/trunk/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipTextExtractor.java
lucene/nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
lucene/nutch/trunk/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java
lucene/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
lucene/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/FtpResponse.java
lucene/nutch/trunk/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/PrintCommandListener.java
lucene/nutch/trunk/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/Http.java
lucene/nutch/trunk/src/plugin/protocol-http/src/java/org/apache/nutch/protocol/http/HttpResponse.java
lucene/nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/Http.java
lucene/nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpAuthenticationFactory.java
lucene/nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpBasicAuthentication.java
lucene/nutch/trunk/src/plugin/protocol-httpclient/src/java/org/apache/nutch/protocol/httpclient/HttpResponse.java
lucene/nutch/trunk/src/plugin/query-more/src/java/org/apache/nutch/searcher/more/DateQueryFilter.java
lucene/nutch/trunk/src/plugin/scoring-opic/src/java/org/apache/nutch/scoring/opic/OPICScoringFilter.java
lucene/nutch/trunk/src/plugin/subcollection/src/java/org/apache/nutch/collection/CollectionManager.java
lucene/nutch/trunk/src/plugin/subcollection/src/java/org/apache/nutch/indexer/subcollection/SubcollectionIndexingFilter.java
lucene/nutch/trunk/src/plugin/urlfilter-prefix/src/java/org/apache/nutch/urlfilter/prefix/PrefixURLFilter.java
lucene/nutch/trunk/src/plugin/urlfilter-suffix/src/java/org/apache/nutch/urlfilter/suffix/SuffixURLFilter.java
Modified: lucene/nutch/trunk/bin/nutch
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/bin/nutch?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/bin/nutch (original)
+++ lucene/nutch/trunk/bin/nutch Mon Jun 12 13:51:40 2006
@@ -121,6 +121,16 @@
# restore ordinary behaviour
unset IFS
+# default log directory & file
+if [ "$NUTCH_LOG_DIR" = "" ]; then
+ NUTCH_LOG_DIR="$NUTCH_HOME/logs"
+fi
+if [ "$NUTCH_LOGFILE" = "" ]; then
+ NUTCH_LOGFILE='nutch.log'
+fi
+NUTCH_OPTS="$NUTCH_OPTS -Dnutch.log.dir=$NUTCH_LOG_DIR"
+NUTCH_OPTS="$NUTCH_OPTS -Dnutch.log.file=$NUTCH_LOGFILE"
+
# figure out which class to run
if [ "$COMMAND" = "crawl" ] ; then
CLASS=org.apache.nutch.crawl.Crawl
Modified: lucene/nutch/trunk/build.xml
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/build.xml?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/build.xml (original)
+++ lucene/nutch/trunk/build.xml Mon Jun 12 13:51:40 2006
@@ -173,6 +173,7 @@
</lib>
<classes dir="${conf.dir}" excludes="**/*.template"/>
<classes dir="${web.src.dir}/locale"/>
+ <classes file="${web.src.dir}/log4j.properties"/>
<zipfileset prefix="WEB-INF/classes/plugins" dir="${build.plugins}"/>
<webinf dir="${lib.dir}">
<include name="taglibs-*.tld"/>
Added: lucene/nutch/trunk/conf/log4j.properties
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/conf/log4j.properties?rev=413742&view=auto
==============================================================================
--- lucene/nutch/trunk/conf/log4j.properties (added)
+++ lucene/nutch/trunk/conf/log4j.properties Mon Jun 12 13:51:40 2006
@@ -0,0 +1,54 @@
+# RootLogger - DailyRollingFileAppender
+log4j.rootLogger=INFO,DRFA
+
+# Logging Threshold
+log4j.threshhold=ALL
+
+log4j.logger.org.apache.nutch=INFO
+log4j.logger.org.apache.hadoop=WARN
+
+#
+# Daily Rolling File Appender
+#
+
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${nutch.log.dir}/${nutch.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n
+# Debugging Pattern format: Date LogLevel LoggerName (FileName:MethodName:LineNo) LogMessage
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+#
+# stdout
+# Add *stdout* to rootlogger above if you want to use this
+#
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+#
+# Rolling File Appender
+#
+
+#log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+#log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+# Logfile size and and 30-day backups
+#log4j.appender.RFA.MaxFileSize=1MB
+#log4j.appender.RFA.MaxBackupIndex=30
+
+#log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n
+#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
Propchange: lucene/nutch/trunk/conf/log4j.properties
------------------------------------------------------------------------------
svn:eol-style = native
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/analysis/AnalyzerFactory.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/analysis/AnalyzerFactory.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/analysis/AnalyzerFactory.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/analysis/AnalyzerFactory.java Mon Jun 12 13:51:40 2006
@@ -15,15 +15,15 @@
*/
package org.apache.nutch.analysis;
-// JDK imports
-import java.util.logging.Logger;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
// Nutch imports
import org.apache.nutch.plugin.Extension;
import org.apache.nutch.plugin.ExtensionPoint;
import org.apache.nutch.plugin.PluginRuntimeException;
import org.apache.nutch.plugin.PluginRepository;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.conf.Configuration;
@@ -36,7 +36,7 @@
private final static String KEY = AnalyzerFactory.class.getName();
- public final static Logger LOG = LogFormatter.getLogger(KEY);
+ public final static Log LOG = LogFactory.getLog(KEY);
private NutchAnalyzer DEFAULT_ANALYZER;
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/analysis/CommonGrams.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/analysis/CommonGrams.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/analysis/CommonGrams.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/analysis/CommonGrams.java Mon Jun 12 13:51:40 2006
@@ -22,10 +22,12 @@
import java.io.*;
import java.util.*;
-import java.util.logging.Logger;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.*;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.searcher.Query.*;
@@ -34,8 +36,7 @@
* too, with n-grams overlaid. This is achieved through the use of {@link
* Token#setPositionIncrement(int)}.*/
public class CommonGrams {
- private static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.analysis.CommonGrams");
+ private static final Log LOG = LogFactory.getLog(CommonGrams.class);
private static final char SEPARATOR = '-';
/** The key used to cache commonTerms in Configuration */
private static final String KEY = CommonGrams.class.getName();
@@ -156,13 +157,13 @@
TokenStream ts = new NutchDocumentTokenizer(new StringReader(line));
Token token = ts.next();
if (token == null) {
- LOG.warning("Line does not contain a field name: " + line);
+ LOG.warn("Line does not contain a field name: " + line);
continue;
}
String field = token.termText();
token = ts.next();
if (token == null) {
- LOG.warning("Line contains only a field name, no word: " + line);
+ LOG.warn("Line contains only a field name, no word: " + line);
continue;
}
String gram = token.termText();
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/clustering/OnlineClustererFactory.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/clustering/OnlineClustererFactory.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/clustering/OnlineClustererFactory.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/clustering/OnlineClustererFactory.java Mon Jun 12 13:51:40 2006
@@ -16,10 +16,12 @@
package org.apache.nutch.clustering;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.nutch.plugin.*;
import org.apache.hadoop.conf.Configuration;
-import java.util.logging.Logger;
-import org.apache.hadoop.util.LogFormatter;
/**
* A factory for retrieving {@link OnlineClusterer} extensions.
@@ -28,8 +30,7 @@
* @version $Id: OnlineClustererFactory.java,v 1.2 2005/02/07 19:09:26 cutting Exp $
*/
public class OnlineClustererFactory {
- public static final Logger LOG = LogFormatter
- .getLogger(OnlineClustererFactory.class.getName());
+ public static final Log LOG = LogFactory.getLog(OnlineClustererFactory.class);
/**
* Nutch configuration key specifying a particular clustering extension
@@ -78,8 +79,8 @@
LOG.info("Using clustering extension: " + extensionName);
return (OnlineClusterer) extension.getExtensionInstance();
}
- LOG.warning("Clustering extension not found: '" + extensionName
- + "', trying the default");
+ LOG.warn("Clustering extension not found: '" + extensionName
+ + "', trying the default");
// not found, fallback to the default, if available.
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Crawl.java Mon Jun 12 13:51:40 2006
@@ -18,12 +18,14 @@
import java.util.*;
import java.text.*;
-import java.util.logging.*;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.nutch.fetcher.Fetcher;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.conf.*;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.mapred.*;
import org.apache.nutch.parse.ParseSegment;
import org.apache.nutch.indexer.DeleteDuplicates;
@@ -33,8 +35,7 @@
import org.apache.nutch.util.NutchJob;
public class Crawl {
- public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.crawl.Crawl");
+ public static final Log LOG = LogFactory.getLog(Crawl.class);
private static String getDate() {
return new SimpleDateFormat("yyyyMMddHHmmss").format
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDb.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDb.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDb.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDb.java Mon Jun 12 13:51:40 2006
@@ -18,12 +18,14 @@
import java.io.*;
import java.util.*;
-import java.util.logging.*;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.conf.*;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.mapred.*;
import org.apache.nutch.util.NutchConfiguration;
@@ -33,8 +35,7 @@
* crawled. Useful for bootstrapping the system. */
public class CrawlDb extends Configured {
- public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.crawl.CrawlDb");
+ public static final Log LOG = LogFactory.getLog(CrawlDb.class);
/** Construct an CrawlDb. */
public CrawlDb(Configuration conf) {
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbMerger.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbMerger.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbMerger.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbMerger.java Mon Jun 12 13:51:40 2006
@@ -18,7 +18,10 @@
import java.io.IOException;
import java.util.*;
-import java.util.logging.Logger;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
@@ -47,7 +50,7 @@
* @author Andrzej Bialecki
*/
public class CrawlDbMerger extends Configured {
- private static final Logger LOG = Logger.getLogger(CrawlDbMerger.class.getName());
+ private static final Log LOG = LogFactory.getLog(CrawlDbMerger.class);
public static class Merger extends MapReduceBase implements Reducer {
private URLFilters filters = null;
@@ -67,7 +70,7 @@
if (filters.filter(((UTF8) key).toString()) == null)
return;
} catch (Exception e) {
- LOG.fine("Can't filter " + key + ": " + e);
+ LOG.debug("Can't filter " + key + ": " + e);
}
}
CrawlDatum res = null;
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReader.java Mon Jun 12 13:51:40 2006
@@ -20,7 +20,10 @@
import java.util.Iterator;
import java.util.Random;
import java.util.TreeMap;
-import java.util.logging.Logger;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -46,7 +49,6 @@
import org.apache.hadoop.mapred.lib.HashPartitioner;
import org.apache.hadoop.mapred.lib.IdentityMapper;
import org.apache.hadoop.mapred.lib.IdentityReducer;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
@@ -58,7 +60,7 @@
*/
public class CrawlDbReader implements Closeable {
- public static final Logger LOG = LogFormatter.getLogger(CrawlDbReader.class.getName());
+ public static final Log LOG = LogFactory.getLog(CrawlDbReader.class);
private MapFile.Reader[] readers = null;
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReducer.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReducer.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReducer.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/CrawlDbReducer.java Mon Jun 12 13:51:40 2006
@@ -20,18 +20,18 @@
import java.util.Iterator;
import java.io.IOException;
-import java.util.logging.*;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.*;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.nutch.scoring.ScoringFilterException;
import org.apache.nutch.scoring.ScoringFilters;
/** Merge new page entries with existing entries. */
public class CrawlDbReducer implements Reducer {
- public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.crawl.CrawlDbReducer");
+ public static final Log LOG = LogFactory.getLog(CrawlDbReducer.class);
private int retryMax;
private CrawlDatum result = new CrawlDatum();
private ArrayList linked = new ArrayList();
@@ -108,8 +108,8 @@
try {
scfilters.initialScore((UTF8)key, result);
} catch (ScoringFilterException e) {
- LOG.warning("Cannot filter init score for url " + key +
- ", using default: " + e.getMessage());
+ LOG.warn("Cannot filter init score for url " + key +
+ ", using default: " + e.getMessage());
result.setScore(scoreInjected);
}
}
@@ -122,7 +122,7 @@
break;
case CrawlDatum.STATUS_SIGNATURE:
- LOG.warning("Lone CrawlDatum.STATUS_SIGNATURE: " + key);
+ LOG.warn("Lone CrawlDatum.STATUS_SIGNATURE: " + key);
return;
case CrawlDatum.STATUS_FETCH_RETRY: // temporary failure
if (old != null)
@@ -147,7 +147,7 @@
try {
scfilters.updateDbScore((UTF8)key, result, linked);
} catch (Exception e) {
- LOG.warning("Couldn't update score, key=" + key + ": " + e);
+ LOG.warn("Couldn't update score, key=" + key + ": " + e);
}
output.collect(key, result);
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Generator.java Mon Jun 12 13:51:40 2006
@@ -20,11 +20,13 @@
import java.net.*;
import java.util.*;
import java.text.*;
-import java.util.logging.*;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.*;
import org.apache.hadoop.conf.*;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.fs.Path;
@@ -38,8 +40,7 @@
/** Generates a subset of a crawl db to fetch. */
public class Generator extends Configured {
- public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.crawl.Generator");
+ public static final Log LOG = LogFactory.getLog(Generator.class);
public static class SelectorEntry implements Writable {
public UTF8 url;
@@ -97,7 +98,7 @@
if (filters.filter(url.toString()) == null)
return;
} catch (URLFilterException e) {
- LOG.warning("Couldn't filter url: " + url + " (" + e.getMessage() + ")");
+ LOG.warn("Couldn't filter url: " + url + " (" + e.getMessage() + ")");
}
CrawlDatum crawlDatum = (CrawlDatum)value;
@@ -111,7 +112,7 @@
try {
sort = scfilters.generatorSortValue((UTF8)key, crawlDatum, sort);
} catch (ScoringFilterException sfe) {
- LOG.warning("Couldn't filter generatorSortValue for " + key + ": " + sfe);
+ LOG.warn("Couldn't filter generatorSortValue for " + key + ": " + sfe);
}
// sort by decreasing score
sortValue.set(sort);
@@ -149,9 +150,9 @@
InetAddress ia = InetAddress.getByName(host);
host = ia.getHostAddress();
} catch (UnknownHostException uhe) {
- LOG.fine("DNS lookup failed: " + host + ", skipping.");
+ LOG.debug("DNS lookup failed: " + host + ", skipping.");
dnsFailure++;
- if (dnsFailure % 1000 == 0) LOG.warning("DNS failures: " + dnsFailure);
+ if (dnsFailure % 1000 == 0) LOG.warn("DNS failures: " + dnsFailure);
continue;
}
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/Injector.java Mon Jun 12 13:51:40 2006
@@ -18,12 +18,14 @@
import java.io.*;
import java.util.*;
-import java.util.logging.*;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.conf.*;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.mapred.*;
import org.apache.nutch.net.*;
@@ -35,8 +37,7 @@
/** This class takes a flat file of URLs and adds them to the of pages to be
* crawled. Useful for bootstrapping the system. */
public class Injector extends Configured {
- public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.crawl.Injector");
+ public static final Log LOG = LogFactory.getLog(Injector.class);
/** Normalize and filter injected urls. */
@@ -69,7 +70,7 @@
url = urlNormalizer.normalize(url); // normalize the url
url = filters.filter(url); // filter the url
} catch (Exception e) {
- LOG.warning("Skipping " +url+":"+e);
+ LOG.warn("Skipping " +url+":"+e);
url = null;
}
if (url != null) { // if it passes
@@ -79,8 +80,8 @@
try {
scfilters.initialScore(value, datum);
} catch (ScoringFilterException e) {
- LOG.warning("Cannot filter init score for url " + url +
- ", using default (" + e.getMessage() + ")");
+ LOG.warn("Cannot filter init score for url " + url +
+ ", using default (" + e.getMessage() + ")");
datum.setScore(scoreInjected);
}
output.collect(value, datum);
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDb.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDb.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDb.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDb.java Mon Jun 12 13:51:40 2006
@@ -18,14 +18,16 @@
import java.io.*;
import java.util.*;
-import java.util.logging.*;
import java.net.*;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.hadoop.io.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.conf.*;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.mapred.*;
import org.apache.nutch.net.URLFilters;
@@ -36,8 +38,7 @@
/** Maintains an inverted link map, listing incoming links for each url. */
public class LinkDb extends Configured implements Mapper, Reducer {
- public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.crawl.LinkDb");
+ public static final Log LOG = LogFactory.getLog(LinkDb.class);
public static String CURRENT_NAME = "current";
@@ -63,7 +64,7 @@
if (filters.filter(((UTF8)key).toString()) == null)
return;
} catch (Exception e) {
- LOG.fine("Can't filter " + key + ": " + e);
+ LOG.debug("Can't filter " + key + ": " + e);
}
}
Inlinks inlinks = null;
@@ -84,7 +85,7 @@
if (filters.filter(in.getFromUrl()) == null)
continue;
} catch (Exception e) {
- LOG.fine("Can't filter " + key + ": " + e);
+ LOG.debug("Can't filter " + key + ": " + e);
}
}
inlinks.add(in);
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/LinkDbReader.java Mon Jun 12 13:51:40 2006
@@ -18,22 +18,24 @@
import java.io.IOException;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.hadoop.io.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.mapred.lib.HashPartitioner;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
import java.util.Iterator;
-import java.util.logging.Logger;
/** . */
public class LinkDbReader implements Closeable {
- public static final Logger LOG = LogFormatter.getLogger(LinkDbReader.class.getName());
+ public static final Log LOG = LogFactory.getLog(LinkDbReader.class);
private static final Partitioner PARTITIONER = new HashPartitioner();
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/MapWritable.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/MapWritable.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/MapWritable.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/MapWritable.java Mon Jun 12 13:51:40 2006
@@ -25,7 +25,10 @@
import java.util.Iterator;
import java.util.LinkedList;
import java.util.Set;
-import java.util.logging.Logger;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.FloatWritable;
@@ -36,7 +39,6 @@
import org.apache.hadoop.io.ObjectWritable;
import org.apache.hadoop.io.UTF8;
import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.LogFormatter;
/**
* A writable map, with a similar behavior as <code>java.util.HashMap</code>.
@@ -52,8 +54,7 @@
*/
public class MapWritable implements Writable {
- public static final Logger LOG = LogFormatter.getLogger(MapWritable.class
- .getName());
+ public static final Log LOG = LogFactory.getLog(MapWritable.class);
private KeyValueEntry fFirst;
@@ -312,7 +313,7 @@
clazz = Class.forName(UTF8.readString(in));
addIdEntry(id, clazz);
} catch (Exception e) {
- LOG.warning("MapWritable: unable to load internal map entry" + e.toString());
+ LOG.warn("MapWritable: unable to load internal map entry" + e.toString());
fIdCount--;
}
}
@@ -328,8 +329,8 @@
fLast = fLast.fNextEntry = entry;
}
} catch (IOException e) {
- LOG.warning("MapWritable: unable to load meta data entry, ignoring.. : "
- + e.toString());
+ LOG.warn("MapWritable: unable to load meta data entry, ignoring.. : "
+ + e.toString());
fSize--;
}
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/crawl/SignatureFactory.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/crawl/SignatureFactory.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/crawl/SignatureFactory.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/crawl/SignatureFactory.java Mon Jun 12 13:51:40 2006
@@ -16,9 +16,11 @@
package org.apache.nutch.crawl;
-import java.util.logging.Logger;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.util.LogFormatter;
+// Hadoop imports
import org.apache.hadoop.conf.Configuration;
/**
@@ -29,8 +31,7 @@
* @author Andrzej Bialecki <ab@getopt.org>
*/
public class SignatureFactory {
- private static final Logger LOG =
- LogFormatter.getLogger(SignatureFactory.class.getName());
+ private static final Log LOG = LogFactory.getLog(SignatureFactory.class);
private SignatureFactory() {} // no public ctor
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java Mon Jun 12 13:51:40 2006
@@ -18,10 +18,13 @@
import java.io.IOException;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.hadoop.io.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.conf.*;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.mapred.*;
import org.apache.nutch.crawl.CrawlDatum;
@@ -33,13 +36,11 @@
import org.apache.nutch.scoring.ScoringFilters;
import org.apache.nutch.util.*;
-import java.util.logging.*;
/** The fetcher. Most of the work is done by plugins. */
public class Fetcher extends Configured implements MapRunnable {
- public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.fetcher.Fetcher");
+ public static final Log LOG = LogFactory.getLog(Fetcher.class);
public static final String SIGNATURE_KEY = "nutch.content.digest";
public static final String SEGMENT_NAME_KEY = "nutch.segment.name";
@@ -103,16 +104,19 @@
CrawlDatum datum = new CrawlDatum();
while (true) {
- if (LogFormatter.hasLoggedSevere()) // something bad happened
- break; // exit
+ // TODO : NUTCH-258 ...
+ // If something bad happened, then exit
+ // if (conf.getBoolean("fetcher.exit", false)) {
+ // break;
+ // ]
try { // get next entry from input
if (!input.next(key, datum)) {
break; // at eof, exit
}
} catch (IOException e) {
- e.printStackTrace();
- LOG.severe("fetcher caught:"+e.toString());
+ e.printStackTrace(LogUtil.getFatalStream(LOG));
+ LOG.fatal("fetcher caught:"+e.toString());
break;
}
@@ -130,7 +134,7 @@
int redirectCount = 0;
do {
redirecting = false;
- LOG.fine("redirectCount=" + redirectCount);
+ LOG.debug("redirectCount=" + redirectCount);
Protocol protocol = this.protocolFactory.getProtocol(url.toString());
ProtocolOutput output = protocol.getProtocolOutput(url, datum);
ProtocolStatus status = output.getStatus();
@@ -151,9 +155,9 @@
url = new UTF8(newUrl);
redirecting = true;
redirectCount++;
- LOG.fine(" - content redirect to " + url);
+ LOG.debug(" - content redirect to " + url);
} else {
- LOG.fine(" - content redirect skipped: " +
+ LOG.debug(" - content redirect skipped: " +
(newUrl != null ? "to same url" : "filtered"));
}
}
@@ -168,9 +172,9 @@
url = new UTF8(newUrl);
redirecting = true;
redirectCount++;
- LOG.fine(" - protocol redirect to " + url);
+ LOG.debug(" - protocol redirect to " + url);
} else {
- LOG.fine(" - protocol redirect skipped: " +
+ LOG.debug(" - protocol redirect skipped: " +
(newUrl != null ? "to same url" : "filtered"));
}
break;
@@ -191,7 +195,7 @@
break;
default:
- LOG.warning("Unknown ProtocolStatus: " + status.getCode());
+ LOG.warn("Unknown ProtocolStatus: " + status.getCode());
output(url, datum, null, CrawlDatum.STATUS_FETCH_GONE);
}
@@ -211,8 +215,8 @@
}
} catch (Throwable e) {
- e.printStackTrace();
- LOG.severe("fetcher caught:"+e.toString());
+ e.printStackTrace(LogUtil.getFatalStream(LOG));
+ LOG.fatal("fetcher caught:"+e.toString());
} finally {
synchronized (Fetcher.this) {activeThreads--;} // count threads
}
@@ -242,8 +246,8 @@
try {
scfilters.passScoreBeforeParsing(key, datum, content);
} catch (Exception e) {
- e.printStackTrace();
- LOG.warning("Couldn't pass score, url " + key + " (" + e + ")");
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
+ LOG.warn("Couldn't pass score, url " + key + " (" + e + ")");
}
Parse parse = null;
@@ -256,7 +260,7 @@
parseStatus = new ParseStatus(e);
}
if (!parseStatus.isSuccess()) {
- LOG.warning("Error parsing: " + key + ": " + parseStatus);
+ LOG.warn("Error parsing: " + key + ": " + parseStatus);
parse = parseStatus.getEmptyParse(getConf());
}
// Calculate page signature. For non-parsing fetchers this will
@@ -270,8 +274,8 @@
try {
scfilters.passScoreAfterParsing(key, content, parse);
} catch (Exception e) {
- e.printStackTrace();
- LOG.warning("Couldn't pass score, url " + key + " (" + e + ")");
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
+ LOG.warn("Couldn't pass score, url " + key + " (" + e + ")");
}
}
@@ -283,8 +287,8 @@
storingContent ? content : null,
parse != null ? new ParseImpl(parse) : null));
} catch (IOException e) {
- e.printStackTrace();
- LOG.severe("fetcher caught:"+e.toString());
+ e.printStackTrace(LogUtil.getFatalStream(LOG));
+ LOG.fatal("fetcher caught:"+e.toString());
}
if (parse != null) return parse.getData().getStatus();
else return null;
@@ -320,9 +324,9 @@
this.storingContent = isStoringContent(job);
this.parsing = isParsing(job);
- if (job.getBoolean("fetcher.verbose", false)) {
- LOG.setLevel(Level.FINE);
- }
+// if (job.getBoolean("fetcher.verbose", false)) {
+// LOG.setLevel(Level.FINE);
+// }
}
public void close() {}
@@ -364,7 +368,7 @@
// some requests seem to hang, despite all intentions
synchronized (this) {
if ((System.currentTimeMillis() - lastRequestStart) > timeout) {
- LOG.warning("Aborting with "+activeThreads+" hung threads.");
+ LOG.warn("Aborting with "+activeThreads+" hung threads.");
return;
}
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/indexer/DeleteDuplicates.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/indexer/DeleteDuplicates.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/indexer/DeleteDuplicates.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/indexer/DeleteDuplicates.java Mon Jun 12 13:51:40 2006
@@ -18,12 +18,13 @@
import java.io.*;
import java.util.*;
-import java.util.logging.*;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.*;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.conf.*;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.mapred.*;
import org.apache.nutch.util.NutchConfiguration;
@@ -38,8 +39,7 @@
******************************************************************/
public class DeleteDuplicates extends Configured
implements Mapper, Reducer, OutputFormat {
- private static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.indexer.DeleteDuplicates");
+ private static final Log LOG = LogFactory.getLog(DeleteDuplicates.class);
// Algorithm:
//
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexMerger.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexMerger.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexMerger.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexMerger.java Mon Jun 12 13:51:40 2006
@@ -18,12 +18,14 @@
import java.io.*;
import java.util.*;
-import java.util.logging.*;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.conf.*;
-import org.apache.hadoop.util.LogFormatter;
+import org.apache.nutch.util.LogUtil;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.lucene.store.Directory;
@@ -37,8 +39,7 @@
* @author Mike Cafarella
*************************************************************************/
public class IndexMerger {
- public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.indexer.IndexMerger");
+ public static final Log LOG = LogFactory.getLog(IndexMerger.class);
public static final String DONE_NAME = "merge.done";
@@ -88,7 +89,7 @@
writer.setMaxBufferedDocs(conf.getInt("indexer.minMergeDocs", IndexWriter.DEFAULT_MAX_BUFFERED_DOCS));
writer.setMaxMergeDocs(conf.getInt("indexer.maxMergeDocs", IndexWriter.DEFAULT_MAX_MERGE_DOCS));
writer.setTermIndexInterval(conf.getInt("indexer.termIndexInterval", IndexWriter.DEFAULT_TERM_INDEX_INTERVAL));
- writer.setInfoStream(LogFormatter.getLogStream(LOG, Level.FINE));
+ writer.setInfoStream(LogUtil.getDebugStream(LOG));
writer.setUseCompoundFile(false);
writer.setSimilarity(new NutchSimilarity());
writer.addIndexes(dirs);
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/indexer/Indexer.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/indexer/Indexer.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/indexer/Indexer.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/indexer/Indexer.java Mon Jun 12 13:51:40 2006
@@ -18,19 +18,21 @@
import java.io.*;
import java.util.*;
-import java.util.logging.*;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.*;
import org.apache.nutch.fetcher.Fetcher;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.conf.*;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.mapred.*;
import org.apache.nutch.parse.*;
import org.apache.nutch.analysis.*;
import org.apache.nutch.scoring.ScoringFilterException;
import org.apache.nutch.scoring.ScoringFilters;
+import org.apache.nutch.util.LogUtil;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
@@ -47,8 +49,7 @@
public static final String DONE_NAME = "index.done";
- public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.crawl.Indexer");
+ public static final Log LOG = LogFactory.getLog(Indexer.class);
/** Wraps inputs in an {@link ObjectWritable}, to permit merging different
* types in reduce. */
@@ -96,7 +97,7 @@
writer.setTermIndexInterval
(job.getInt("indexer.termIndexInterval", 128));
writer.setMaxFieldLength(job.getInt("indexer.max.tokens", 10000));
- writer.setInfoStream(LogFormatter.getLogStream(LOG, Level.INFO));
+ writer.setInfoStream(LogUtil.getInfoStream(LOG));
writer.setUseCompoundFile(false);
writer.setSimilarity(new NutchSimilarity());
@@ -195,7 +196,7 @@
} else if (value instanceof ParseText) {
parseText = (ParseText)value;
} else {
- LOG.warning("Unrecognized type: "+value.getClass());
+ LOG.warn("Unrecognized type: "+value.getClass());
}
}
@@ -227,7 +228,7 @@
// run indexing filters
doc = this.filters.filter(doc, parse, (UTF8)key, fetchDatum, inlinks);
} catch (IndexingException e) {
- LOG.warning("Error indexing "+key+": "+e);
+ LOG.warn("Error indexing "+key+": "+e);
return;
}
@@ -237,7 +238,7 @@
boost = this.scfilters.indexerScore((UTF8)key, doc, dbDatum,
fetchDatum, parse, inlinks, boost);
} catch (ScoringFilterException e) {
- LOG.warning("Error calculating score " + key + ": " + e);
+ LOG.warn("Error calculating score " + key + ": " + e);
return;
}
// apply boost to all indexed fields.
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexingFilters.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexingFilters.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexingFilters.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/indexer/IndexingFilters.java Mon Jun 12 13:51:40 2006
@@ -18,6 +18,10 @@
import java.util.HashMap;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.lucene.document.Document;
import org.apache.nutch.plugin.*;
@@ -30,6 +34,8 @@
/** Creates and caches {@link IndexingFilter} implementing plugins.*/
public class IndexingFilters {
+ public final static Log LOG = LogFactory.getLog(IndexingFilters.class);
+
private IndexingFilter[] indexingFilters;
public IndexingFilters(Configuration conf) {
@@ -44,7 +50,7 @@
for (int i = 0; i < extensions.length; i++) {
Extension extension = extensions[i];
IndexingFilter filter = (IndexingFilter) extension.getExtensionInstance();
- System.out.println("-adding " + filter.getClass().getName());
+ LOG.info("Adding " + filter.getClass().getName());
if (!filterMap.containsKey(filter.getClass().getName())) {
filterMap.put(filter.getClass().getName(), filter);
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/net/BasicUrlNormalizer.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/net/BasicUrlNormalizer.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/net/BasicUrlNormalizer.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/net/BasicUrlNormalizer.java Mon Jun 12 13:51:40 2006
@@ -18,18 +18,20 @@
import java.net.URL;
import java.net.MalformedURLException;
-// import java.net.URI;
-// import java.net.URISyntaxException;
-import java.util.logging.Logger;
-import org.apache.hadoop.util.LogFormatter;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+// Nutch imports
+import org.apache.nutch.util.LogUtil;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.oro.text.regex.*;
/** Converts URLs to a normal form . */
public class BasicUrlNormalizer implements UrlNormalizer {
- public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.net.BasicUrlNormalizer");
+ public static final Log LOG = LogFactory.getLog(BasicUrlNormalizer.class);
private Perl5Compiler compiler = new Perl5Compiler();
private ThreadLocal matchers = new ThreadLocal() {
@@ -61,7 +63,7 @@
leadingRelativePathRule.substitution = new Perl5Substitution("/");
} catch (MalformedPatternException e) {
- e.printStackTrace();
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
throw new RuntimeException(e);
}
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/net/RegexUrlNormalizer.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/net/RegexUrlNormalizer.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/net/RegexUrlNormalizer.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/net/RegexUrlNormalizer.java Mon Jun 12 13:51:40 2006
@@ -106,7 +106,7 @@
.parse(filename);
Element root = doc.getDocumentElement();
if (!"regex-normalize".equals(root.getTagName()))
- LOG.severe("bad conf file: top-level element not <regex-normalize>");
+ LOG.fatal("bad conf file: top-level element not <regex-normalize>");
NodeList regexes = root.getChildNodes();
for (int i = 0; i < regexes.getLength(); i++) {
Node regexNode = regexes.item(i);
@@ -114,7 +114,7 @@
continue;
Element regex = (Element)regexNode;
if (!"regex".equals(regex.getTagName()))
- LOG.warning("bad conf file: element not <regex>");
+ LOG.warn("bad conf file: element not <regex>");
NodeList fields = regex.getChildNodes();
String patternValue = null;
String subValue = null;
@@ -139,7 +139,7 @@
}
} catch (Exception e) {
- LOG.severe("error parsing " + filename +" conf file: " + e);
+ LOG.fatal("error parsing " + filename +" conf file: " + e);
}
return rules;
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/net/URLFilterChecker.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/net/URLFilterChecker.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/net/URLFilterChecker.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/net/URLFilterChecker.java Mon Jun 12 13:51:40 2006
@@ -20,13 +20,10 @@
import org.apache.nutch.plugin.ExtensionPoint;
import org.apache.nutch.plugin.PluginRepository;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.util.NutchConfiguration;
-import java.util.logging.Logger;
-
import java.io.BufferedReader;
import java.io.InputStreamReader;
@@ -38,8 +35,6 @@
public class URLFilterChecker {
- public static final Logger LOG =
- LogFormatter.getLogger(URLFilterChecker.class.getName());
private Configuration conf;
public URLFilterChecker(Configuration conf) {
@@ -60,7 +55,6 @@
for (int i = 0; i < extensions.length; i++) {
Extension extension = extensions[i];
filter = (URLFilter)extension.getExtensionInstance();
- //LOG.info(filter.getClass().getName());
if (filter.getClass().getName().equals(filterName)) {
break;
} else {
@@ -71,10 +65,11 @@
if (filter == null)
throw new RuntimeException("Filter "+filterName+" not found.");
- if (LogFormatter.hasLoggedSevere())
- throw new RuntimeException("Severe error encountered.");
+ // jerome : should we keep this behavior?
+ //if (LogFormatter.hasLoggedSevere())
+ // throw new RuntimeException("Severe error encountered.");
- LOG.info("Checking URLFilter "+filterName);
+ System.out.println("Checking URLFilter "+filterName);
BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
String line;
@@ -91,7 +86,7 @@
}
private void checkAll() throws Exception {
- LOG.info("Checking combination of all URLFilters available");
+ System.out.println("Checking combination of all URLFilters available");
BufferedReader in = new BufferedReader(new InputStreamReader(System.in));
String line;
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/net/UrlNormalizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/net/UrlNormalizerFactory.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/net/UrlNormalizerFactory.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/net/UrlNormalizerFactory.java Mon Jun 12 13:51:40 2006
@@ -16,15 +16,16 @@
package org.apache.nutch.net;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.hadoop.conf.*;
-import org.apache.hadoop.util.LogFormatter;
-import java.util.logging.*;
/** Factory to create a UrlNormalizer from "urlnormalizer.class" config property. */
public class UrlNormalizerFactory {
- private static final Logger LOG = LogFormatter
- .getLogger("org.apache.nutch.net.UrlNormalizerFactory");
+ private static final Log LOG = LogFactory.getLog(UrlNormalizerFactory.class);
private Configuration conf;
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/ontology/OntologyFactory.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/ontology/OntologyFactory.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/ontology/OntologyFactory.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/ontology/OntologyFactory.java Mon Jun 12 13:51:40 2006
@@ -16,10 +16,12 @@
package org.apache.nutch.ontology;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.nutch.plugin.*;
import org.apache.hadoop.conf.Configuration;
-import java.util.logging.Logger;
-import org.apache.hadoop.util.LogFormatter;
/**
* A factory for retrieving {@link Ontology} extensions.
@@ -28,8 +30,7 @@
* @version $Id: OntologyFactory.java,v 1.2 2005/02/07 19:09:58 cutting Exp $
*/
public class OntologyFactory {
- public final static Logger LOG =
- LogFormatter.getLogger(OntologyFactory.class.getName());
+ public final static Log LOG = LogFactory.getLog(OntologyFactory.class);
private ExtensionPoint extensionPoint;
private Configuration conf;
@@ -60,8 +61,8 @@
LOG.info("Using ontology extension: " + extensionName);
return (Ontology) extension.getExtensionInstance();
}
- LOG.warning("Ontology extension not found: '" + extensionName
- + "', trying the default");
+ LOG.warn("Ontology extension not found: '" + extensionName
+ + "', trying the default");
// not found, fallback to the default, if available.
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/parse/OutlinkExtractor.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/parse/OutlinkExtractor.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/parse/OutlinkExtractor.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/parse/OutlinkExtractor.java Mon Jun 12 13:51:40 2006
@@ -18,9 +18,10 @@
import java.util.ArrayList;
import java.util.List;
-import java.util.logging.Logger;
-import org.apache.hadoop.util.LogFormatter;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.oro.text.regex.MatchResult;
import org.apache.oro.text.regex.Pattern;
@@ -45,8 +46,7 @@
* @since 0.7
*/
public class OutlinkExtractor {
- private static final Logger LOG = LogFormatter
- .getLogger(OutlinkExtractor.class.getName());
+ private static final Log LOG = LogFactory.getLog(OutlinkExtractor.class);
/**
* Regex pattern to get URLs within a plain text.
@@ -101,7 +101,7 @@
// (SHOULD really check cpu time used so that heavily loaded systems
// do not unnecessarily hit this limit.)
if (System.currentTimeMillis() - start >= 60000L) {
- LOG.warning("Time limit exceeded for getOutLinks");
+ LOG.warn("Time limit exceeded for getOutLinks");
break;
}
result = matcher.getMatch();
@@ -110,7 +110,7 @@
}
} catch (Exception ex) {
// if the matcher fails (perhaps a malformed URL) we just log it and move on
- LOG.throwing(OutlinkExtractor.class.getName(), "getOutlinks", ex);
+ LOG.error("getOutlinks", ex);
}
final Outlink[] retval;
@@ -152,7 +152,7 @@
//
// url = re.getParen(0);
//
- // LOG.finest("Extracted url: " + url);
+ // LOG.trace("Extracted url: " + url);
//
// try {
//
@@ -162,7 +162,7 @@
// } catch (MalformedURLException ex) {
// // if it is a malformed URL we just throw it away and continue with
// // extraction.
- // LOG.throwing(this.getClass().getName(), "getOutlinks", ex);
+ // LOG.error("getOutlinks", ex);
// }
//
// pos = re.getParenEnd(0);
@@ -215,7 +215,7 @@
// } catch (MalformedURLException ex) {
// // if it is a malformed URL we just throw it away and continue with
// // extraction.
- // LOG.throwing(this.getClass().getName(), "getOutlinks", ex);
+ // LOG.error("getOutlinks", ex);
// }
//
// pos = re.getParenEnd(0);
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseOutputFormat.java Mon Jun 12 13:51:40 2006
@@ -16,6 +16,10 @@
package org.apache.nutch.parse;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.hadoop.io.*;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.fetcher.Fetcher;
@@ -27,11 +31,10 @@
import org.apache.nutch.net.*;
import java.io.*;
-import java.util.logging.Logger;
/* Parse content in a segment. */
public class ParseOutputFormat implements OutputFormat {
- private static final Logger LOG = Logger.getLogger(ParseOutputFormat.class.getName());
+ private static final Log LOG = LogFactory.getLog(ParseOutputFormat.class);
private UrlNormalizer urlNormalizer;
private URLFilters filters;
@@ -116,8 +119,8 @@
adjust = scfilters.distributeScoreToOutlink((UTF8)key, targetUrl,
parseData, target, null, links.length, validCount);
} catch (ScoringFilterException e) {
- LOG.warning("Cannot distribute score from " + key + " to " + targetUrl +
- " - skipped (" + e.getMessage());
+ LOG.warn("Cannot distribute score from " + key + " to " + targetUrl +
+ " - skipped (" + e.getMessage());
continue;
}
crawlOut.append(targetUrl, target);
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParsePluginsReader.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParsePluginsReader.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParsePluginsReader.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParsePluginsReader.java Mon Jun 12 13:51:40 2006
@@ -23,7 +23,6 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.logging.Logger;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -33,9 +32,14 @@
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
-// Nutch imports
-import org.apache.hadoop.util.LogFormatter;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+// Hadoop imports
import org.apache.hadoop.conf.Configuration;
+
+// Nutch imports
import org.apache.nutch.util.NutchConfiguration;
@@ -49,8 +53,7 @@
class ParsePluginsReader {
/* our log stream */
- public static final Logger LOG =
- LogFormatter.getLogger(ParsePluginsReader.class.getName());
+ public static final Log LOG = LogFactory.getLog(ParsePluginsReader.class);
/** The property name of the parse-plugins location */
private static final String PP_FILE_PROP = "parse.plugin.file";
@@ -90,8 +93,8 @@
parsePluginUrl = new URL(fParsePluginsFile);
ppInputStream = parsePluginUrl.openStream();
} catch (Exception e) {
- LOG.warning("Unable to load parse plugins file from URL " +
- "[" + fParsePluginsFile + "]. Reason is [" + e + "]");
+ LOG.warn("Unable to load parse plugins file from URL " +
+ "[" + fParsePluginsFile + "]. Reason is [" + e + "]");
return pList;
}
} else {
@@ -106,8 +109,8 @@
parser = factory.newDocumentBuilder();
document = parser.parse(inputSource);
} catch (Exception e) {
- LOG.warning("Unable to parse [" + fParsePluginsFile + "]." +
- "Reason is [" + e + "]");
+ LOG.warn("Unable to parse [" + fParsePluginsFile + "]." +
+ "Reason is [" + e + "]");
return null;
}
@@ -161,8 +164,8 @@
pList.setPluginList(mimeTypeStr, plugList);
} else {
- LOG.warning("ParsePluginsReader:ERROR:no plugins defined for mime type: "
- + mimeTypeStr + ", continuing parse");
+ LOG.warn("ParsePluginsReader:ERROR:no plugins defined for mime type: "
+ + mimeTypeStr + ", continuing parse");
}
}
return pList;
@@ -237,13 +240,13 @@
NodeList aliasRoot = parsePluginsRoot.getElementsByTagName("aliases");
if (aliasRoot == null || (aliasRoot != null && aliasRoot.getLength() == 0)) {
- LOG.warning("No aliases defined in parse-plugins.xml!");
+ LOG.warn("No aliases defined in parse-plugins.xml!");
return aliases;
}
if (aliasRoot.getLength() > 1) {
// log a warning, but try and continue processing
- LOG.warning("There should only be one \"aliases\" tag in parse-plugins.xml");
+ LOG.warn("There should only be one \"aliases\" tag in parse-plugins.xml");
}
Element aliasRootElem = (Element)aliasRoot.item(0);
@@ -254,8 +257,8 @@
Element aliasElem = (Element)aliasElements.item(i);
String parsePluginId = aliasElem.getAttribute("name");
String extensionId = aliasElem.getAttribute("extension-id");
- LOG.finest("Found alias: plugin-id: " + parsePluginId +
- ", extension-id: " + extensionId);
+ LOG.trace("Found alias: plugin-id: " + parsePluginId +
+ ", extension-id: " + extensionId);
if (parsePluginId != null && extensionId != null) {
aliases.put(parsePluginId, extensionId);
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java Mon Jun 12 13:51:40 2006
@@ -16,12 +16,14 @@
package org.apache.nutch.parse;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.nutch.crawl.SignatureFactory;
import org.apache.nutch.fetcher.Fetcher;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.conf.*;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.nutch.protocol.*;
import org.apache.nutch.scoring.ScoringFilterException;
import org.apache.nutch.scoring.ScoringFilters;
@@ -30,13 +32,11 @@
import java.io.*;
import java.util.*;
-import java.util.logging.*;
/* Parse content in a segment. */
public class ParseSegment extends Configured implements Mapper, Reducer {
- public static final Logger LOG =
- LogFormatter.getLogger(Parser.class.getName());
+ public static final Log LOG = LogFactory.getLog(Parser.class);
private ScoringFilters scfilters;
@@ -77,13 +77,13 @@
try {
scfilters.passScoreAfterParsing((UTF8)key, content, parse);
} catch (ScoringFilterException e) {
- e.printStackTrace();
- LOG.warning("Error passing score: "+key+": "+e.getMessage());
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
+ LOG.warn("Error passing score: "+key+": "+e.getMessage());
return;
}
output.collect(key, new ParseImpl(parse.getText(), parse.getData()));
} else {
- LOG.warning("Error parsing: "+key+": "+status.toString());
+ LOG.warn("Error parsing: "+key+": "+status.toString());
}
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseUtil.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseUtil.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseUtil.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParseUtil.java Mon Jun 12 13:51:40 2006
@@ -15,12 +15,14 @@
*/
package org.apache.nutch.parse;
-// JDK imports
-import java.util.logging.Logger;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
// Nutch Imports
import org.apache.nutch.protocol.Content;
-import org.apache.hadoop.util.LogFormatter;
+
+// Hadoop imports
import org.apache.hadoop.conf.Configuration;
@@ -36,8 +38,7 @@
public class ParseUtil {
/* our log stream */
- public static final Logger LOG =
- LogFormatter.getLogger(ParseUtil.class.getName());
+ public static final Log LOG = LogFactory.getLog(ParseUtil.class);
private Configuration conf;
private ParserFactory parserFactory;
@@ -66,22 +67,22 @@
try {
parsers = this.parserFactory.getParsers(content.getContentType(), "");
} catch (ParserNotFound e) {
- LOG.warning("No suitable parser found when trying to parse content " +
- content);
+ LOG.warn("No suitable parser found when trying to parse content " +
+ content);
throw new ParseException(e.getMessage());
}
Parse parse = null;
for (int i=0; i<parsers.length; i++) {
- LOG.fine("Parsing [" + content.getUrl() + "] with [" + parsers[i] + "]");
+ LOG.debug("Parsing [" + content.getUrl() + "] with [" + parsers[i] + "]");
parse = parsers[i].getParse(content);
if ((parse != null) && (parse.getData().getStatus().isSuccess())) {
return parse;
}
}
- LOG.warning("Unable to successfully parse content " + content.getUrl() +
- " of type " + content.getContentType());
+ LOG.warn("Unable to successfully parse content " + content.getUrl() +
+ " of type " + content.getContentType());
ParseStatus ps = (parse.getData() != null) ? parse.getData().getStatus() : null;
return (ps == null) ? new ParseStatus().getEmptyParse(this.conf)
@@ -115,8 +116,8 @@
try {
p = this.parserFactory.getParserById(extId);
} catch (ParserNotFound e) {
- LOG.warning("No suitable parser found when trying to parse content " +
- content);
+ LOG.warn("No suitable parser found when trying to parse content " +
+ content);
throw new ParseException(e.getMessage());
}
@@ -125,8 +126,8 @@
if (parse != null && parse.getData().getStatus().isSuccess()) {
return parse;
} else {
- LOG.warning("Unable to successfully parse content " + content.getUrl() +
- " of type " + content.getContentType());
+ LOG.warn("Unable to successfully parse content " + content.getUrl() +
+ " of type " + content.getContentType());
return new ParseStatus().getEmptyParse(this.conf);
}
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParserChecker.java Mon Jun 12 13:51:40 2006
@@ -16,7 +16,9 @@
package org.apache.nutch.parse;
-import org.apache.hadoop.util.LogFormatter;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.util.NutchConfiguration;
@@ -29,8 +31,6 @@
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.Content;
-import java.util.logging.Logger;
-
/**
* Parser checker, useful for testing parser.
*
@@ -39,8 +39,7 @@
public class ParserChecker {
- public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.parse.ParserChecker");
+ public static final Log LOG = LogFactory.getLog(ParserChecker.class);
public ParserChecker() {}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParserFactory.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParserFactory.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParserFactory.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/parse/ParserFactory.java Mon Jun 12 13:51:40 2006
@@ -21,15 +21,20 @@
import java.util.Iterator;
import java.util.List;
import java.util.Vector;
-import java.util.logging.Logger;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+// Hadoop imports
+import org.apache.hadoop.conf.Configuration;
// Nutch imports
import org.apache.nutch.plugin.Extension;
import org.apache.nutch.plugin.ExtensionPoint;
import org.apache.nutch.plugin.PluginRuntimeException;
import org.apache.nutch.plugin.PluginRepository;
-import org.apache.hadoop.util.LogFormatter;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.util.LogUtil;
import org.apache.nutch.util.mime.MimeType;
import org.apache.nutch.util.mime.MimeTypeException;
@@ -37,8 +42,7 @@
/** Creates and caches {@link Parser} plugins.*/
public final class ParserFactory {
- public static final Logger LOG =
- LogFormatter.getLogger(ParserFactory.class.getName());
+ public static final Log LOG = LogFactory.getLog(ParserFactory.class);
/** Wildcard for default plugins. */
public static final String DEFAULT_PLUGIN = "*";
@@ -127,12 +131,12 @@
}
parsers.add(p);
} catch (PluginRuntimeException e) {
- e.printStackTrace();
- LOG.warning("ParserFactory:PluginRuntimeException when "
- + "initializing parser plugin "
- + ext.getDescriptor().getPluginId()
- + " instance in getParsers "
- + "function: attempting to continue instantiating parsers");
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
+ LOG.warn("ParserFactory:PluginRuntimeException when "
+ + "initializing parser plugin "
+ + ext.getDescriptor().getPluginId()
+ + " instance in getParsers "
+ + "function: attempting to continue instantiating parsers");
}
}
return (Parser[]) parsers.toArray(new Parser[]{});
@@ -185,9 +189,9 @@
this.conf.setObject(parserExt.getId(), p);
return p;
} catch (PluginRuntimeException e) {
- LOG.warning("Canno initialize parser " +
- parserExt.getDescriptor().getPluginId() +
- " (cause: " + e.toString());
+ LOG.warn("Canno initialize parser " +
+ parserExt.getDescriptor().getPluginId() +
+ " (cause: " + e.toString());
throw new ParserNotFound("Cannot init parser for id [" + id + "]");
}
}
@@ -207,8 +211,8 @@
try {
type = MimeType.clean(contentType);
} catch (MimeTypeException mte) {
- LOG.fine("Could not clean the content-type [" + contentType +
- "], Reason is [" + mte + "]. Using its raw version...");
+ LOG.debug("Could not clean the content-type [" + contentType +
+ "], Reason is [" + mte + "]. Using its raw version...");
type = contentType;
}
@@ -300,17 +304,17 @@
// plugin was enabled via plugin.includes
// its plugin.xml just doesn't claim to support that
// particular mimeType
- LOG.warning("ParserFactory:Plugin: " + parsePluginId +
- " mapped to contentType " + contentType +
- " via parse-plugins.xml, but " + "its plugin.xml " +
- "file does not claim to support contentType: " +
- contentType);
+ LOG.warn("ParserFactory:Plugin: " + parsePluginId +
+ " mapped to contentType " + contentType +
+ " via parse-plugins.xml, but " + "its plugin.xml " +
+ "file does not claim to support contentType: " +
+ contentType);
} else {
// plugin wasn't enabled via plugin.includes
- LOG.warning("ParserFactory: Plugin: " + parsePluginId +
- " mapped to contentType " + contentType +
- " via parse-plugins.xml, but not enabled via " +
- "plugin.includes in nutch-default.xml");
+ LOG.warn("ParserFactory: Plugin: " + parsePluginId +
+ " mapped to contentType " + contentType +
+ " via parse-plugins.xml, but not enabled via " +
+ "plugin.includes in nutch-default.xml");
}
}
@@ -344,8 +348,8 @@
contentType + ", but they are not mapped to it in the " +
"parse-plugins.xml file");
} else {
- LOG.fine("ParserFactory:No parse plugins mapped or enabled for " +
- "contentType " + contentType);
+ LOG.debug("ParserFactory:No parse plugins mapped or enabled for " +
+ "contentType " + contentType);
}
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginDescriptor.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginDescriptor.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginDescriptor.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginDescriptor.java Mon Jun 12 13:51:40 2006
@@ -15,6 +15,8 @@
* limitations under the License.
*/
package org.apache.nutch.plugin;
+
+// JDK imports
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
@@ -25,8 +27,15 @@
import java.util.MissingResourceException;
import java.util.ResourceBundle;
import java.util.logging.Logger;
-import org.apache.hadoop.util.LogFormatter;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+// Hadoop imports
import org.apache.hadoop.conf.Configuration;
+
+
/**
* The <code>PluginDescriptor</code> provide access to all meta information of
* a nutch-plugin, as well to the internationalizable resources and the plugin
@@ -51,8 +60,7 @@
private ArrayList fNotExportedLibs = new ArrayList();
private ArrayList fExtensions = new ArrayList();
private PluginClassLoader fClassLoader;
- public static final Logger LOG = LogFormatter
- .getLogger(PluginDescriptor.class.getName());
+ public static final Log LOG = LogFactory.getLog(PluginDescriptor.class);
private Configuration fConf;
/**
@@ -265,7 +273,7 @@
arrayList.add(file2.getParentFile().toURL());
}
} catch (MalformedURLException e) {
- LOG.fine(getPluginId() + " " + e.toString());
+ LOG.debug(getPluginId() + " " + e.toString());
}
URL[] urls = (URL[]) arrayList.toArray(new URL[arrayList.size()]);
fClassLoader = new PluginClassLoader(urls, PluginDescriptor.class
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginManifestParser.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginManifestParser.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginManifestParser.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginManifestParser.java Mon Jun 12 13:51:40 2006
@@ -24,12 +24,13 @@
import java.net.URLDecoder;
import java.util.HashMap;
import java.util.Map;
-import java.util.logging.Logger;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
+import org.apache.commons.logging.Log;
+
import org.apache.hadoop.conf.Configuration;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
@@ -44,7 +45,7 @@
* @author joa23
*/
public class PluginManifestParser {
- public static final Logger LOG = PluginRepository.LOG;
+ public static final Log LOG = PluginRepository.LOG;
private static final boolean WINDOWS = System.getProperty("os.name")
.startsWith("Windows");
@@ -87,17 +88,17 @@
String manifestPath = oneSubFolder.getAbsolutePath()
+ File.separator + "plugin.xml";
try {
- LOG.fine("parsing: " + manifestPath);
+ LOG.debug("parsing: " + manifestPath);
PluginDescriptor p = parseManifestFile(manifestPath);
map.put(p.getPluginId(), p);
} catch (MalformedURLException e) {
- LOG.warning(e.toString());
+ LOG.warn(e.toString());
} catch (SAXException e) {
- LOG.warning(e.toString());
+ LOG.warn(e.toString());
} catch (IOException e) {
- LOG.warning(e.toString());
+ LOG.warn(e.toString());
} catch (ParserConfigurationException e) {
- LOG.warning(e.toString());
+ LOG.warn(e.toString());
}
}
}
@@ -118,10 +119,10 @@
&& directory.isDirectory() && directory.listFiles().length > 0) {
return directory; // relative path that is not in the classpath
} else if (url == null) {
- LOG.warning("Plugins: directory not found: " + name);
+ LOG.warn("Plugins: directory not found: " + name);
return null;
} else if (!"file".equals(url.getProtocol())) {
- LOG.warning("Plugins: not a file: url. Can't load plugins from: "
+ LOG.warn("Plugins: not a file: url. Can't load plugins from: "
+ url);
return null;
}
@@ -184,7 +185,7 @@
}
PluginDescriptor pluginDescriptor = new PluginDescriptor(id, version,
name, providerName, pluginClazz, pPath, this.conf);
- LOG.fine("plugin: id="+id+" name="+name+" version="+version
+ LOG.debug("plugin: id="+id+" name="+name+" version="+version
+" provider="+providerName+"class="+pluginClazz);
parseExtension(rootElement, pluginDescriptor);
parseExtensionPoints(rootElement, pluginDescriptor);
@@ -259,7 +260,7 @@
String schema = oneExtensionPoint.getAttribute("schema");
ExtensionPoint extensionPoint = new ExtensionPoint(id, name,
schema);
- //LOG.fine("plugin: point="+id);
+ //LOG.debug("plugin: point="+id);
pPluginDescriptor.addExtensionPoint(extensionPoint);
}
}
@@ -289,7 +290,7 @@
String id = oneImplementation.getAttribute("id");
String extensionClass = oneImplementation
.getAttribute("class");
- LOG.fine("impl: point=" + pointId + " class="
+ LOG.debug("impl: point=" + pointId + " class="
+ extensionClass);
Extension extension = new Extension(pPluginDescriptor,
pointId, id, extensionClass, this.conf, this.pluginRepository);