You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by je...@apache.org on 2006/06/12 22:51:45 UTC
svn commit: r413742 [2/3] - in /lucene/nutch/trunk: ./ bin/ conf/
src/java/org/apache/nutch/analysis/ src/java/org/apache/nutch/clustering/
src/java/org/apache/nutch/crawl/ src/java/org/apache/nutch/fetcher/
src/java/org/apache/nutch/indexer/ src/java/...
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/plugin/PluginRepository.java Mon Jun 12 13:51:40 2006
@@ -26,14 +26,19 @@
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.logging.Logger;
import java.util.regex.Pattern;
-// Nutch imports
-import org.apache.hadoop.util.LogFormatter;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+// Hadoop imports
import org.apache.hadoop.conf.Configuration;
+
+// Nutch imports
import org.apache.nutch.util.NutchConfiguration;
+
/**
* The plugin repositority is a registry of all plugins.
*
@@ -60,8 +65,7 @@
private Configuration conf;
- public static final Logger LOG = LogFormatter
- .getLogger("org.apache.nutch.plugin.PluginRepository");
+ public static final Log LOG = LogFactory.getLog(PluginRepository.class);
/**
* @throws PluginRuntimeException
@@ -87,7 +91,7 @@
try {
installExtensions(fRegisteredPlugins);
} catch (PluginRuntimeException e) {
- LOG.severe(e.toString());
+ LOG.fatal(e.toString());
throw new RuntimeException(e.getMessage());
}
displayStatus();
@@ -115,7 +119,7 @@
for (int j=0; j<points.length; j++) {
ExtensionPoint point = points[j];
String xpId = point.getId();
- LOG.fine("Adding extension point " + xpId);
+ LOG.debug("Adding extension point " + xpId);
fExtensionPoints.put(xpId, point);
}
}
@@ -206,10 +210,10 @@
checked.put(plugin.getPluginId(), plugin);
} catch (MissingDependencyException mde) {
// Simply ignore this plugin
- LOG.warning(mde.getMessage());
+ LOG.warn(mde.getMessage());
} catch (CircularDependencyException cde) {
// Simply ignore this plugin
- LOG.warning(cde.getMessage());
+ LOG.warn(cde.getMessage());
}
}
return new ArrayList(checked.values());
@@ -362,11 +366,11 @@
if (id == null) { continue; }
if (!includes.matcher(id).matches()) {
- LOG.fine("not including: " + id);
+ LOG.debug("not including: " + id);
continue;
}
if (excludes.matcher(id).matches()) {
- LOG.fine("excluding: " + id);
+ LOG.debug("excluding: " + id);
continue;
}
map.put(plugin.getPluginId(), plugin);
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolFactory.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolFactory.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolFactory.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/protocol/ProtocolFactory.java Mon Jun 12 13:51:40 2006
@@ -19,10 +19,12 @@
import java.net.URL;
import java.net.MalformedURLException;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.nutch.plugin.*;
-import java.util.logging.Logger;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.conf.Configuration;
/** Creates and caches {@link Protocol} plugins. Protocol plugins should
@@ -30,8 +32,7 @@
* implement. */
public class ProtocolFactory {
- public static final Logger LOG = LogFormatter
- .getLogger(ProtocolFactory.class.getName());
+ public static final Log LOG = LogFactory.getLog(ProtocolFactory.class);
private ExtensionPoint extensionPoint;
private Configuration conf;
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/searcher/DistributedSearch.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/searcher/DistributedSearch.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/searcher/DistributedSearch.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/searcher/DistributedSearch.java Mon Jun 12 13:51:40 2006
@@ -19,14 +19,15 @@
import java.net.InetSocketAddress;
import java.io.*;
import java.util.*;
-import java.util.logging.Logger;
import java.lang.reflect.Method;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.nutch.parse.ParseData;
import org.apache.nutch.parse.ParseText;
import org.apache.nutch.crawl.Inlinks;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.fs.Path;
@@ -36,8 +37,7 @@
/** Implements the search API over IPC connnections. */
public class DistributedSearch {
- public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.searcher.DistributedSearch");
+ public static final Log LOG = LogFactory.getLog(DistributedSearch.class);
private DistributedSearch() {} // no public ctor
@@ -171,11 +171,11 @@
InetSocketAddress addr = defaultAddresses[i];
String[] segments = results[i];
if (segments == null) {
- LOG.warning("Client: no segments from: " + addr);
+ LOG.warn("Client: no segments from: " + addr);
continue;
}
for (int j = 0; j < segments.length; j++) {
- LOG.finest("Client: segment "+segments[j]+" at "+addr);
+ LOG.trace("Client: segment "+segments[j]+" at "+addr);
segmentToAddress.put(segments[j], addr);
}
liveAddresses.add(addr);
@@ -355,7 +355,7 @@
LOG.info("Querying segments from search servers...");
updateSegments();
} catch (IOException ioe) {
- LOG.warning("No search servers available!");
+ LOG.warn("No search servers available!");
liveAddresses=new InetSocketAddress[0];
}
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/searcher/NutchBean.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/searcher/NutchBean.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/searcher/NutchBean.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/searcher/NutchBean.java Mon Jun 12 13:51:40 2006
@@ -18,13 +18,14 @@
import java.io.*;
import java.util.*;
-import java.util.logging.Logger;
import javax.servlet.ServletContext;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.Closeable;
import org.apache.hadoop.conf.*;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.nutch.parse.*;
import org.apache.nutch.indexer.*;
import org.apache.nutch.crawl.Inlinks;
@@ -38,12 +39,11 @@
implements Searcher, HitDetailer, HitSummarizer, HitContent, HitInlinks,
DistributedSearch.Protocol, Closeable {
- public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.searcher.NutchBean");
+ public static final Log LOG = LogFactory.getLog(NutchBean.class);
- static {
- LogFormatter.setShowThreadIDs(true);
- }
+// static {
+// LogFormatter.setShowThreadIDs(true);
+// }
private String[] segmentNames;
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/searcher/OpenSearchServlet.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/searcher/OpenSearchServlet.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/searcher/OpenSearchServlet.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/searcher/OpenSearchServlet.java Mon Jun 12 13:51:40 2006
@@ -18,7 +18,6 @@
import java.io.IOException;
import java.net.URLEncoder;
-import java.util.logging.Level;
import java.util.Map;
import java.util.HashMap;
import java.util.Set;
@@ -133,7 +132,7 @@
hits = bean.search(query, start + hitsPerPage, hitsPerDup, dedupField,
sort, reverse);
} catch (IOException e) {
- NutchBean.LOG.log(Level.WARNING, "Search Error", e);
+ NutchBean.LOG.warn("Search Error", e);
hits = new Hits(0,new Hit[0]);
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/searcher/Query.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/searcher/Query.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/searcher/Query.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/searcher/Query.java Mon Jun 12 13:51:40 2006
@@ -23,12 +23,14 @@
import java.io.InputStreamReader;
import java.util.Arrays;
import java.util.ArrayList;
-import java.util.logging.Logger;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.nutch.analysis.AnalyzerFactory;
import org.apache.nutch.analysis.NutchAnalysis;
@@ -36,8 +38,7 @@
/** A Nutch query. */
public final class Query implements Writable, Cloneable, Configurable {
- public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.searcher.Query");
+ public static final Log LOG = LogFactory.getLog(Query.class);
/** A query clause. */
public static class Clause implements Cloneable {
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/searcher/QueryFilters.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/searcher/QueryFilters.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/searcher/QueryFilters.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/searcher/QueryFilters.java Mon Jun 12 13:51:40 2006
@@ -16,12 +16,13 @@
package org.apache.nutch.searcher;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.nutch.plugin.*;
import org.apache.nutch.searcher.Query.Clause;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.conf.Configuration;
-import java.util.logging.Logger;
import java.util.*;
import org.apache.lucene.search.BooleanQuery;
@@ -34,8 +35,7 @@
* are parsed as multi-token Query.Phrase's.
*/
public class QueryFilters {
- private static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.searcher.QueryFilters");
+ private static final Log LOG = LogFactory.getLog(QueryFilters.class);
private QueryFilter[] queryFilters;
private HashSet FIELD_NAMES ;
@@ -66,7 +66,7 @@
ArrayList fieldNames = parseFieldNames(extension, "fields");
ArrayList rawFieldNames = parseFieldNames(extension, "raw-fields");
if (fieldNames.size() == 0 && rawFieldNames.size() == 0) {
- LOG.warning("QueryFilter: " + extension.getId()
+ LOG.warn("QueryFilter: " + extension.getId()
+ " names no fields.");
continue;
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/searcher/SummarizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/searcher/SummarizerFactory.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/searcher/SummarizerFactory.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/searcher/SummarizerFactory.java Mon Jun 12 13:51:40 2006
@@ -15,12 +15,12 @@
*/
package org.apache.nutch.searcher;
-// JDK imports
-import java.util.logging.Logger;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
// Hadoop imports
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.LogFormatter;
// Nutch imports
import org.apache.nutch.plugin.Extension;
@@ -35,8 +35,7 @@
public class SummarizerFactory {
/** My logger */
- public final static Logger LOG =
- LogFormatter.getLogger(SummarizerFactory.class.getName());
+ public final static Log LOG = LogFactory.getLog(SummarizerFactory.class);
/** The first available {@link Summarizer} */
private Summarizer summarizer = null;
@@ -52,7 +51,7 @@
LOG.info("Using the first summarizer extension found: " +
extensions[0].getId());
} catch (Exception e) {
- LOG.warning(e.toString());
+ LOG.warn(e.toString());
}
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentMerger.java Mon Jun 12 13:51:40 2006
@@ -18,7 +18,9 @@
import java.io.IOException;
import java.util.*;
-import java.util.logging.Logger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.*;
import org.apache.hadoop.fs.FileSystem;
@@ -84,7 +86,7 @@
* @author Andrzej Bialecki
*/
public class SegmentMerger extends Configured implements Mapper, Reducer {
- private static final Logger LOG = Logger.getLogger(SegmentMerger.class.getName());
+ private static final Log LOG = LogFactory.getLog(SegmentMerger.class);
private static final UTF8 SEGMENT_PART_KEY = new UTF8("_PaRt_");
private static final UTF8 SEGMENT_NAME_KEY = new UTF8("_NaMe_");
@@ -335,7 +337,7 @@
return;
}
} catch (Exception e) {
- LOG.warning("Cannot filter key " + key + ": " + e.getMessage());
+ LOG.warn("Cannot filter key " + key + ": " + e.getMessage());
}
}
output.collect(key, value);
@@ -545,7 +547,7 @@
boolean pt = true;
for (int i = 0; i < segs.length; i++) {
if (!fs.exists(segs[i])) {
- LOG.warning("SegmentMerger: input dir " + segs[i] + " doesn't exist, skipping.");
+ LOG.warn("SegmentMerger: input dir " + segs[i] + " doesn't exist, skipping.");
segs[i] = null;
continue;
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentReader.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentReader.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/segment/SegmentReader.java Mon Jun 12 13:51:40 2006
@@ -19,24 +19,26 @@
import java.io.*;
import java.text.SimpleDateFormat;
import java.util.*;
-import java.util.logging.Logger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.*;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.parse.ParseData;
import org.apache.nutch.parse.ParseText;
import org.apache.nutch.protocol.Content;
+import org.apache.nutch.util.LogUtil;
import org.apache.nutch.util.NutchConfiguration;
/** Dump the content of a segment. */
public class SegmentReader extends Configured implements Reducer {
- public static final Logger LOG = LogFormatter.getLogger(SegmentReader.class.getName());
+ public static final Log LOG = LogFactory.getLog(SegmentReader.class);
long recNo = 0L;
@@ -105,7 +107,7 @@
try {
this.fs = FileSystem.get(getConf());
} catch (IOException e) {
- e.printStackTrace();
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
}
}
@@ -120,7 +122,7 @@
try {
this.fs = FileSystem.get(getConf());
} catch (IOException e) {
- e.printStackTrace();
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
}
}
@@ -154,7 +156,7 @@
} else if (value instanceof ParseText) {
dump.append("\nParseText::\n").append(((ParseText) value).toString());
} else {
- LOG.warning("Unrecognized type: " + value.getClass());
+ LOG.warn("Unrecognized type: " + value.getClass());
}
}
output.collect(key, new ObjectWritable(dump.toString()));
@@ -206,8 +208,8 @@
try {
currentRecordNumber = append(fs, job, partFile, writer, currentRecordNumber);
} catch (IOException exception) {
- LOG.warning("Couldn't copy the content of " + partFile.toString() + " into " + dumpFile.toString());
- LOG.warning(exception.getMessage());
+ LOG.warn("Couldn't copy the content of " + partFile.toString() + " into " + dumpFile.toString());
+ LOG.warn(exception.getMessage());
}
}
} finally {
@@ -256,7 +258,7 @@
List res = getMapRecords(new Path(segment, Content.DIR_NAME), key);
results.put("co", res);
} catch (Exception e) {
- e.printStackTrace();
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
}
}
});
@@ -266,7 +268,7 @@
List res = getMapRecords(new Path(segment, CrawlDatum.FETCH_DIR_NAME), key);
results.put("fe", res);
} catch (Exception e) {
- e.printStackTrace();
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
}
}
});
@@ -276,7 +278,7 @@
List res = getSeqRecords(new Path(segment, CrawlDatum.GENERATE_DIR_NAME), key);
results.put("ge", res);
} catch (Exception e) {
- e.printStackTrace();
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
}
}
});
@@ -286,7 +288,7 @@
List res = getSeqRecords(new Path(segment, CrawlDatum.PARSE_DIR_NAME), key);
results.put("pa", res);
} catch (Exception e) {
- e.printStackTrace();
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
}
}
});
@@ -296,7 +298,7 @@
List res = getMapRecords(new Path(segment, ParseData.DIR_NAME), key);
results.put("pd", res);
} catch (Exception e) {
- e.printStackTrace();
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
}
}
});
@@ -306,7 +308,7 @@
List res = getMapRecords(new Path(segment, ParseText.DIR_NAME), key);
results.put("pt", res);
} catch (Exception e) {
- e.printStackTrace();
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
}
}
});
@@ -321,7 +323,7 @@
while (it.hasNext()) {
if (((Thread)it.next()).isAlive()) cnt++;
}
- if (cnt > 0) System.err.println("(" + cnt + " to retrieve)");
+ if (cnt > 0) LOG.debug("(" + cnt + " to retrieve)");
} while (cnt > 0);
for (int i = 0; i < keys.length; i++) {
List res = (List)results.get(keys[i][0]);
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/tools/DmozParser.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/tools/DmozParser.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/tools/DmozParser.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/tools/DmozParser.java Mon Jun 12 13:51:40 2006
@@ -18,7 +18,6 @@
import java.io.*;
import java.util.*;
-import java.util.logging.*;
import java.util.regex.*;
import javax.xml.parsers.*;
@@ -26,15 +25,20 @@
import org.xml.sax.helpers.*;
import org.apache.xerces.util.XMLChar;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.hadoop.io.*;
import org.apache.hadoop.fs.*;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.conf.Configuration;
+import org.apache.nutch.util.LogUtil;
import org.apache.nutch.util.NutchConfiguration;
+
/** Utility that converts DMOZ RDF into a flat file of URLs to be injected. */
public class DmozParser {
- public static final Logger LOG = LogFormatter.getLogger("org.apache.nutch.tools.DmozParser");
+ public static final Log LOG = LogFactory.getLog(DmozParser.class);
long pages = 0;
@@ -176,7 +180,7 @@
// Inc the number of pages, insert the page, and
// possibly print status.
//
- System.out.println(curURL);
+ LOG.info(curURL);
pages++;
//
@@ -232,25 +236,25 @@
* Emit the exception message
*/
public void error(SAXParseException spe) {
- LOG.severe("Error: " + spe.toString() + ": " + spe.getMessage());
- spe.printStackTrace(System.err);
+ LOG.fatal("Error: " + spe.toString() + ": " + spe.getMessage());
+ spe.printStackTrace(LogUtil.getFatalStream(LOG));
}
/**
* Emit the exception message, with line numbers
*/
public void fatalError(SAXParseException spe) {
- LOG.severe("Fatal err: " + spe.toString() + ": " + spe.getMessage());
- LOG.severe("Last known line is " + location.getLineNumber() + ", column " + location.getColumnNumber());
- spe.printStackTrace(System.err);
+ LOG.fatal("Fatal err: " + spe.toString() + ": " + spe.getMessage());
+ LOG.fatal("Last known line is " + location.getLineNumber() + ", column " + location.getColumnNumber());
+ spe.printStackTrace(LogUtil.getFatalStream(LOG));
}
/**
* Emit exception warning message
*/
public void warning(SAXParseException spe) {
- LOG.warning("Warning: " + spe.toString() + ": " + spe.getMessage());
- spe.printStackTrace(System.err);
+ LOG.warn("Warning: " + spe.toString() + ": " + spe.getMessage());
+ spe.printStackTrace(LogUtil.getFatalStream(LOG));
}
}
@@ -287,8 +291,8 @@
InputSource is = new InputSource(in);
reader.parse(is);
} catch (Exception e) {
- LOG.severe(e.toString());
- e.printStackTrace(System.err);
+ LOG.fatal(e.toString());
+ e.printStackTrace(LogUtil.getFatalStream(LOG));
System.exit(0);
} finally {
in.close();
@@ -306,8 +310,8 @@
}
}
catch (Exception e) {
- LOG.severe(e.toString());
- e.printStackTrace(System.out);
+ LOG.fatal(e.toString());
+ e.printStackTrace(LogUtil.getFatalStream(LOG));
System.exit(0);
} finally {
in.close();
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/tools/PruneIndexTool.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/tools/PruneIndexTool.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/tools/PruneIndexTool.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/tools/PruneIndexTool.java Mon Jun 12 13:51:40 2006
@@ -35,9 +35,11 @@
import java.util.BitSet;
import java.util.StringTokenizer;
import java.util.Vector;
-import java.util.logging.Logger;
-import org.apache.hadoop.util.LogFormatter;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.nutch.util.NutchConfiguration;
@@ -96,7 +98,7 @@
* @author Andrzej Bialecki <ab@getopt.org>
*/
public class PruneIndexTool implements Runnable {
- public static final Logger LOG = LogFormatter.getLogger("org.apache.nutch.tools.PruneIndexTool");
+ public static final Log LOG = LogFactory.getLog(PruneIndexTool.class);
/** Log the progress every LOG_STEP number of processed documents. */
public static int LOG_STEP = 50000;
@@ -187,7 +189,7 @@
output.flush();
output.close();
} catch (Exception e) {
- LOG.warning("Error closing: " + e.getMessage());
+ LOG.warn("Error closing: " + e.getMessage());
}
}
@@ -255,7 +257,7 @@
}
if (!dryrun) {
IndexReader.unlock(dir);
- LOG.fine(" - had to unlock index in " + dir);
+ LOG.debug(" - had to unlock index in " + dir);
}
}
reader = IndexReader.open(dir);
@@ -268,19 +270,19 @@
dir = FSDirectory.getDirectory(indexDirs[i], false);
if (IndexReader.isLocked(dir)) {
if (!unlock) {
- LOG.warning(dr + "Index " + indexDirs[i] + " is locked. Skipping...");
+ LOG.warn(dr + "Index " + indexDirs[i] + " is locked. Skipping...");
continue;
}
if (!dryrun) {
IndexReader.unlock(dir);
- LOG.fine(" - had to unlock index in " + dir);
+ LOG.debug(" - had to unlock index in " + dir);
}
}
IndexReader r = IndexReader.open(dir);
indexes.add(r);
numIdx++;
} catch (Exception e) {
- LOG.warning(dr + "Invalid index in " + indexDirs[i] + " - skipping...");
+ LOG.warn(dr + "Invalid index in " + indexDirs[i] + " - skipping...");
}
}
if (indexes.size() == 0) throw new Exception("No input indexes.");
@@ -325,7 +327,7 @@
try {
searcher.search(queries[i], ahc);
} catch (IOException e) {
- LOG.warning(dr + " - failed: " + e.getMessage());
+ LOG.warn(dr + " - failed: " + e.getMessage());
continue;
}
if (bits.cardinality() == 0) {
@@ -353,7 +355,7 @@
cnt++;
}
} catch (Exception e) {
- LOG.warning(dr + " - failed to delete doc #" + docNum);
+ LOG.warn(dr + " - failed to delete doc #" + docNum);
}
start = docNum + 1;
}
@@ -368,20 +370,20 @@
try {
reader.close();
} catch (IOException e) {
- LOG.warning(dr + "Exception when closing reader(s): " + e.getMessage());
+ LOG.warn(dr + "Exception when closing reader(s): " + e.getMessage());
}
}
public static void main(String[] args) throws Exception {
if (args.length == 0) {
usage();
- LOG.severe("Missing arguments");
+ LOG.fatal("Missing arguments");
return;
}
File idx = new File(args[0]);
if (!idx.isDirectory()) {
usage();
- LOG.severe("Not a directory: " + idx);
+ LOG.fatal("Not a directory: " + idx);
return;
}
Vector paths = new Vector();
@@ -396,7 +398,7 @@
});
if (dirs == null || dirs.length == 0) {
usage();
- LOG.severe("No indexes in " + idx);
+ LOG.fatal("No indexes in " + idx);
return;
}
for (int i = 0; i < dirs.length; i++) {
@@ -407,7 +409,7 @@
}
if (paths.size() == 0) {
usage();
- LOG.severe("No indexes in " + idx + " or its subdirs.");
+ LOG.fatal("No indexes in " + idx + " or its subdirs.");
return;
}
}
@@ -430,7 +432,7 @@
dryrun = true;
} else {
usage();
- LOG.severe("Unrecognized option: " + args[i]);
+ LOG.fatal("Unrecognized option: " + args[i]);
return;
}
}
@@ -463,20 +465,20 @@
is = conf.getConfResourceAsInputStream(qPath);
}
if (is == null) {
- LOG.severe("Can't load queries from " + qPath);
+ LOG.fatal("Can't load queries from " + qPath);
return;
}
try {
queries = parseQueries(is);
} catch (Exception e) {
- LOG.severe("Error parsing queries: " + e.getMessage());
+ LOG.fatal("Error parsing queries: " + e.getMessage());
return;
}
try {
PruneIndexTool pit = new PruneIndexTool(indexes, queries, checkers, force, dryrun);
pit.run();
} catch (Exception e) {
- LOG.severe("Error running PruneIndexTool: " + e.getMessage());
+ LOG.fatal("Error running PruneIndexTool: " + e.getMessage());
return;
}
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/util/DomUtil.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/util/DomUtil.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/util/DomUtil.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/util/DomUtil.java Mon Jun 12 13:51:40 2006
@@ -33,8 +33,15 @@
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+
public class DomUtil {
+ private final static Log LOG = LogFactory.getLog(DomUtil.class);
+
/**
* Returns parsed dom tree or null if any error
*
@@ -54,11 +61,11 @@
parser.parse(input);
element = (Element) parser.getDocument().getChildNodes().item(0);
} catch (FileNotFoundException e) {
- e.printStackTrace();
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
} catch (SAXException e) {
- e.printStackTrace();
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
} catch (IOException e) {
- e.printStackTrace();
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
}
return element;
}
@@ -81,13 +88,13 @@
transformer.transform(source, result);
os.flush();
} catch (UnsupportedEncodingException e1) {
- e1.printStackTrace();
+ e1.printStackTrace(LogUtil.getWarnStream(LOG));
} catch (IOException e1) {
- e1.printStackTrace();
+ e1.printStackTrace(LogUtil.getWarnStream(LOG));
} catch (TransformerConfigurationException e2) {
- e2.printStackTrace();
+ e2.printStackTrace(LogUtil.getWarnStream(LOG));
} catch (TransformerException ex) {
- ex.printStackTrace();
+ ex.printStackTrace(LogUtil.getWarnStream(LOG));
}
}
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/util/GZIPUtils.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/util/GZIPUtils.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/util/GZIPUtils.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/util/GZIPUtils.java Mon Jun 12 13:51:40 2006
@@ -22,11 +22,16 @@
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
/**
* A collection of utility methods for working on GZIPed data.
*/
public class GZIPUtils {
-
+
+ private static final Log LOG = LogFactory.getLog(GZIPUtils.class);
private static final int EXPECTED_COMPRESSION_RATIO= 5;
private static final int BUF_SIZE= 4096;
@@ -124,19 +129,19 @@
try {
outStream.write(in);
} catch (Exception e) {
- e.printStackTrace();
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
}
try {
outStream.close();
} catch (IOException e) {
- e.printStackTrace();
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
}
return byteOut.toByteArray();
} catch (IOException e) {
- e.printStackTrace();
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
return null;
}
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/util/ThreadPool.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/util/ThreadPool.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/util/ThreadPool.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/util/ThreadPool.java Mon Jun 12 13:51:40 2006
@@ -17,8 +17,11 @@
package org.apache.nutch.util;
import java.util.*;
-import java.util.logging.*;
-import org.apache.hadoop.util.LogFormatter;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
/************************************************
* ThreadPool.java
@@ -70,8 +73,8 @@
TaskThread t = new TaskThread();
t.start();
}
- Logger l = LogFormatter.getLogger("org.apache.nutch.util");
- l.fine("ThreadPool created with " + numThreads + " threads.");
+ Log l = LogFactory.getLog("org.apache.nutch.util");
+ l.debug("ThreadPool created with " + numThreads + " threads.");
}
/**
@@ -126,7 +129,7 @@
*/
public void shutdown() {
running = false;
- Logger l = LogFormatter.getLogger("org.apache.nutch.util");
- l.fine("ThreadPool shutting down.");
+ Log l = LogFactory.getLog("org.apache.nutch.util");
+ l.debug("ThreadPool shutting down.");
}
}
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/util/mime/MimeTypes.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/util/mime/MimeTypes.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/util/mime/MimeTypes.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/util/mime/MimeTypes.java Mon Jun 12 13:51:40 2006
@@ -23,7 +23,9 @@
import java.util.HashMap;
import java.util.Iterator;
import java.util.ArrayList;
-import java.util.logging.Logger;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
/**
@@ -64,7 +66,7 @@
/** Should never be instanciated from outside */
- private MimeTypes(String filepath, Logger logger) {
+ private MimeTypes(String filepath, Log logger) {
MimeTypesReader reader = new MimeTypesReader(logger);
add(reader.read(filepath));
}
@@ -93,7 +95,7 @@
* @param logger is it Logger to uses for ouput messages.
* @return A MimeTypes instance for the specified filepath xml file.
*/
- public static MimeTypes get(String filepath, Logger logger) {
+ public static MimeTypes get(String filepath, Log logger) {
MimeTypes instance = null;
synchronized(instances) {
instance = (MimeTypes) instances.get(filepath);
Modified: lucene/nutch/trunk/src/java/org/apache/nutch/util/mime/MimeTypesReader.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/java/org/apache/nutch/util/mime/MimeTypesReader.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/java/org/apache/nutch/util/mime/MimeTypesReader.java (original)
+++ lucene/nutch/trunk/src/java/org/apache/nutch/util/mime/MimeTypesReader.java Mon Jun 12 13:51:40 2006
@@ -15,6 +15,10 @@
*/
package org.apache.nutch.util.mime;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
// DOM imports
import org.w3c.dom.Text;
import org.w3c.dom.Attr;
@@ -28,8 +32,6 @@
// JDK imports
import java.io.InputStream;
import java.util.ArrayList;
-import java.util.logging.Level;
-import java.util.logging.Logger;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -42,12 +44,12 @@
final class MimeTypesReader {
/** The logger to use */
- private Logger logger = null;
+ private Log logger = null;
- MimeTypesReader(Logger logger) {
+ MimeTypesReader(Log logger) {
if (logger == null) {
- this.logger = Logger.getLogger(this.getClass().getName());
+ this.logger = LogFactory.getLog(this.getClass());
} else {
this.logger = logger;
}
@@ -66,7 +68,7 @@
Document document = builder.parse(new InputSource(stream));
types = visit(document);
} catch (Exception e) {
- logger.warning(e.toString() + " while loading mime-types");
+ logger.warn(e.toString() + " while loading mime-types");
types = new MimeType[0];
}
return types;
@@ -121,8 +123,8 @@
type = new MimeType(name);
} catch (MimeTypeException mte) {
// Mime Type not valid... just ignore it
- if (logger.isLoggable(Level.INFO)) {
- logger.log(Level.INFO, mte.toString() + " ... Ignoring!");
+ if (logger.isInfoEnabled()) {
+ logger.info(mte.toString() + " ... Ignoring!");
}
return null;
}
Modified: lucene/nutch/trunk/src/plugin/analysis-fr/plugin.xml
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/analysis-fr/plugin.xml?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/analysis-fr/plugin.xml (original)
+++ lucene/nutch/trunk/src/plugin/analysis-fr/plugin.xml Mon Jun 12 13:51:40 2006
@@ -18,10 +18,10 @@
</requires>
<extension id="org.apache.nutch.analysis.fr"
- name="FrenchAnalyzer"
+ name="French Analyzer"
point="org.apache.nutch.analysis.NutchAnalyzer">
- <implementation id="org.apache.nutch.analysis.fr.FrenchAnalyzer"
+ <implementation id="FrenchAnalyzer"
class="org.apache.nutch.analysis.fr.FrenchAnalyzer">
<parameter name="lang" value="fr"/>
</implementation>
Modified: lucene/nutch/trunk/src/plugin/clustering-carrot2/src/java/org/apache/nutch/clustering/carrot2/Clusterer.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/clustering-carrot2/src/java/org/apache/nutch/clustering/carrot2/Clusterer.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/clustering-carrot2/src/java/org/apache/nutch/clustering/carrot2/Clusterer.java (original)
+++ lucene/nutch/trunk/src/plugin/clustering-carrot2/src/java/org/apache/nutch/clustering/carrot2/Clusterer.java Mon Jun 12 13:51:40 2006
@@ -17,12 +17,12 @@
package org.apache.nutch.clustering.carrot2;
import java.util.*;
-import java.util.logging.Level;
-import java.util.logging.Logger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.nutch.clustering.HitsCluster;
import org.apache.nutch.clustering.OnlineClusterer;
import org.apache.nutch.searcher.HitDetails;
@@ -66,8 +66,7 @@
/** Internal clustering process ID in Carrot2 LocalController */
private final static String PROCESS_ID = "nutch-lingo";
- public static final Logger logger =
- LogFormatter.getLogger(Clusterer.class.getName());
+ public static final Log logger = LogFactory.getLog(Clusterer.class);
/** The LocalController instance used for clustering */
private LocalController controller;
@@ -129,13 +128,13 @@
try {
Language lang = AllKnownLanguages.getLanguageForIsoCode(lcode);
if (lang == null) {
- logger.log(Level.WARNING, "Language not supported in Carrot2: " + lcode);
+ logger.warn("Language not supported in Carrot2: " + lcode);
} else {
languageList.add(lang);
- logger.log(Level.FINE, "Language loaded: " + lcode);
+ logger.debug("Language loaded: " + lcode);
}
} catch (Throwable t) {
- logger.log(Level.WARNING, "Language could not be loaded: " + lcode, t);
+ logger.warn("Language could not be loaded: " + lcode, t);
}
}
return new LingoLocalFilterComponent(
@@ -223,8 +222,8 @@
this.languages = conf.getStrings(CONF_PROP_LANGUAGES);
}
- logger.log(Level.INFO, "Default language: " + defaultLanguage);
- logger.log(Level.INFO, "Enabled languages: " + Arrays.asList(languages));
+ logger.info("Default language: " + defaultLanguage);
+ logger.info("Enabled languages: " + Arrays.asList(languages));
initialize();
}
Modified: lucene/nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCDeleteUnlicensedTool.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCDeleteUnlicensedTool.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCDeleteUnlicensedTool.java (original)
+++ lucene/nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCDeleteUnlicensedTool.java Mon Jun 12 13:51:40 2006
@@ -16,21 +16,21 @@
package org.creativecommons.nutch;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.nutch.indexer.Indexer;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.document.Document;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import java.io.*;
import java.util.Vector;
-import java.util.logging.Logger;
/** Deletes documents in a set of Lucene indexes that do not have a Creative
* Commons license. */
public class CCDeleteUnlicensedTool {
- private static final Logger LOG =
- LogFormatter.getLogger("org.creativecommons.nutch.CCDeleteUnlicensedTool");
+ private static final Log LOG = LogFactory.getLog(CCDeleteUnlicensedTool.class);
private IndexReader[] readers;
Modified: lucene/nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCIndexingFilter.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCIndexingFilter.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCIndexingFilter.java (original)
+++ lucene/nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCIndexingFilter.java Mon Jun 12 13:51:40 2006
@@ -31,18 +31,18 @@
import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.metadata.CreativeCommons;
-import java.util.logging.Logger;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.conf.Configuration;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import java.util.*;
import java.net.URL;
import java.net.MalformedURLException;
/** Adds basic searchable fields to a document. */
public class CCIndexingFilter implements IndexingFilter {
- public static final Logger LOG
- = LogFormatter.getLogger(CCIndexingFilter.class.getName());
+ public static final Log LOG = LogFactory.getLog(CCIndexingFilter.class);
/** The name of the document field we use. */
public static String FIELD = "cc";
@@ -99,7 +99,7 @@
addFeature(doc, feature);
}
} catch (MalformedURLException e) {
- LOG.warning("CC: failed to parse url: " + urlString + " : " + e);
+ LOG.warn("CC: failed to parse url: " + urlString + " : " + e);
}
}
Modified: lucene/nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java (original)
+++ lucene/nutch/trunk/src/plugin/creativecommons/src/java/org/creativecommons/nutch/CCParseFilter.java Mon Jun 12 13:51:40 2006
@@ -22,6 +22,9 @@
import org.apache.nutch.metadata.Metadata;
import org.apache.hadoop.conf.Configuration;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import java.util.*;
import java.io.*;
import java.net.*;
@@ -29,13 +32,10 @@
import org.xml.sax.InputSource;
import org.w3c.dom.*;
-import java.util.logging.Logger;
-import org.apache.hadoop.util.LogFormatter;
/** Adds metadata identifying the Creative Commons license used, if any. */
public class CCParseFilter implements HtmlParseFilter {
- public static final Logger LOG
- = LogFormatter.getLogger(CCParseFilter.class.getName());
+ public static final Log LOG = LogFactory.getLog(CCParseFilter.class);
/** Walks DOM tree, looking for RDF in comments and licenses in anchors.*/
@@ -175,7 +175,7 @@
DocumentBuilder parser = FACTORY.newDocumentBuilder();
doc = parser.parse(new InputSource(new StringReader(comment)));
} catch (Exception e) {
- LOG.warning("CC: Failed to parse RDF in "+base+": "+e);
+ LOG.warn("CC: Failed to parse RDF in "+base+": "+e);
//e.printStackTrace();
return;
}
@@ -183,7 +183,7 @@
// check that root is rdf:RDF
NodeList roots = doc.getElementsByTagNameNS(RDF_NS, "RDF");
if (roots.getLength() != 1) {
- LOG.warning("CC: No RDF root in "+base);
+ LOG.warn("CC: No RDF root in "+base);
return;
}
Element rdf = (Element)roots.item(0);
Modified: lucene/nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java (original)
+++ lucene/nutch/trunk/src/plugin/index-basic/src/java/org/apache/nutch/indexer/basic/BasicIndexingFilter.java Mon Jun 12 13:51:40 2006
@@ -16,6 +16,9 @@
package org.apache.nutch.indexer.basic;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -31,14 +34,11 @@
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
-import java.util.logging.Logger;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.conf.Configuration;
/** Adds basic searchable fields to a document. */
public class BasicIndexingFilter implements IndexingFilter {
- public static final Logger LOG
- = LogFormatter.getLogger(BasicIndexingFilter.class.getName());
+ public static final Log LOG = LogFactory.getLog(BasicIndexingFilter.class);
private int MAX_TITLE_LENGTH;
private Configuration conf;
@@ -75,7 +75,7 @@
doc.add(new Field("anchor", anchors[i], Field.Store.NO, Field.Index.TOKENIZED));
}
} catch (IOException ioe) {
- LOG.warning("BasicIndexingFilter: can't get anchors for " + url.toString());
+ LOG.warn("BasicIndexingFilter: can't get anchors for " + url.toString());
}
// title
Modified: lucene/nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java (original)
+++ lucene/nutch/trunk/src/plugin/index-more/src/java/org/apache/nutch/indexer/more/MoreIndexingFilter.java Mon Jun 12 13:51:40 2006
@@ -23,6 +23,9 @@
import org.apache.oro.text.regex.MatchResult;
import org.apache.oro.text.regex.MalformedPatternException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -46,9 +49,6 @@
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.UTF8;
-import org.apache.hadoop.util.LogFormatter;
-
-import java.util.logging.Logger;
import java.text.ParseException;
import java.text.SimpleDateFormat;
@@ -73,8 +73,7 @@
*/
public class MoreIndexingFilter implements IndexingFilter {
- public static final Logger LOG
- = LogFormatter.getLogger(MoreIndexingFilter.class.getName());
+ public static final Log LOG = LogFactory.getLog(MoreIndexingFilter.class);
/** A flag that tells if magic resolution must be performed */
private boolean MAGIC;
@@ -156,9 +155,9 @@
"dd.MM.yyyy zzz"
});
time = parsedDate.getTime();
- // LOG.warning(url + ": parsed date: " + date +" to:"+time);
+ // LOG.warn(url + ": parsed date: " + date +" to:"+time);
} catch (Exception e2) {
- LOG.warning(url + ": can't parse erroneous date: " + date);
+ LOG.warn(url + ": can't parse erroneous date: " + date);
}
}
return time;
@@ -195,7 +194,7 @@
try {
mimeType = new MimeType(contentType);
} catch (MimeTypeException e) {
- LOG.warning(url + e.toString());
+ LOG.warn(url + e.toString());
mimeType = null;
}
}
Modified: lucene/nutch/trunk/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/HTMLLanguageParser.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/HTMLLanguageParser.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/HTMLLanguageParser.java (original)
+++ lucene/nutch/trunk/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/HTMLLanguageParser.java Mon Jun 12 13:51:40 2006
@@ -20,7 +20,10 @@
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
-import java.util.logging.Logger;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
// Nutch imports
import org.apache.nutch.metadata.Metadata;
@@ -30,7 +33,6 @@
import org.apache.nutch.protocol.Content;
// Hadoop imports
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.conf.Configuration;
// DOM imports
@@ -47,8 +49,7 @@
*/
public class HTMLLanguageParser implements HtmlParseFilter {
- public static final Logger LOG = LogFormatter
- .getLogger(HTMLLanguageParser.class.getName());
+ public static final Log LOG = LogFactory.getLog(HTMLLanguageParser.class);
/* A static Map of ISO-639 language codes */
private static Map LANGUAGES_MAP = new HashMap();
@@ -67,7 +68,7 @@
}
}
} catch (Exception e) {
- LOG.severe(e.toString());
+ LOG.fatal(e.toString());
}
}
Modified: lucene/nutch/trunk/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/LanguageIdentifier.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/LanguageIdentifier.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/LanguageIdentifier.java (original)
+++ lucene/nutch/trunk/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/LanguageIdentifier.java Mon Jun 12 13:51:40 2006
@@ -30,12 +30,18 @@
import java.util.ArrayList;
import java.util.Properties;
import java.util.Enumeration;
-import java.util.logging.Logger;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+// Hadoop imports
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.UTF8;
// Nutch imports
import org.apache.nutch.analysis.lang.NGramProfile.NGramEntry;
import org.apache.nutch.crawl.CrawlDatum;
-import org.apache.hadoop.io.UTF8;
import org.apache.nutch.parse.Parse;
import org.apache.nutch.parse.ParseUtil;
import org.apache.nutch.parse.ParseException;
@@ -45,8 +51,6 @@
import org.apache.nutch.protocol.ProtocolFactory;
import org.apache.nutch.protocol.ProtocolNotFound;
import org.apache.nutch.protocol.ProtocolException;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.nutch.util.NutchConfiguration;
@@ -66,8 +70,7 @@
private final static float SCORE_THRESOLD = 0.00F;
- private final static Logger LOG =
- LogFormatter.getLogger(LanguageIdentifier.class.getName());
+ private final static Log LOG = LogFactory.getLog(LanguageIdentifier.class);
private ArrayList languages = new ArrayList();
@@ -152,7 +155,7 @@
list.append(" " + lang + "(" + ngrams.size() + ")");
is.close();
} catch (IOException e1) {
- LOG.severe(e1.toString());
+ LOG.fatal(e1.toString());
}
}
}
@@ -170,7 +173,7 @@
// Create the suspect profile
suspect = new NGramProfile("suspect", minLength, maxLength);
} catch (Exception e) {
- LOG.severe(e.toString());
+ LOG.fatal(e.toString());
}
}
Modified: lucene/nutch/trunk/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/NGramProfile.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/NGramProfile.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/NGramProfile.java (original)
+++ lucene/nutch/trunk/src/plugin/languageidentifier/src/java/org/apache/nutch/analysis/lang/NGramProfile.java Mon Jun 12 13:51:40 2006
@@ -32,14 +32,17 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
-import java.util.logging.Logger;
-// Nutch imports
-import org.apache.hadoop.util.LogFormatter;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
// Lucene imports
import org.apache.lucene.analysis.Token;
+// Nutch imports
+import org.apache.nutch.util.LogUtil;
+
/**
* This class runs a ngram analysis over submitted text, results might be used
@@ -54,8 +57,7 @@
*/
public class NGramProfile {
- public static final Logger LOG = LogFormatter
- .getLogger("org.apache.nutch.analysis.lang.NGramProfile");
+ public static final Log LOG = LogFactory.getLog(NGramProfile.class);
/** The minimum length allowed for a ngram. */
final static int ABSOLUTE_MIN_NGRAM_LENGTH = 1;
@@ -317,7 +319,7 @@
}
}
} catch (Exception e) {
- LOG.severe(e.toString());
+ LOG.fatal(e.toString());
}
return sum;
}
@@ -374,7 +376,7 @@
text.append(new String(buffer, 0, len, encoding));
}
} catch (IOException e) {
- e.printStackTrace();
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
}
newProfile.analyze(text);
@@ -521,7 +523,7 @@
}
} catch (Exception e) {
- LOG.severe("Caught an exception:" + e);
+ LOG.fatal("Caught an exception:" + e);
}
}
Modified: lucene/nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java (original)
+++ lucene/nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java Mon Jun 12 13:51:40 2006
@@ -15,15 +15,17 @@
*/
package org.apache.nutch.protocol.http.api;
-// Nutch imports
+// JDK imports
import java.io.IOException;
import java.net.InetAddress;
import java.net.URL;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.LinkedList;
-import java.util.logging.Level;
-import java.util.logging.Logger;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
// Nutch imports
import org.apache.nutch.crawl.CrawlDatum;
@@ -34,6 +36,7 @@
import org.apache.nutch.protocol.ProtocolOutput;
import org.apache.nutch.protocol.ProtocolStatus;
import org.apache.nutch.util.GZIPUtils;
+import org.apache.nutch.util.LogUtil;
// Hadoop imports
import org.apache.hadoop.conf.Configuration;
@@ -109,10 +112,10 @@
private static LinkedList BLOCKED_ADDR_QUEUE = new LinkedList();
/** The default logger */
- private final static Logger LOGGER = Logger.getLogger(HttpBase.class.getName());
+ private final static Log LOGGER = LogFactory.getLog(HttpBase.class);
/** The specified logger */
- private Logger logger = LOGGER;
+ private Log logger = LOGGER;
/** The nutch configuration */
private Configuration conf = null;
@@ -129,7 +132,7 @@
}
/** Creates a new instance of HttpBase */
- public HttpBase(Logger logger) {
+ public HttpBase(Log logger) {
if (logger != null) {
this.logger = logger;
}
@@ -175,7 +178,7 @@
}
} catch (Throwable e) {
// XXX Maybe bogus: assume this is allowed.
- logger.fine("Exception checking robot rules for " + url + ": " + e);
+ logger.trace("Exception checking robot rules for " + url + ": " + e);
}
String host = blockAddr(u);
@@ -228,10 +231,10 @@
// handle this in the higher layer.
return new ProtocolOutput(c, new ProtocolStatus(protocolStatusCode, u));
} else if (code == 400) { // bad request, mark as GONE
- logger.fine("400 Bad request: " + u);
+ logger.trace("400 Bad request: " + u);
return new ProtocolOutput(c, new ProtocolStatus(ProtocolStatus.GONE, u));
} else if (code == 401) { // requires authorization, but no valid auth provided.
- logger.fine("401 Authentication Required");
+ logger.trace("401 Authentication Required");
return new ProtocolOutput(c, new ProtocolStatus(ProtocolStatus.ACCESS_DENIED, "Authentication required: "
+ urlString));
} else if (code == 404) {
@@ -243,7 +246,7 @@
+ u));
}
} catch (Throwable e) {
- e.printStackTrace();
+ e.printStackTrace(LogUtil.getErrorStream(logger));
return new ProtocolOutput(null, new ProtocolStatus(e));
}
}
@@ -390,7 +393,7 @@
String agentEmail) {
if ( (agentName == null) || (agentName.trim().length() == 0) )
- LOGGER.severe("No User-Agent string set (http.agent.name)!");
+ LOGGER.fatal("No User-Agent string set (http.agent.name)!");
StringBuffer buf= new StringBuffer();
@@ -435,17 +438,18 @@
}
public byte[] processGzipEncoded(byte[] compressed, URL url) throws IOException {
- LOGGER.fine("uncompressing....");
+ LOGGER.trace("uncompressing....");
byte[] content = GZIPUtils.unzipBestEffort(compressed, getMaxContent());
if (content == null)
throw new IOException("unzipBestEffort returned null");
- if (LOGGER.isLoggable(Level.FINE))
- LOGGER.fine("fetched " + compressed.length
- + " bytes of compressed content (expanded to "
- + content.length + " bytes) from " + url);
+ if (LOGGER.isTraceEnabled()) {
+ LOGGER.trace("fetched " + compressed.length
+ + " bytes of compressed content (expanded to "
+ + content.length + " bytes) from " + url);
+ }
return content;
}
@@ -472,9 +476,9 @@
url = args[i];
}
- if (verbose) {
- LOGGER.setLevel(Level.FINE);
- }
+// if (verbose) {
+// LOGGER.setLevel(Level.FINE);
+// }
ProtocolOutput out = http.getProtocolOutput(new UTF8(url), new CrawlDatum());
Content content = out.getContent();
Modified: lucene/nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/RobotRulesParser.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/RobotRulesParser.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/RobotRulesParser.java (original)
+++ lucene/nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/RobotRulesParser.java Mon Jun 12 13:51:40 2006
@@ -26,12 +26,13 @@
import java.util.HashMap;
import java.util.Hashtable;
import java.util.StringTokenizer;
-import java.util.logging.Logger;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
// Nutch imports
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configurable;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.ProtocolException;
@@ -48,8 +49,7 @@
*/
public class RobotRulesParser implements Configurable {
- public static final Logger LOG=
- LogFormatter.getLogger(RobotRulesParser.class.getName());
+ public static final Log LOG = LogFactory.getLog(RobotRulesParser.class);
private boolean allowForbidden = false;
@@ -206,11 +206,11 @@
//
if (agents.size() == 0) {
agents.add(agentName);
- LOG.severe("No agents listed in 'http.robots.agents' property!");
+ LOG.fatal("No agents listed in 'http.robots.agents' property!");
} else if (!((String)agents.get(0)).equalsIgnoreCase(agentName)) {
agents.add(0, agentName);
- LOG.severe("Agent we advertise (" + agentName
- + ") not listed first in 'http.robots.agents' property!");
+ LOG.fatal("Agent we advertise (" + agentName
+ + ") not listed first in 'http.robots.agents' property!");
}
setRobotNames((String[]) agents.toArray(new String[agents.size()]));
}
@@ -320,8 +320,7 @@
try {
path= URLDecoder.decode(path, CHARACTER_ENCODING);
} catch (Exception e) {
- LOG.warning("error parsing robots rules- can't decode path: "
- + path);
+ LOG.warn("error parsing robots rules- can't decode path: " + path);
}
if (path.length() == 0) { // "empty rule"
@@ -389,7 +388,7 @@
RobotRuleSet robotRules = (RobotRuleSet)CACHE.get(host);
if (robotRules == null) { // cache miss
- LOG.fine("cache miss " + url);
+ LOG.trace("cache miss " + url);
try {
Response response = http.getResponse(new URL(url, "/robots.txt"),
new CrawlDatum(), true);
Modified: lucene/nutch/trunk/src/plugin/lib-parsems/src/java/org/apache/nutch/parse/ms/MSBaseParser.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/lib-parsems/src/java/org/apache/nutch/parse/ms/MSBaseParser.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/lib-parsems/src/java/org/apache/nutch/parse/ms/MSBaseParser.java (original)
+++ lucene/nutch/trunk/src/plugin/lib-parsems/src/java/org/apache/nutch/parse/ms/MSBaseParser.java Mon Jun 12 13:51:40 2006
@@ -20,11 +20,13 @@
import java.io.File;
import java.io.FileInputStream;
import java.util.Properties;
-import java.util.logging.Logger;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
// Hadoop imports
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.LogFormatter;
// Nutch imports
import org.apache.nutch.metadata.DublinCore;
@@ -38,6 +40,7 @@
import org.apache.nutch.parse.ParseStatus;
import org.apache.nutch.parse.Parser;
import org.apache.nutch.protocol.Content;
+import org.apache.nutch.util.LogUtil;
import org.apache.nutch.util.NutchConfiguration;
@@ -50,8 +53,7 @@
private Configuration conf;
- protected static final Logger LOG =
- LogFormatter.getLogger(MSBaseParser.class.getName());
+ protected static final Log LOG = LogFactory.getLog(MSBaseParser.class);
/**
@@ -135,7 +137,7 @@
fin.close();
return buffer;
} catch (Exception err) {
- err.printStackTrace();
+ err.printStackTrace(LogUtil.getErrorStream(LOG));
return null;
}
Modified: lucene/nutch/trunk/src/plugin/lib-parsems/src/java/org/apache/nutch/parse/ms/MSExtractor.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/lib-parsems/src/java/org/apache/nutch/parse/ms/MSExtractor.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/lib-parsems/src/java/org/apache/nutch/parse/ms/MSExtractor.java (original)
+++ lucene/nutch/trunk/src/plugin/lib-parsems/src/java/org/apache/nutch/parse/ms/MSExtractor.java Mon Jun 12 13:51:40 2006
@@ -19,10 +19,10 @@
import java.io.InputStream;
import java.util.Date;
import java.util.Properties;
-import java.util.logging.Logger;
-// Hadoop imports
-import org.apache.hadoop.util.LogFormatter;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
// Nutch imports
import org.apache.nutch.metadata.DublinCore;
@@ -47,8 +47,7 @@
*/
public abstract class MSExtractor {
- protected final static Logger LOG =
- LogFormatter.getLogger(MSExtractor.class.getName());
+ protected final static Log LOG = LogFactory.getLog(MSExtractor.class);
private String text = null;
private POIFSReader reader = null;
Modified: lucene/nutch/trunk/src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexURLFilterBase.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexURLFilterBase.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexURLFilterBase.java (original)
+++ lucene/nutch/trunk/src/plugin/lib-regex-filter/src/java/org/apache/nutch/urlfilter/api/RegexURLFilterBase.java Mon Jun 12 13:51:40 2006
@@ -23,11 +23,15 @@
import java.io.IOException;
import java.util.List;
import java.util.ArrayList;
-import java.util.logging.Logger;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
// Hadoop imports
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.LogFormatter;
+
+// Nutch imports
import org.apache.nutch.net.*;
@@ -51,8 +55,7 @@
public abstract class RegexURLFilterBase implements URLFilter {
/** My logger */
- private final static Logger LOG =
- LogFormatter.getLogger(RegexURLFilterBase.class.getName());
+ private final static Log LOG = LogFactory.getLog(RegexURLFilterBase.class);
/** An array of applicable rules */
private RegexRule[] rules;
@@ -131,12 +134,12 @@
String file = getRulesFile(conf);
Reader reader = conf.getConfResourceAsReader(file);
if (reader == null) {
- LOG.severe("Can't find resource: " + file);
+ LOG.fatal("Can't find resource: " + file);
} else {
try {
rules = readRulesFile(reader);
} catch (IOException e) {
- LOG.severe(e.getMessage());
+ LOG.fatal(e.getMessage());
//TODO mb@media-style.com: throw Exception? Because broken api.
throw new RuntimeException(e.getMessage(), e);
}
@@ -184,7 +187,7 @@
}
String regex = line.substring(1);
- LOG.fine("Adding rule [" + regex + "]");
+ LOG.trace("Adding rule [" + regex + "]");
RegexRule rule = createRule(sign, regex);
rules.add(rule);
}
Modified: lucene/nutch/trunk/src/plugin/lib-regex-filter/src/test/org/apache/nutch/urlfilter/api/RegexURLFilterBaseTest.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/lib-regex-filter/src/test/org/apache/nutch/urlfilter/api/RegexURLFilterBaseTest.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/lib-regex-filter/src/test/org/apache/nutch/urlfilter/api/RegexURLFilterBaseTest.java (original)
+++ lucene/nutch/trunk/src/plugin/lib-regex-filter/src/test/org/apache/nutch/urlfilter/api/RegexURLFilterBaseTest.java Mon Jun 12 13:51:40 2006
@@ -22,13 +22,13 @@
import java.io.Reader;
import java.util.ArrayList;
import java.util.List;
-import java.util.logging.Logger;
// JUnit imports
import junit.framework.TestCase;
-// Hadoop imports
-import org.apache.hadoop.util.LogFormatter;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
// Nutch imports
import org.apache.nutch.net.URLFilter;
@@ -42,8 +42,7 @@
public abstract class RegexURLFilterBaseTest extends TestCase {
/** My logger */
- protected static final Logger LOG =
- LogFormatter.getLogger(RegexURLFilterBaseTest.class.getName());
+ protected static final Log LOG = LogFactory.getLog(RegexURLFilterBaseTest.class);
private final static String SEPARATOR = System.getProperty("file.separator");
private final static String SAMPLES = System.getProperty("test.data", ".");
Modified: lucene/nutch/trunk/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java (original)
+++ lucene/nutch/trunk/src/plugin/microformats-reltag/src/java/org/apache/nutch/microformats/reltag/RelTagParser.java Mon Jun 12 13:51:40 2006
@@ -21,12 +21,15 @@
import java.util.Iterator;
import java.util.Set;
import java.util.TreeSet;
-import java.util.logging.Logger;
import org.w3c.dom.DocumentFragment;
import org.w3c.dom.NamedNodeMap;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
// Nutch imports
import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.parse.HTMLMetaTags;
@@ -37,7 +40,6 @@
// Hadoop imports
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.LogFormatter;
/**
@@ -49,8 +51,7 @@
*/
public class RelTagParser implements HtmlParseFilter {
- public final static Logger LOG =
- LogFormatter.getLogger(RelTagParser.class.getName());
+ public final static Log LOG = LogFactory.getLog(RelTagParser.class);
public final static String REL_TAG = "Rel-Tag";
Modified: lucene/nutch/trunk/src/plugin/ontology/src/java/org/apache/nutch/ontology/jena/OntologyImpl.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/ontology/src/java/org/apache/nutch/ontology/jena/OntologyImpl.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/ontology/src/java/org/apache/nutch/ontology/jena/OntologyImpl.java (original)
+++ lucene/nutch/trunk/src/plugin/ontology/src/java/org/apache/nutch/ontology/jena/OntologyImpl.java Mon Jun 12 13:51:40 2006
@@ -17,7 +17,6 @@
package org.apache.nutch.ontology.jena;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.nutch.ontology.*;
import org.apache.nutch.util.NutchConfiguration;
@@ -32,6 +31,9 @@
import com.hp.hpl.jena.rdf.model.ModelFactory;
import com.hp.hpl.jena.shared.PrefixMapping;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import java.util.Map;
import java.util.HashMap;
import java.util.Hashtable;
@@ -39,7 +41,6 @@
import java.util.List;
import java.util.LinkedList;
-import java.util.logging.Logger;
import java.io.PrintStream;
@@ -51,8 +52,7 @@
* @author michael j pan
*/
public class OntologyImpl implements org.apache.nutch.ontology.Ontology {
- public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.ontology.Ontology");
+ public static final Log LOG = LogFactory.getLog("org.apache.nutch.ontology.Ontology");
public final static String DELIMITER_SEARCHTERM = " ";
@@ -104,11 +104,11 @@
LOG.info( "reading "+url);
((OntModel)m).read(url);
} catch (Exception e) {
- LOG.severe("failed on attempting to read ontology "+url);
- LOG.severe(e.getMessage());
+ LOG.fatal("failed on attempting to read ontology "+url);
+ LOG.fatal(e.getMessage());
StackTraceElement[] traces = e.getStackTrace();
for (int i=0; i<traces.length; i++) {
- LOG.severe(traces[i].toString());
+ LOG.fatal(traces[i].toString());
}
}
}
@@ -332,7 +332,7 @@
String urls = conf.get("extension.ontology.urls");
if (urls==null || urls.trim().equals("")) {
- LOG.severe("No ontology url found.");
+ LOG.fatal("No ontology url found.");
return;
}
ontology.load(urls.split("\\s+"));
Modified: lucene/nutch/trunk/src/plugin/ontology/src/java/org/apache/nutch/ontology/jena/OwlParser.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/ontology/src/java/org/apache/nutch/ontology/jena/OwlParser.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/ontology/src/java/org/apache/nutch/ontology/jena/OwlParser.java (original)
+++ lucene/nutch/trunk/src/plugin/ontology/src/java/org/apache/nutch/ontology/jena/OwlParser.java Mon Jun 12 13:51:40 2006
@@ -21,13 +21,17 @@
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-//import org.apache.hadoop.util.LogFormatter;
+
+// Commons Logging imports
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import com.hp.hpl.jena.ontology.OntClass;
import com.hp.hpl.jena.ontology.OntModel;
import com.hp.hpl.jena.ontology.Individual;
import com.hp.hpl.jena.rdf.model.Literal;
import org.apache.nutch.ontology.*;
+import org.apache.nutch.util.LogUtil;
/**
* implementation of parser for w3c's OWL files
@@ -35,6 +39,9 @@
* @author michael j pan
*/
public class OwlParser implements Parser {
+
+ private final static Log LOG = LogFactory.getLog(OwlParser.class);
+
public OwlParser () {
}
@@ -123,8 +130,7 @@
roots.add( c );
}
} catch (Exception e) {
- //e.printStackTrace();
- System.out.println(e.getMessage());
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
}
}
Modified: lucene/nutch/trunk/src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java (original)
+++ lucene/nutch/trunk/src/plugin/parse-ext/src/java/org/apache/nutch/parse/ext/ExtParser.java Mon Jun 12 13:51:40 2006
@@ -25,7 +25,6 @@
import org.apache.nutch.parse.Outlink;
import org.apache.nutch.parse.OutlinkExtractor;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.nutch.util.CommandRunner;
import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.net.protocols.Response;
@@ -34,9 +33,10 @@
import org.apache.nutch.plugin.Extension;
import org.apache.nutch.plugin.PluginRepository;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import java.util.Hashtable;
-import java.util.logging.Level;
-import java.util.logging.Logger;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
@@ -49,8 +49,7 @@
public class ExtParser implements Parser {
- public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.parse.ext");
+ public static final Log LOG = LogFactory.getLog("org.apache.nutch.parse.ext");
static final int BUFFER_SIZE = 4096;
@@ -77,8 +76,9 @@
String command = params[0];
int timeout = Integer.parseInt(params[1]);
- if (LOG.isLoggable(Level.FINE))
- LOG.fine("Use "+command+ " with timeout="+timeout+"secs");
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Use "+command+ " with timeout="+timeout+"secs");
+ }
String text = null;
String title = null;
Modified: lucene/nutch/trunk/src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java (original)
+++ lucene/nutch/trunk/src/plugin/parse-html/src/java/org/apache/nutch/parse/html/HtmlParser.java Mon Jun 12 13:51:40 2006
@@ -17,7 +17,6 @@
package org.apache.nutch.parse.html;
import java.util.ArrayList;
-import java.util.logging.*;
import java.net.URL;
import java.net.MalformedURLException;
import java.io.*;
@@ -29,17 +28,18 @@
import org.w3c.dom.*;
import org.apache.html.dom.*;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.Content;
import org.apache.hadoop.conf.*;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.nutch.parse.*;
import org.apache.nutch.util.*;
public class HtmlParser implements Parser {
- public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.parse.html");
+ public static final Log LOG = LogFactory.getLog("org.apache.nutch.parse.html");
// I used 1000 bytes at first, but found that some documents have
// meta tag well past the first 1000 bytes.
@@ -126,7 +126,7 @@
metadata.set(Metadata.ORIGINAL_CHAR_ENCODING, encoding);
if ((encoding = StringUtil.resolveEncodingAlias(encoding)) != null) {
metadata.set(Metadata.CHAR_ENCODING_FOR_CONVERSION, encoding);
- LOG.fine(base + ": setting encoding to " + encoding);
+ LOG.trace(base + ": setting encoding to " + encoding);
}
}
@@ -137,7 +137,7 @@
metadata.set(Metadata.ORIGINAL_CHAR_ENCODING, encoding);
if ((encoding = StringUtil.resolveEncodingAlias(encoding)) != null) {
metadata.set(Metadata.CHAR_ENCODING_FOR_CONVERSION, encoding);
- LOG.fine(base + ": setting encoding to " + encoding);
+ LOG.trace(base + ": setting encoding to " + encoding);
}
}
}
@@ -151,10 +151,10 @@
// same share)
encoding = defaultCharEncoding;
metadata.set(Metadata.CHAR_ENCODING_FOR_CONVERSION, defaultCharEncoding);
- LOG.fine(base + ": falling back to " + defaultCharEncoding);
+ LOG.trace(base + ": falling back to " + defaultCharEncoding);
}
input.setEncoding(encoding);
- LOG.fine("Parsing...");
+ LOG.trace("Parsing...");
root = parse(input);
} catch (IOException e) {
return new ParseStatus(e).getEmptyParse(getConf());
@@ -163,21 +163,21 @@
} catch (SAXException e) {
return new ParseStatus(e).getEmptyParse(getConf());
} catch (Exception e) {
- e.printStackTrace();
+ e.printStackTrace(LogUtil.getWarnStream(LOG));
return new ParseStatus(e).getEmptyParse(getConf());
}
// get meta directives
HTMLMetaProcessor.getMetaTags(metaTags, root, base);
- LOG.fine("Meta tags for " + base + ": " + metaTags.toString());
+ LOG.trace("Meta tags for " + base + ": " + metaTags.toString());
// check meta directives
if (!metaTags.getNoIndex()) { // okay to index
StringBuffer sb = new StringBuffer();
- LOG.fine("Getting text...");
+ LOG.trace("Getting text...");
utils.getText(sb, root); // extract text
text = sb.toString();
sb.setLength(0);
- LOG.fine("Getting title...");
+ LOG.trace("Getting title...");
utils.getTitle(sb, root); // extract title
title = sb.toString().trim();
}
@@ -185,10 +185,10 @@
if (!metaTags.getNoFollow()) { // okay to follow links
ArrayList l = new ArrayList(); // extract outlinks
URL baseTag = utils.getBase(root);
- LOG.fine("Getting links...");
+ LOG.trace("Getting links...");
utils.getOutlinks(baseTag!=null?baseTag:base, l, root);
outlinks = (Outlink[])l.toArray(new Outlink[l.size()]);
- LOG.fine("found "+outlinks.length+" outlinks in "+content.getUrl());
+ LOG.trace("found "+outlinks.length+" outlinks in "+content.getUrl());
}
if (!metaTags.getNoCache()) { // okay to cache
@@ -259,12 +259,12 @@
LOG.info(" - new frag, " + frag.getChildNodes().getLength() + " nodes.");
res.appendChild(frag);
}
- } catch (Exception x) { x.printStackTrace();};
+ } catch (Exception x) { x.printStackTrace(LogUtil.getWarnStream(LOG));};
return res;
}
public static void main(String[] args) throws Exception {
- LOG.setLevel(Level.FINE);
+ //LOG.setLevel(Level.FINE);
String name = args[0];
String url = "file:"+name;
File file = new File(name);
Modified: lucene/nutch/trunk/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java
URL: http://svn.apache.org/viewvc/lucene/nutch/trunk/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java?rev=413742&r1=413741&r2=413742&view=diff
==============================================================================
--- lucene/nutch/trunk/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java (original)
+++ lucene/nutch/trunk/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java Mon Jun 12 13:51:40 2006
@@ -8,7 +8,9 @@
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import java.util.logging.Logger;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.nutch.parse.HTMLMetaTags;
import org.apache.nutch.parse.HtmlParseFilter;
@@ -20,7 +22,6 @@
import org.apache.nutch.parse.Parser;
import org.apache.nutch.protocol.Content;
import org.apache.nutch.util.NutchConfiguration;
-import org.apache.hadoop.util.LogFormatter;
import org.apache.hadoop.conf.Configuration;
import org.apache.oro.text.regex.MatchResult;
import org.apache.oro.text.regex.Pattern;
@@ -44,8 +45,7 @@
* @author Andrzej Bialecki <ab@getopt.org>
*/
public class JSParseFilter implements HtmlParseFilter, Parser {
- public static final Logger LOG =
- LogFormatter.getLogger("org.apache.nutch.parse.js.JSParseFilter");
+ public static final Log LOG = LogFactory.getLog(JSParseFilter.class);
private static final int MAX_TITLE_LEN = 80;
@@ -164,7 +164,7 @@
try {
baseURL = new URL(base);
} catch (Exception e) {
- LOG.throwing(JSParseFilter.class.getName(), "getJSLinks", e);
+ LOG.error("getJSLinks", e);
}
try {
@@ -189,20 +189,20 @@
url = result.group(2);
PatternMatcherInput input1 = new PatternMatcherInput(url);
if (!matcher1.matches(input1, pattern1)) {
- //LOG.fine(" - invalid '" + url + "'");
+ //LOG.trace(" - invalid '" + url + "'");
continue;
}
if (url.startsWith("www.")) {
url = "http://" + url;
} else url = new URL(baseURL, url).toString();
url = url.replaceAll("&", "&");
- LOG.fine(" - outlink from JS: '" + url + "'");
+ LOG.trace(" - outlink from JS: '" + url + "'");
outlinks.add(new Outlink(url, anchor, getConf()));
}
} catch (Exception ex) {
// if it is a malformed URL we just throw it away and continue with
// extraction.
- LOG.throwing(JSParseFilter.class.getName(), "getJSLinks", ex);
+ LOG.error("getJSLinks", ex);
}
final Outlink[] retval;