You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by jn...@apache.org on 2011/08/16 15:35:52 UTC
svn commit: r1158269 - in /nutch/branches/branch-1.4: CHANGES.txt
src/java/org/apache/nutch/parse/ParserChecker.java
Author: jnioche
Date: Tue Aug 16 13:35:52 2011
New Revision: 1158269
URL: http://svn.apache.org/viewvc?rev=1158269&view=rev
Log:
NUTCH-1083 ParserChecker implements Tools
Modified:
nutch/branches/branch-1.4/CHANGES.txt
nutch/branches/branch-1.4/src/java/org/apache/nutch/parse/ParserChecker.java
Modified: nutch/branches/branch-1.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.4/CHANGES.txt?rev=1158269&r1=1158268&r2=1158269&view=diff
==============================================================================
--- nutch/branches/branch-1.4/CHANGES.txt (original)
+++ nutch/branches/branch-1.4/CHANGES.txt Tue Aug 16 13:35:52 2011
@@ -2,6 +2,8 @@ Nutch Change Log
Release 1.4 - Current development
+* NUTCH-1083 ParserChecker implements Tools (jnioche)
+
* NUTCH-1082 IndexingFiltersChecker utility does not list multi valued fields (markus)
* NUTCH-1004 Do not index empty values for title field (markus)
Modified: nutch/branches/branch-1.4/src/java/org/apache/nutch/parse/ParserChecker.java
URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.4/src/java/org/apache/nutch/parse/ParserChecker.java?rev=1158269&r1=1158268&r2=1158269&view=diff
==============================================================================
--- nutch/branches/branch-1.4/src/java/org/apache/nutch/parse/ParserChecker.java (original)
+++ nutch/branches/branch-1.4/src/java/org/apache/nutch/parse/ParserChecker.java Tue Aug 16 13:35:52 2011
@@ -19,18 +19,15 @@ package org.apache.nutch.parse;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
import org.apache.hadoop.conf.Configuration;
-
-import org.apache.nutch.util.NutchConfiguration;
-
-import org.apache.nutch.crawl.CrawlDatum;
import org.apache.hadoop.io.Text;
-import org.apache.nutch.parse.ParseUtil;
-
-import org.apache.nutch.protocol.ProtocolFactory;
-import org.apache.nutch.protocol.Protocol;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.protocol.Content;
+import org.apache.nutch.protocol.Protocol;
+import org.apache.nutch.protocol.ProtocolFactory;
+import org.apache.nutch.util.NutchConfiguration;
/**
* Parser checker, useful for testing parser.
@@ -38,13 +35,16 @@ import org.apache.nutch.protocol.Content
* @author John Xing
*/
-public class ParserChecker {
+public class ParserChecker implements Tool {
public static final Log LOG = LogFactory.getLog(ParserChecker.class);
- public ParserChecker() {}
+ public ParserChecker() {
+ }
- public static void main(String[] args) throws Exception {
+ Configuration conf = null;
+
+ public int run(String[] args) throws Exception {
boolean dumpText = false;
boolean force = false;
String contentType = null;
@@ -63,7 +63,7 @@ public class ParserChecker {
contentType = args[++i];
} else if (args[i].equals("-dumpText")) {
dumpText = true;
- } else if (i != args.length-1) {
+ } else if (i != args.length - 1) {
System.err.println(usage);
System.exit(-1);
} else {
@@ -71,12 +71,19 @@ public class ParserChecker {
}
}
- if (LOG.isInfoEnabled()) { LOG.info("fetching: "+url); }
+ if (LOG.isInfoEnabled()) {
+ LOG.info("fetching: " + url);
+ }
- Configuration conf = NutchConfiguration.create();
ProtocolFactory factory = new ProtocolFactory(conf);
Protocol protocol = factory.getProtocol(url);
- Content content = protocol.getProtocolOutput(new Text(url), new CrawlDatum()).getContent();
+ Content content = protocol.getProtocolOutput(new Text(url),
+ new CrawlDatum()).getContent();
+
+ if (content == null) {
+ System.err.println("Can't fetch URL successfully");
+ return (-1);
+ }
if (force) {
content.setContentType(contentType);
@@ -86,12 +93,12 @@ public class ParserChecker {
if (contentType == null) {
System.err.println("");
- System.exit(-1);
+ return (-1);
}
if (LOG.isInfoEnabled()) {
- LOG.info("parsing: "+url);
- LOG.info("contentType: "+contentType);
+ LOG.info("parsing: " + url);
+ LOG.info("contentType: " + contentType);
}
ParseResult parseResult = new ParseUtil(conf).parse(content);
@@ -108,6 +115,23 @@ public class ParserChecker {
}
}
- System.exit(0);
+ return 0;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+
+ @Override
+ public void setConf(Configuration c) {
+ conf = c;
}
+
+ public static void main(String[] args) throws Exception {
+ int res = ToolRunner.run(NutchConfiguration.create(), new ParserChecker(),
+ args);
+ System.exit(res);
+ }
+
}