You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by jn...@apache.org on 2011/08/16 15:35:52 UTC

svn commit: r1158269 - in /nutch/branches/branch-1.4: CHANGES.txt src/java/org/apache/nutch/parse/ParserChecker.java

Author: jnioche
Date: Tue Aug 16 13:35:52 2011
New Revision: 1158269

URL: http://svn.apache.org/viewvc?rev=1158269&view=rev
Log:
NUTCH-1083 ParserChecker implements Tools

Modified:
    nutch/branches/branch-1.4/CHANGES.txt
    nutch/branches/branch-1.4/src/java/org/apache/nutch/parse/ParserChecker.java

Modified: nutch/branches/branch-1.4/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.4/CHANGES.txt?rev=1158269&r1=1158268&r2=1158269&view=diff
==============================================================================
--- nutch/branches/branch-1.4/CHANGES.txt (original)
+++ nutch/branches/branch-1.4/CHANGES.txt Tue Aug 16 13:35:52 2011
@@ -2,6 +2,8 @@ Nutch Change Log
 
 Release 1.4 - Current development
 
+* NUTCH-1083 ParserChecker implements Tools (jnioche)
+
 * NUTCH-1082 IndexingFiltersChecker utility does not list multi valued fields (markus)
 
 * NUTCH-1004 Do not index empty values for title field (markus)

Modified: nutch/branches/branch-1.4/src/java/org/apache/nutch/parse/ParserChecker.java
URL: http://svn.apache.org/viewvc/nutch/branches/branch-1.4/src/java/org/apache/nutch/parse/ParserChecker.java?rev=1158269&r1=1158268&r2=1158269&view=diff
==============================================================================
--- nutch/branches/branch-1.4/src/java/org/apache/nutch/parse/ParserChecker.java (original)
+++ nutch/branches/branch-1.4/src/java/org/apache/nutch/parse/ParserChecker.java Tue Aug 16 13:35:52 2011
@@ -19,18 +19,15 @@ package org.apache.nutch.parse;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-
 import org.apache.hadoop.conf.Configuration;
-
-import org.apache.nutch.util.NutchConfiguration;
-
-import org.apache.nutch.crawl.CrawlDatum;
 import org.apache.hadoop.io.Text;
-import org.apache.nutch.parse.ParseUtil;
-
-import org.apache.nutch.protocol.ProtocolFactory;
-import org.apache.nutch.protocol.Protocol;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.nutch.crawl.CrawlDatum;
 import org.apache.nutch.protocol.Content;
+import org.apache.nutch.protocol.Protocol;
+import org.apache.nutch.protocol.ProtocolFactory;
+import org.apache.nutch.util.NutchConfiguration;
 
 /**
  * Parser checker, useful for testing parser.
@@ -38,13 +35,16 @@ import org.apache.nutch.protocol.Content
  * @author John Xing
  */
 
-public class ParserChecker {
+public class ParserChecker implements Tool {
 
   public static final Log LOG = LogFactory.getLog(ParserChecker.class);
 
-  public ParserChecker() {}
+  public ParserChecker() {
+  }
 
-  public static void main(String[] args) throws Exception {
+  Configuration conf = null;
+
+  public int run(String[] args) throws Exception {
     boolean dumpText = false;
     boolean force = false;
     String contentType = null;
@@ -63,7 +63,7 @@ public class ParserChecker {
         contentType = args[++i];
       } else if (args[i].equals("-dumpText")) {
         dumpText = true;
-      } else if (i != args.length-1) {
+      } else if (i != args.length - 1) {
         System.err.println(usage);
         System.exit(-1);
       } else {
@@ -71,12 +71,19 @@ public class ParserChecker {
       }
     }
 
-    if (LOG.isInfoEnabled()) { LOG.info("fetching: "+url); }
+    if (LOG.isInfoEnabled()) {
+      LOG.info("fetching: " + url);
+    }
 
-    Configuration conf = NutchConfiguration.create();
     ProtocolFactory factory = new ProtocolFactory(conf);
     Protocol protocol = factory.getProtocol(url);
-    Content content = protocol.getProtocolOutput(new Text(url), new CrawlDatum()).getContent();
+    Content content = protocol.getProtocolOutput(new Text(url),
+        new CrawlDatum()).getContent();
+
+    if (content == null) {
+      System.err.println("Can't fetch URL successfully");
+      return (-1);
+    }
 
     if (force) {
       content.setContentType(contentType);
@@ -86,12 +93,12 @@ public class ParserChecker {
 
     if (contentType == null) {
       System.err.println("");
-      System.exit(-1);
+      return (-1);
     }
 
     if (LOG.isInfoEnabled()) {
-      LOG.info("parsing: "+url);
-      LOG.info("contentType: "+contentType);
+      LOG.info("parsing: " + url);
+      LOG.info("contentType: " + contentType);
     }
 
     ParseResult parseResult = new ParseUtil(conf).parse(content);
@@ -108,6 +115,23 @@ public class ParserChecker {
       }
     }
 
-    System.exit(0);
+    return 0;
+  }
+
+  @Override
+  public Configuration getConf() {
+    return conf;
+  }
+
+  @Override
+  public void setConf(Configuration c) {
+    conf = c;
   }
+
+  public static void main(String[] args) throws Exception {
+    int res = ToolRunner.run(NutchConfiguration.create(), new ParserChecker(),
+        args);
+    System.exit(res);
+  }
+
 }