You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by sn...@apache.org on 2013/03/27 22:33:38 UTC
svn commit: r1461857 - in /nutch/branches/2.x: CHANGES.txt
src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
src/java/org/apache/nutch/parse/ParserChecker.java
Author: snagel
Date: Wed Mar 27 21:33:38 2013
New Revision: 1461857
URL: http://svn.apache.org/r1461857
Log:
parsechecker and indexchecker to report truncated content
Modified:
nutch/branches/2.x/CHANGES.txt
nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserChecker.java
Modified: nutch/branches/2.x/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/CHANGES.txt?rev=1461857&r1=1461856&r2=1461857&view=diff
==============================================================================
--- nutch/branches/2.x/CHANGES.txt (original)
+++ nutch/branches/2.x/CHANGES.txt Wed Mar 27 21:33:38 2013
@@ -2,6 +2,8 @@ Nutch Change Log
Release 2.2 - Current Development
+* NUTCH-1389 parsechecker and indexchecker to report truncated content (snagel)
+
* NUTCH-1419 parsechecker and indexchecker to report protocol status (snagel via lewismc)
* NUTCH-1038 Port IndexingFiltersChecker to 2.0 (snagel via lewismc)
Modified: nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java?rev=1461857&r1=1461856&r2=1461857&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/indexer/IndexingFiltersChecker.java Wed Mar 27 21:33:38 2013
@@ -28,6 +28,7 @@ import org.apache.hadoop.util.ToolRunner
import org.apache.nutch.crawl.CrawlStatus;
import org.apache.nutch.parse.ParseStatusUtils;
import org.apache.nutch.parse.ParseUtil;
+import org.apache.nutch.parse.ParserJob;
import org.apache.nutch.protocol.Content;
import org.apache.nutch.protocol.Protocol;
import org.apache.nutch.protocol.ProtocolFactory;
@@ -109,6 +110,10 @@ public class IndexingFiltersChecker exte
LOG.info("contentType: " + contentType);
}
+ if (ParserJob.isTruncated(url, page)) {
+ LOG.warn("Content is truncated, parse may fail!");
+ }
+
(new ParseUtil(conf)).process(url, page);
if (!ParseStatusUtils.isSuccess(page.getParseStatus())) {
LOG.warn("Problem with parse - check log");
Modified: nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserChecker.java
URL: http://svn.apache.org/viewvc/nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserChecker.java?rev=1461857&r1=1461856&r2=1461857&view=diff
==============================================================================
--- nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserChecker.java (original)
+++ nutch/branches/2.x/src/java/org/apache/nutch/parse/ParserChecker.java Wed Mar 27 21:33:38 2013
@@ -121,6 +121,10 @@ public class ParserChecker implements To
page.setContentType(new Utf8(contentType));
+ if (ParserJob.isTruncated(url, page)) {
+ LOG.warn("Content is truncated, parse may fail!");
+ }
+
Parse parse = new ParseUtil(conf).parse(url, page);
if (parse == null) {