You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by fe...@apache.org on 2012/02/24 15:49:26 UTC
svn commit: r1293278 - in /nutch/trunk: CHANGES.txt conf/nutch-default.xml
src/java/org/apache/nutch/fetcher/Fetcher.java
src/java/org/apache/nutch/parse/ParseSegment.java
Author: ferdy
Date: Fri Feb 24 14:49:26 2012
New Revision: 1293278
URL: http://svn.apache.org/viewvc?rev=1293278&view=rev
Log:
RECOMMIT NUTCH-965 Skip parsing for truncated document
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/conf/nutch-default.xml
nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java
nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1293278&r1=1293277&r2=1293278&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Fri Feb 24 14:49:26 2012
@@ -1,5 +1,7 @@
Nutch Change Log
+* NUTCH-965 Skip parsing for truncated documents (alexis, lewismc, ferdy)
+
* NUTCH-1210 DomainBlacklistFilter (markus)
* NUTCH-1193 Incorrect url transform to lowercase: parameter solr (Eduardo dos Santos Leggiero via lewismc)
Modified: nutch/trunk/conf/nutch-default.xml
URL: http://svn.apache.org/viewvc/nutch/trunk/conf/nutch-default.xml?rev=1293278&r1=1293277&r2=1293278&view=diff
==============================================================================
--- nutch/trunk/conf/nutch-default.xml (original)
+++ nutch/trunk/conf/nutch-default.xml Fri Feb 24 14:49:26 2012
@@ -1062,6 +1062,14 @@
<description>Whether the parser will normalize URLs (with the configured URL normalizers).</description>
</property>
+<property>
+ <name>parser.skip.truncated</name>
+ <value>true</value>
+ <description>Boolean value for whether we should skip parsing for truncated documents. By default this
+ property is activated due to extremely high levels of CPU which parsing can sometimes take.
+ </description>
+</property>
+
<!-- urlfilter plugin properties -->
<property>
Modified: nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java?rev=1293278&r1=1293277&r2=1293278&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/fetcher/Fetcher.java Fri Feb 24 14:49:26 2012
@@ -589,6 +589,7 @@ public class Fetcher extends Configured
private int maxOutlinkDepth;
private int maxOutlinkDepthNumLinks;
private int outlinksDepthDivisor;
+ private boolean skipTruncated;
public FetcherThread(Configuration conf) {
this.setDaemon(true); // don't hang JVM on exit
@@ -597,6 +598,7 @@ public class Fetcher extends Configured
this.urlFilters = new URLFilters(conf);
this.scfilters = new ScoringFilters(conf);
this.parseUtil = new ParseUtil(conf);
+ this.skipTruncated = conf.getBoolean(ParseSegment.SKIP_TRUNCATED, true);
this.protocolFactory = new ProtocolFactory(conf);
this.normalizers = new URLNormalizers(conf, URLNormalizers.SCOPE_FETCHER);
this.maxCrawlDelay = conf.getInt("fetcher.max.crawl.delay", 30) * 1000;
@@ -945,12 +947,14 @@ public class Fetcher extends Configured
/* Note: Fetcher will only follow meta-redirects coming from the
* original URL. */
if (parsing && status == CrawlDatum.STATUS_FETCH_SUCCESS) {
- try {
- parseResult = this.parseUtil.parse(content);
- } catch (Exception e) {
- LOG.warn("Error parsing: " + key + ": " + StringUtils.stringifyException(e));
+ if (!skipTruncated || (skipTruncated && !ParseSegment.isTruncated(content))) {
+ try {
+ parseResult = this.parseUtil.parse(content);
+ } catch (Exception e) {
+ LOG.warn("Error parsing: " + key + ": " + StringUtils.stringifyException(e));
+ }
}
-
+
if (parseResult == null) {
byte[] signature =
SignatureFactory.getSignature(getConf()).calculate(content,
Modified: nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java?rev=1293278&r1=1293277&r2=1293278&view=diff
==============================================================================
--- nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java (original)
+++ nutch/trunk/src/java/org/apache/nutch/parse/ParseSegment.java Fri Feb 24 14:49:26 2012
@@ -26,7 +26,9 @@ import org.apache.hadoop.io.*;
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.util.*;
import org.apache.hadoop.conf.*;
+import org.apache.nutch.metadata.Metadata;
import org.apache.nutch.metadata.Nutch;
+import org.apache.nutch.net.protocols.Response;
import org.apache.nutch.protocol.*;
import org.apache.nutch.scoring.ScoringFilterException;
import org.apache.nutch.scoring.ScoringFilters;
@@ -45,8 +47,12 @@ public class ParseSegment extends Config
public static final Logger LOG = LoggerFactory.getLogger(ParseSegment.class);
+ public static final String SKIP_TRUNCATED = "parser.skip.truncated";
+
private ScoringFilters scfilters;
+ private boolean skipTruncated;
+
public ParseSegment() {
this(null);
}
@@ -58,6 +64,7 @@ public class ParseSegment extends Config
public void configure(JobConf job) {
setConf(job);
this.scfilters = new ScoringFilters(job);
+ skipTruncated=job.getBoolean(SKIP_TRUNCATED, true);
}
public void close() {}
@@ -80,6 +87,10 @@ public class ParseSegment extends Config
LOG.debug("Skipping " + key + " as content is not fetched successfully");
return;
}
+
+ if (skipTruncated && isTruncated(content)) {
+ return;
+ }
ParseResult parseResult = null;
try {
@@ -128,6 +139,43 @@ public class ParseSegment extends Config
parse.getData(), parse.isCanonical()));
}
}
+
+ /**
+ * Checks if the page's content is truncated.
+ * @param content
+ * @return If the page is truncated <code>true</code>. When it is not,
+ * or when it could be determined, <code>false</code>.
+ */
+ public static boolean isTruncated(Content content) {
+ byte[] contentBytes = content.getContent();
+ if (contentBytes == null) return false;
+ Metadata metadata = content.getMetadata();
+ if (metadata == null) return false;
+
+ String lengthStr = metadata.get(Response.CONTENT_LENGTH);
+ if (lengthStr != null) lengthStr=lengthStr.trim();
+ if (StringUtil.isEmpty(lengthStr)) {
+ return false;
+ }
+ int inHeaderSize;
+ String url = content.getUrl();
+ try {
+ inHeaderSize = Integer.parseInt(lengthStr);
+ } catch (NumberFormatException e) {
+ LOG.warn("Wrong contentlength format for " + url, e);
+ return false;
+ }
+ int actualSize = contentBytes.length;
+ if (inHeaderSize > actualSize) {
+ LOG.info(url + " skipped. Content of size " + inHeaderSize
+ + " was truncated to " + actualSize);
+ return true;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(url + " actualSize=" + actualSize + " inHeaderSize=" + inHeaderSize);
+ }
+ return false;
+ }
public void reduce(Text key, Iterator<Writable> values,
OutputCollector<Text, Writable> output, Reporter reporter)