You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by jn...@apache.org on 2014/06/17 10:41:57 UTC
svn commit: r1603094 - in /nutch/trunk: CHANGES.txt
src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
Author: jnioche
Date: Tue Jun 17 08:41:57 2014
New Revision: 1603094
URL: http://svn.apache.org/r1603094
Log:
NUTCH-1793 HttpRobotRulesParser not configured properly (jnioche)
Modified:
nutch/trunk/CHANGES.txt
nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
Modified: nutch/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/nutch/trunk/CHANGES.txt?rev=1603094&r1=1603093&r2=1603094&view=diff
==============================================================================
--- nutch/trunk/CHANGES.txt (original)
+++ nutch/trunk/CHANGES.txt Tue Jun 17 08:41:57 2014
@@ -2,6 +2,8 @@ Nutch Change Log
Nutch Current Development
+* NUTCH-1793 HttpRobotRulesParser not configured properly (jnioche)
+
* NUTCH-1647 protocol-http throws 'unzipBestEffort returned null' for redirected pages (jnioche)
* NUTCH-1736 Can't fetch page if http response header contains Transfer-Encodingï¼chunked (ysc via jnioche)
Modified: nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java
URL: http://svn.apache.org/viewvc/nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java?rev=1603094&r1=1603093&r2=1603094&view=diff
==============================================================================
--- nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java (original)
+++ nutch/trunk/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpRobotRulesParser.java Tue Jun 17 08:41:57 2014
@@ -44,8 +44,12 @@ public class HttpRobotRulesParser extend
HttpRobotRulesParser() { }
public HttpRobotRulesParser(Configuration conf) {
- super(conf);
- allowForbidden = conf.getBoolean("http.robots.403.allow", false);
+ setConf(conf);
+ }
+
+ public void setConf(Configuration conf) {
+ super.setConf(conf);
+ allowForbidden = conf.getBoolean("http.robots.403.allow", true);
}
/** Compose unique key to store and access robot rules in cache for given URL */