You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by sn...@apache.org on 2016/08/16 18:45:18 UTC
[1/2] nutch git commit: Remove obsolete properties
protocol.plugin.check.blocking and protocol.plugin.check.robots
Repository: nutch
Updated Branches:
refs/heads/master d27c351f4 -> d37b7ce13
Remove obsolete properties protocol.plugin.check.blocking and protocol.plugin.check.robots
Project: http://git-wip-us.apache.org/repos/asf/nutch/repo
Commit: http://git-wip-us.apache.org/repos/asf/nutch/commit/070a637b
Tree: http://git-wip-us.apache.org/repos/asf/nutch/tree/070a637b
Diff: http://git-wip-us.apache.org/repos/asf/nutch/diff/070a637b
Branch: refs/heads/master
Commit: 070a637babedc324948c0c58b333668bab6b813d
Parents: d27c351
Author: Sebastian Nagel <sn...@apache.org>
Authored: Mon Aug 15 11:19:46 2016 +0200
Committer: Sebastian Nagel <sn...@apache.org>
Committed: Mon Aug 15 11:19:46 2016 +0200
----------------------------------------------------------------------
src/java/org/apache/nutch/fetcher/Fetcher.java | 4 ----
src/java/org/apache/nutch/protocol/Protocol.java | 18 ------------------
2 files changed, 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nutch/blob/070a637b/src/java/org/apache/nutch/fetcher/Fetcher.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/nutch/fetcher/Fetcher.java b/src/java/org/apache/nutch/fetcher/Fetcher.java
index aad9ee9..e60b10f 100644
--- a/src/java/org/apache/nutch/fetcher/Fetcher.java
+++ b/src/java/org/apache/nutch/fetcher/Fetcher.java
@@ -209,10 +209,6 @@ MapRunnable<Text, CrawlDatum, Text, NutchWritable> {
feeder.setTimeLimit(timelimit);
feeder.start();
- // set non-blocking & no-robots mode for HTTP protocol plugins.
- getConf().setBoolean(Protocol.CHECK_BLOCKING, false);
- getConf().setBoolean(Protocol.CHECK_ROBOTS, false);
-
for (int i = 0; i < threadCount; i++) { // spawn threads
FetcherThread t = new FetcherThread(getConf(), getActiveThreads(), fetchQueues,
feeder, spinWaiting, lastRequestStart, reporter, errors, segmentName,
http://git-wip-us.apache.org/repos/asf/nutch/blob/070a637b/src/java/org/apache/nutch/protocol/Protocol.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/nutch/protocol/Protocol.java b/src/java/org/apache/nutch/protocol/Protocol.java
index 0aa5d29..efd0100 100755
--- a/src/java/org/apache/nutch/protocol/Protocol.java
+++ b/src/java/org/apache/nutch/protocol/Protocol.java
@@ -33,24 +33,6 @@ public interface Protocol extends Pluggable, Configurable {
public final static String X_POINT_ID = Protocol.class.getName();
/**
- * Property name. If in the current configuration this property is set to
- * true, protocol implementations should handle "politeness" limits
- * internally. If this is set to false, it is assumed that these limits are
- * enforced elsewhere, and protocol implementations should not enforce them
- * internally.
- */
- public final static String CHECK_BLOCKING = "protocol.plugin.check.blocking";
-
- /**
- * Property name. If in the current configuration this property is set to
- * true, protocol implementations should handle robot exclusion rules
- * internally. If this is set to false, it is assumed that these limits are
- * enforced elsewhere, and protocol implementations should not enforce them
- * internally.
- */
- public final static String CHECK_ROBOTS = "protocol.plugin.check.robots";
-
- /**
* Returns the {@link Content} for a fetchlist entry.
*/
ProtocolOutput getProtocolOutput(Text url, CrawlDatum datum);
[2/2] nutch git commit: Merge branch 'NUTCH-2299' of
https://github.com/sebastian-nagel/nutch this closes #140 - Remove obsolete
properties protocol.plugin.check.*
Posted by sn...@apache.org.
Merge branch 'NUTCH-2299' of https://github.com/sebastian-nagel/nutch this closes #140
- Remove obsolete properties protocol.plugin.check.*
Project: http://git-wip-us.apache.org/repos/asf/nutch/repo
Commit: http://git-wip-us.apache.org/repos/asf/nutch/commit/d37b7ce1
Tree: http://git-wip-us.apache.org/repos/asf/nutch/tree/d37b7ce1
Diff: http://git-wip-us.apache.org/repos/asf/nutch/diff/d37b7ce1
Branch: refs/heads/master
Commit: d37b7ce13ee82f0a7d1388f87c2be5d636e425aa
Parents: d27c351 070a637
Author: Sebastian Nagel <sn...@apache.org>
Authored: Tue Aug 16 20:43:01 2016 +0200
Committer: Sebastian Nagel <sn...@apache.org>
Committed: Tue Aug 16 20:43:01 2016 +0200
----------------------------------------------------------------------
src/java/org/apache/nutch/fetcher/Fetcher.java | 4 ----
src/java/org/apache/nutch/protocol/Protocol.java | 18 ------------------
2 files changed, 22 deletions(-)
----------------------------------------------------------------------