You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nutch.apache.org by sn...@apache.org on 2025/12/18 08:45:12 UTC
(nutch) 01/02: NUTCH-3136 Upgrade crawler-commons dependency
This is an automated email from the ASF dual-hosted git repository.
snagel pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nutch.git
commit 50b1ee63918d264c2064120282185a14d88209f3
Author: Sebastian Nagel <sn...@apache.org>
AuthorDate: Fri Dec 12 09:32:14 2025 +0100
NUTCH-3136 Upgrade crawler-commons dependency
Robots.txt parser: use URL objects in newly introduced
methods to avoid the unnecessary parsing of URLs.
---
ivy/ivy.xml | 2 +-
.../org/apache/nutch/fetcher/FetcherThread.java | 4 ++--
src/java/org/apache/nutch/protocol/Protocol.java | 21 +++++++++++++++++++++
src/java/org/apache/nutch/util/URLUtil.java | 2 +-
.../apache/nutch/protocol/http/api/HttpBase.java | 6 ++++++
.../java/org/apache/nutch/protocol/file/File.java | 10 ++++++++++
.../src/java/org/apache/nutch/protocol/ftp/Ftp.java | 9 +++++++++
7 files changed, 50 insertions(+), 4 deletions(-)
diff --git a/ivy/ivy.xml b/ivy/ivy.xml
index f149ce13d..d2cbfc850 100644
--- a/ivy/ivy.xml
+++ b/ivy/ivy.xml
@@ -83,7 +83,7 @@
<dependency org="com.google.guava" name="guava" rev="33.4.8-jre" />
- <dependency org="com.github.crawler-commons" name="crawler-commons" rev="1.4" />
+ <dependency org="com.github.crawler-commons" name="crawler-commons" rev="1.6" />
<dependency org="com.google.code.gson" name="gson" rev="2.13.1"/>
<dependency org="com.martinkl.warc" name="warc-hadoop" rev="0.1.0">
diff --git a/src/java/org/apache/nutch/fetcher/FetcherThread.java b/src/java/org/apache/nutch/fetcher/FetcherThread.java
index 02b7cd3e8..2abcfe9f6 100644
--- a/src/java/org/apache/nutch/fetcher/FetcherThread.java
+++ b/src/java/org/apache/nutch/fetcher/FetcherThread.java
@@ -366,7 +366,7 @@ public class FetcherThread extends Thread {
LOG.debug("redirectCount={}", redirectCount);
redirecting = false;
Protocol protocol = this.protocolFactory.getProtocol(fit.u);
- BaseRobotRules rules = protocol.getRobotRules(fit.url, fit.datum,
+ BaseRobotRules rules = protocol.getRobotRules(fit.u, fit.datum,
robotsTxtContent);
if (robotsTxtContent != null) {
outputRobotsTxt(robotsTxtContent);
@@ -389,7 +389,7 @@ public class FetcherThread extends Thread {
}
continue;
}
- if (!rules.isAllowed(fit.url.toString())) {
+ if (!rules.isAllowed(fit.u)) {
// unblock
fetchQueues.finishFetchItem(fit, true);
LOG.info("Denied by robots.txt: {}", fit.url);
diff --git a/src/java/org/apache/nutch/protocol/Protocol.java b/src/java/org/apache/nutch/protocol/Protocol.java
index ab4162c87..2514eae33 100644
--- a/src/java/org/apache/nutch/protocol/Protocol.java
+++ b/src/java/org/apache/nutch/protocol/Protocol.java
@@ -16,6 +16,7 @@
*/
package org.apache.nutch.protocol;
+import java.net.URL;
import java.util.List;
import org.apache.hadoop.conf.Configurable;
@@ -57,4 +58,24 @@ public interface Protocol extends Pluggable, Configurable {
BaseRobotRules getRobotRules(Text url, CrawlDatum datum,
List<Content> robotsTxtContent);
+ /**
+ * Retrieve robot rules applicable for this URL.
+ *
+ * @param url
+ * URL to check
+ * @param datum
+ * page datum
+ * @param robotsTxtContent
+ * container to store responses when fetching the robots.txt file for
+ * debugging or archival purposes. Instead of a robots.txt file, it
+ * may include redirects or an error page (404, etc.). Response
+ * {@link Content} is appended to the passed list. If null is passed
+ * nothing is stored.
+ * @return robot rules (specific for this URL or default), never null
+ */
+ default BaseRobotRules getRobotRules(URL url, CrawlDatum datum,
+ List<Content> robotsTxtContent) {
+ return getRobotRules(new Text(url.toString()), datum, robotsTxtContent);
+ }
+
}
diff --git a/src/java/org/apache/nutch/util/URLUtil.java b/src/java/org/apache/nutch/util/URLUtil.java
index 0cfce1c65..afd6f1385 100644
--- a/src/java/org/apache/nutch/util/URLUtil.java
+++ b/src/java/org/apache/nutch/util/URLUtil.java
@@ -103,7 +103,7 @@ public class URLUtil {
* <a href= "https://publicsuffix.org/list/public_suffix_list.dat"
* >https://publicsuffix.org/list/public_suffix_list.dat</a> and are compared
* using <a href=
- * "https://crawler-commons.github.io/crawler-commons/1.4/crawlercommons/domains/EffectiveTldFinder.html">
+ * "https://crawler-commons.github.io/crawler-commons/1.6/crawlercommons/domains/EffectiveTldFinder.html">
* crawler-commons' EffectiveTldFinder</a>. Only ICANN domain suffixes are
* used. Because EffectiveTldFinder loads the public suffix list as file
* "effective_tld_names.dat" from the Java classpath, it's possible to use the
diff --git a/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java b/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
index 79b45882e..caa3f861e 100755
--- a/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
+++ b/src/plugin/lib-http/src/java/org/apache/nutch/protocol/http/api/HttpBase.java
@@ -721,6 +721,12 @@ public abstract class HttpBase implements Protocol {
return this.robots.getRobotRulesSet(this, url, robotsTxtContent);
}
+ @Override
+ public BaseRobotRules getRobotRules(URL url, CrawlDatum datum,
+ List<Content> robotsTxtContent) {
+ return this.robots.getRobotRulesSet(this, url, robotsTxtContent);
+ }
+
/**
* Transforming a String[] into a HashMap for faster searching
*
diff --git a/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java b/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
index e4d201069..877873b64 100644
--- a/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
+++ b/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/File.java
@@ -232,4 +232,14 @@ public class File implements Protocol {
return RobotRulesParser.EMPTY_RULES;
}
+ /**
+ * No robots parsing is done for file protocol. So this returns a set of empty
+ * rules which will allow every url.
+ */
+ @Override
+ public BaseRobotRules getRobotRules(URL url, CrawlDatum datum,
+ List<Content> robotsTxtContent) {
+ return RobotRulesParser.EMPTY_RULES;
+ }
+
}
diff --git a/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java b/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
index 2a47b63d6..8cf58f75e 100644
--- a/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
+++ b/src/plugin/protocol-ftp/src/java/org/apache/nutch/protocol/ftp/Ftp.java
@@ -304,6 +304,15 @@ public class Ftp implements Protocol {
return robots.getRobotRulesSet(this, url, robotsTxtContent);
}
+ /**
+ * Get the robots rules for a given url
+ */
+ @Override
+ public BaseRobotRules getRobotRules(URL url, CrawlDatum datum,
+ List<Content> robotsTxtContent) {
+ return robots.getRobotRulesSet(this, url, robotsTxtContent);
+ }
+
public int getBufferSize() {
return BUFFER_SIZE;
}