You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by jn...@apache.org on 2024/10/18 10:15:09 UTC
(incubator-stormcrawler) 01/01: Bugfix nofollow instructions in rel tags ignored when the tag has more than one value
This is an automated email from the ASF dual-hosted git repository.
jnioche pushed a commit to branch relTagMultiValued
in repository https://gitbox.apache.org/repos/asf/incubator-stormcrawler.git
commit 67151351c2185f497354a4ae815fee02e1180c82
Author: Julien Nioche <ju...@digitalpebble.com>
AuthorDate: Fri Oct 18 11:14:42 2024 +0100
Bugfix nofollow instructions in rel tags ignored when the tag has more than one value
Signed-off-by: Julien Nioche <ju...@digitalpebble.com>
---
.../src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java | 6 +++++-
core/src/test/resources/digitalpebble.com.html | 1 +
2 files changed, 6 insertions(+), 1 deletion(-)
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
index 5c828bc1..015403d0 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/JSoupParserBolt.java
@@ -30,6 +30,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.stream.Stream;
import org.apache.commons.lang.StringUtils;
import org.apache.storm.metric.api.MultiCountMetric;
import org.apache.storm.task.OutputCollector;
@@ -269,7 +270,10 @@ public class JSoupParserBolt extends StatusEmitterBolt {
final URL baseURL = new URL(url);
for (Element link : links) {
// nofollow
- boolean noFollow = "nofollow".equalsIgnoreCase(link.attr("rel"));
+ String[] relkeywords = link.attr("rel").split(" ");
+ boolean noFollow =
+ Stream.of(relkeywords).anyMatch(x -> x.equalsIgnoreCase("nofollow"));
+
// remove altogether
if (noFollow && robots_noFollow_strict) {
continue;
diff --git a/core/src/test/resources/digitalpebble.com.html b/core/src/test/resources/digitalpebble.com.html
index 8b1de688..68332164 100644
--- a/core/src/test/resources/digitalpebble.com.html
+++ b/core/src/test/resources/digitalpebble.com.html
@@ -50,6 +50,7 @@ under the License.
</script>
<a rel="nofollow" href="inexistent.html"/>
+<a rel="nofollow somevalue" href="another_inexistent.html"/>
<body style="color: rgb(0, 0, 0); background-color: rgb(255, 255, 255);"
alink="#000000" link="#000000" vlink="#000000">