You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by ta...@apache.org on 2025/02/21 15:54:16 UTC
(incubator-stormcrawler) 01/01: #1475 -- if sitemap discovery is turned off, links should be extracted
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch issue-1475
in repository https://gitbox.apache.org/repos/asf/incubator-stormcrawler.git
commit 8f6123464ead4f2f8b44688237643f70c249ee71
Author: tallison <ta...@apache.org>
AuthorDate: Fri Feb 21 10:54:00 2025 -0500
#1475 -- if sitemap discovery is turned off, links should be extracted
---
.../java/org/apache/stormcrawler/bolt/FetcherBolt.java | 3 +++
.../org/apache/stormcrawler/bolt/SimpleFetcherBolt.java | 3 +++
.../stormcrawler/filtering/sitemap/SitemapFilter.java | 16 ++++++++++------
3 files changed, 16 insertions(+), 6 deletions(-)
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
index 3f1477d1..80d7389b 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
@@ -551,6 +551,9 @@ public class FetcherBolt extends StatusEmitterBolt {
smautodisco = false;
} else {
smautodisco = sitemapsAutoDiscovery;
+ //SitemapFilter needs this
+ metadata.setValue(SITEMAP_DISCOVERY_PARAM_KEY,
+ Boolean.toString(smautodisco).toLowerCase(Locale.ROOT));
}
if (!fromCache && smautodisco) {
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
index 0f783d78..c222fbd0 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
@@ -309,6 +309,9 @@ public class SimpleFetcherBolt extends StatusEmitterBolt {
smautodisco = false;
} else {
smautodisco = sitemapsAutoDiscovery;
+ //SitemapFilter needs this
+ metadata.setValue(SITEMAP_DISCOVERY_PARAM_KEY, Boolean.toString(smautodisco));
+
}
if (!fromCache && smautodisco) {
diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/sitemap/SitemapFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/sitemap/SitemapFilter.java
index 5beec278..c30d329f 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/sitemap/SitemapFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/sitemap/SitemapFilter.java
@@ -43,18 +43,22 @@ import org.jetbrains.annotations.Nullable;
*/
public class SitemapFilter extends URLFilter {
+
+ private static final String SITEMAP_DISCOVERY_PARAM_KEY = "sitemap.discovery";
+
@Override
public @Nullable String filter(
@Nullable URL sourceUrl,
@Nullable Metadata sourceMetadata,
@NotNull String urlToFilter) {
- if (sourceMetadata != null
- && !Boolean.parseBoolean(
- sourceMetadata.getFirstValue(SiteMapParserBolt.isSitemapKey))
- && Boolean.parseBoolean(
- sourceMetadata.getFirstValue(SiteMapParserBolt.foundSitemapKey))) {
- return null;
+ if (sourceMetadata != null) {
+ if (!Boolean.parseBoolean(sourceMetadata.getFirstValue(SITEMAP_DISCOVERY_PARAM_KEY))) {
+ return urlToFilter;
+ } else if (!Boolean.parseBoolean(sourceMetadata.getFirstValue(SiteMapParserBolt.isSitemapKey)) &&
+ Boolean.parseBoolean(sourceMetadata.getFirstValue(SiteMapParserBolt.foundSitemapKey))) {
+ return null;
+ }
}
return urlToFilter;
}