You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by ta...@apache.org on 2025/02/24 15:51:12 UTC
(incubator-stormcrawler) branch issue-1475 updated: #1475 -- move logic to SitemapFilter
This is an automated email from the ASF dual-hosted git repository.
tallison pushed a commit to branch issue-1475
in repository https://gitbox.apache.org/repos/asf/incubator-stormcrawler.git
The following commit(s) were added to refs/heads/issue-1475 by this push:
new 8895f978 #1475 -- move logic to SitemapFilter
8895f978 is described below
commit 8895f9780cb586fa5ea4e7d727d862cfd7fc93f2
Author: tallison <ta...@apache.org>
AuthorDate: Mon Feb 24 10:50:59 2025 -0500
#1475 -- move logic to SitemapFilter
---
.../org/apache/stormcrawler/bolt/FetcherBolt.java | 3 --
.../stormcrawler/bolt/SimpleFetcherBolt.java | 2 -
.../filtering/sitemap/SitemapFilter.java | 46 +++++++++++++++++-----
3 files changed, 36 insertions(+), 15 deletions(-)
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
index 6833ce2d..3f1477d1 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/FetcherBolt.java
@@ -551,9 +551,6 @@ public class FetcherBolt extends StatusEmitterBolt {
smautodisco = false;
} else {
smautodisco = sitemapsAutoDiscovery;
- // SitemapFilter needs this
- metadata.setValue(
- SITEMAP_DISCOVERY_PARAM_KEY, Boolean.toString(smautodisco));
}
if (!fromCache && smautodisco) {
diff --git a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
index a10943db..0f783d78 100644
--- a/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/bolt/SimpleFetcherBolt.java
@@ -309,8 +309,6 @@ public class SimpleFetcherBolt extends StatusEmitterBolt {
smautodisco = false;
} else {
smautodisco = sitemapsAutoDiscovery;
- // SitemapFilter needs this
- metadata.setValue(SITEMAP_DISCOVERY_PARAM_KEY, Boolean.toString(smautodisco));
}
if (!fromCache && smautodisco) {
diff --git a/core/src/main/java/org/apache/stormcrawler/filtering/sitemap/SitemapFilter.java b/core/src/main/java/org/apache/stormcrawler/filtering/sitemap/SitemapFilter.java
index c4a04553..498b7378 100644
--- a/core/src/main/java/org/apache/stormcrawler/filtering/sitemap/SitemapFilter.java
+++ b/core/src/main/java/org/apache/stormcrawler/filtering/sitemap/SitemapFilter.java
@@ -16,10 +16,13 @@
*/
package org.apache.stormcrawler.filtering.sitemap;
+import com.fasterxml.jackson.databind.JsonNode;
import java.net.URL;
+import java.util.Map;
import org.apache.stormcrawler.Metadata;
import org.apache.stormcrawler.bolt.SiteMapParserBolt;
import org.apache.stormcrawler.filtering.URLFilter;
+import org.apache.stormcrawler.util.ConfUtils;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -35,7 +38,7 @@ import org.jetbrains.annotations.Nullable;
* }
* </pre>
*
- * Will be replaced by <a href=
+ * <p>Will be replaced by <a href=
* "https://github.com/apache/incubator-stormcrawler/issues/711">MetadataFilter to filter based on
* multiple key values</a>
*
@@ -45,21 +48,44 @@ public class SitemapFilter extends URLFilter {
private static final String SITEMAP_DISCOVERY_PARAM_KEY = "sitemap.discovery";
+ private boolean sitemapsAutoDiscovery = false;
+
+ @Override
+ public void configure(
+ @NotNull Map<String, Object> stormConf,
+ @NotNull JsonNode filtersConf,
+ @NotNull String name) {
+ super.configure(stormConf, filtersConf);
+ sitemapsAutoDiscovery = ConfUtils.getBoolean(stormConf, SITEMAP_DISCOVERY_PARAM_KEY, false);
+ }
+
@Override
public @Nullable String filter(
@Nullable URL sourceUrl,
@Nullable Metadata sourceMetadata,
@NotNull String urlToFilter) {
+ if (sourceMetadata == null) {
+ return urlToFilter;
+ }
+ boolean smautodisco = false;
+ // check in the metadata if discovery setting has been
+ // overridden
+ String localSitemapDiscoveryVal = sourceMetadata.getFirstValue(SITEMAP_DISCOVERY_PARAM_KEY);
- if (sourceMetadata != null) {
- if (!Boolean.parseBoolean(sourceMetadata.getFirstValue(SITEMAP_DISCOVERY_PARAM_KEY))) {
- return urlToFilter;
- } else if (!Boolean.parseBoolean(
- sourceMetadata.getFirstValue(SiteMapParserBolt.isSitemapKey))
- && Boolean.parseBoolean(
- sourceMetadata.getFirstValue(SiteMapParserBolt.foundSitemapKey))) {
- return null;
- }
+ if ("true".equalsIgnoreCase(localSitemapDiscoveryVal)) {
+ smautodisco = true;
+ } else if ("false".equalsIgnoreCase(localSitemapDiscoveryVal)) {
+ smautodisco = false;
+ } else {
+ smautodisco = sitemapsAutoDiscovery;
+ }
+ if (!smautodisco) {
+ return urlToFilter;
+ } else if (!Boolean.parseBoolean(
+ sourceMetadata.getFirstValue(SiteMapParserBolt.isSitemapKey))
+ && Boolean.parseBoolean(
+ sourceMetadata.getFirstValue(SiteMapParserBolt.foundSitemapKey))) {
+ return null;
}
return urlToFilter;
}