You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@stormcrawler.apache.org by si...@apache.org on 2026/07/22 21:07:56 UTC

(stormcrawler) 01/01: Allow indexer bolts to use a metadata value as the document ID

This is an automated email from the ASF dual-hosted git repository.

sigee pushed a commit to branch feat/671-indexer-docid-from-metadata
in repository https://gitbox.apache.org/repos/asf/stormcrawler.git

commit 7ed1bb32684b4b3cb5b244541905015ba90f2227
Author: Dávid Szigecsán <si...@apache.org>
AuthorDate: Wed Jul 22 22:48:14 2026 +0200

    Allow indexer bolts to use a metadata value as the document ID
---
 .../stormcrawler/indexing/AbstractIndexerBolt.java | 35 +++++++++++++++++++-
 .../stormcrawler/indexer/BasicIndexingTest.java    | 37 ++++++++++++++++++++++
 .../apache/stormcrawler/indexer/DummyIndexer.java  |  4 +++
 docs/src/main/asciidoc/configuration.adoc          |  1 +
 .../aws/bolt/CloudSearchIndexerBolt.java           |  4 +++
 .../stormcrawler/opensearch/bolt/DeletionBolt.java | 21 ++++++++++--
 .../stormcrawler/opensearch/bolt/DeletionBolt.java | 21 ++++++++++--
 7 files changed, 118 insertions(+), 5 deletions(-)

diff --git a/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java b/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java
index f6f7dcea..38e5d231 100644
--- a/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java
+++ b/core/src/main/java/org/apache/stormcrawler/indexing/AbstractIndexerBolt.java
@@ -75,8 +75,19 @@ public abstract class AbstractIndexerBolt extends BaseRichBolt {
     /** Indicates that empty field values should not be emitted at all. */
     public static final String ignoreEmptyFieldValueParamName = "indexer.ignore.empty.fields";
 
+    /**
+     * Metadata key whose value should be used as the document ID, if present. This can be used e.g.
+     * by a {@link org.apache.stormcrawler.parse.ParseFilter} generating a hash of the content so
+     * that duplicate content ends up under the same document ID regardless of the URL(s) it was
+     * found at. Falls back to the default (a SHA-256 digest of the URL) if not set or if the
+     * metadata does not contain a value for the configured key.
+     */
+    public static final String DOC_ID_METADATA_PARAM_NAME = "indexer.md.docid";
+
     private String[] filterKeyValue = null;
 
+    private String docIdMetadataKey = null;
+
     private final List<Key> metadata2field = new ArrayList<>();
 
     private String fieldNameForText = null;
@@ -157,6 +168,8 @@ public abstract class AbstractIndexerBolt extends BaseRichBolt {
 
         canonicalMetadataName = ConfUtils.getString(conf, canonicalMetadataParamName);
 
+        docIdMetadataKey = ConfUtils.getString(conf, DOC_ID_METADATA_PARAM_NAME);
+
         final Pattern indexValuePattern = Pattern.compile("\\[(\\d+)\\]");
 
         for (String mapping : ConfUtils.loadListFromConf(metadata2fieldParamName, conf)) {
@@ -259,12 +272,32 @@ public abstract class AbstractIndexerBolt extends BaseRichBolt {
      *
      * @param metadata The {@link Metadata}.
      * @param normalisedUrl The normalised url.
-     * @return Return the normalised url SHA-256 digest as String.
+     * @return The value found in the metadata for the key configured with {@link
+     *     #DOC_ID_METADATA_PARAM_NAME}, if any, otherwise the normalised url SHA-256 digest as
+     *     String.
      */
     protected String getDocumentID(Metadata metadata, String normalisedUrl) {
+        final String fromMetadata = getDocumentIDFromMetadata(metadata);
+        if (fromMetadata != null) {
+            return fromMetadata;
+        }
         return org.apache.commons.codec.digest.DigestUtils.sha256Hex(normalisedUrl);
     }
 
+    /**
+     * Returns the value found in the metadata for the key configured with {@link
+     * #DOC_ID_METADATA_PARAM_NAME}, or null if the parameter is not set or has no matching value in
+     * the metadata. Subclasses which override {@link #getDocumentID} entirely (e.g. to hash or
+     * sanitize the ID for a specific backend) should call this first so that they still honor the
+     * configured metadata-based override.
+     */
+    protected final String getDocumentIDFromMetadata(Metadata metadata) {
+        if (docIdMetadataKey == null) {
+            return null;
+        }
+        return StringUtils.trimToNull(metadata.getFirstValue(docIdMetadataKey));
+    }
+
     /**
      * Returns the value to be used as the URL for indexing purposes. If present the canonical value
      * is used instead.
diff --git a/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java b/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java
index cf8cde88..74c568bb 100644
--- a/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java
+++ b/core/src/test/java/org/apache/stormcrawler/indexer/BasicIndexingTest.java
@@ -197,6 +197,43 @@ class BasicIndexingTest extends IndexerTester {
                 "Index only the URL if no mapping is provided");
     }
 
+    @Test
+    void testDocumentIDFromMetadata() throws Exception {
+        Map<String, Object> config = new HashMap<>();
+        config.put(AbstractIndexerBolt.DOC_ID_METADATA_PARAM_NAME, "content.hash");
+        Metadata metadata = new Metadata();
+        metadata.setValue("content.hash", "abcdef123456");
+        prepareIndexerBolt(config);
+        String docId = ((DummyIndexer) bolt).docId(metadata, URL);
+        Assertions.assertEquals(
+                "abcdef123456",
+                docId,
+                "The value found in the metadata should be used as the document ID");
+    }
+
+    @Test
+    void testDocumentIDFromMetadataMissingValue() throws Exception {
+        Map<String, Object> config = new HashMap<>();
+        config.put(AbstractIndexerBolt.DOC_ID_METADATA_PARAM_NAME, "content.hash");
+        prepareIndexerBolt(config);
+        String docId = ((DummyIndexer) bolt).docId(new Metadata(), URL);
+        Assertions.assertEquals(
+                org.apache.commons.codec.digest.DigestUtils.sha256Hex(URL),
+                docId,
+                "Should fall back to the URL digest if the metadata key has no value");
+    }
+
+    @Test
+    void testDocumentIDDefault() throws Exception {
+        Map<String, Object> config = new HashMap<>();
+        prepareIndexerBolt(config);
+        String docId = ((DummyIndexer) bolt).docId(new Metadata(), URL);
+        Assertions.assertEquals(
+                org.apache.commons.codec.digest.DigestUtils.sha256Hex(URL),
+                docId,
+                "Should default to the URL digest if no metadata key is configured");
+    }
+
     @Test
     void testGlobFilterMetadata() throws Exception {
         Map<String, Object> config = new HashMap<>();
diff --git a/core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java b/core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java
index 55d5d292..659f542f 100644
--- a/core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java
+++ b/core/src/test/java/org/apache/stormcrawler/indexer/DummyIndexer.java
@@ -86,4 +86,8 @@ public class DummyIndexer extends AbstractIndexerBolt {
     public Map<String, String> returnFields() {
         return fields;
     }
+
+    public String docId(Metadata metadata, String url) {
+        return getDocumentID(metadata, url);
+    }
 }
diff --git a/docs/src/main/asciidoc/configuration.adoc b/docs/src/main/asciidoc/configuration.adoc
index fd9e52b6..c629e13a 100644
--- a/docs/src/main/asciidoc/configuration.adoc
+++ b/docs/src/main/asciidoc/configuration.adoc
@@ -318,6 +318,7 @@ The values below are used by sub-classes of `AbstractIndexerBolt`.
 
 | indexer.canonical.name | canonical | Metadata key for the canonical URL. Used to replace the URL with its canonical form before indexing.
 | indexer.ignore.empty.fields | false | If true, skip fields with empty values when indexing.
+| indexer.md.docid | - | Metadata key whose value, if present, is used as the document ID instead of the default SHA-256 digest of the URL. Useful e.g. for content-based deduplication when a `ParseFilter` stores a content hash in the metadata.
 | indexer.md.filter | - | YAML list of key=value filters for metadata-based indexing.
 | indexer.md.mapping | - | YAML mapping from metadata fields to persistence layer fields.
 | indexer.text.fieldname | content | Field name for indexed HTML body text.
diff --git a/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java
index 26c25b1b..518007ac 100644
--- a/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java
+++ b/external/aws/src/main/java/org/apache/stormcrawler/aws/bolt/CloudSearchIndexerBolt.java
@@ -174,6 +174,10 @@ public class CloudSearchIndexerBolt extends AbstractIndexerBolt {
 
     @Override
     protected String getDocumentID(Metadata metadata, String normalisedUrl) {
+        final String fromMetadata = getDocumentIDFromMetadata(metadata);
+        if (fromMetadata != null) {
+            return CloudSearchUtils.getID(fromMetadata);
+        }
         return CloudSearchUtils.getID(normalisedUrl);
     }
 
diff --git a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
index 0c10dd3c..e6ea14dc 100644
--- a/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
+++ b/external/opensearch-java/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
@@ -19,12 +19,14 @@ package org.apache.stormcrawler.opensearch.bolt;
 
 import java.lang.invoke.MethodHandles;
 import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.storm.task.OutputCollector;
 import org.apache.storm.task.TopologyContext;
 import org.apache.storm.topology.OutputFieldsDeclarer;
 import org.apache.storm.topology.base.BaseRichBolt;
 import org.apache.storm.tuple.Tuple;
 import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
 import org.apache.stormcrawler.metrics.CrawlerMetrics;
 import org.apache.stormcrawler.opensearch.AsyncBulkProcessor;
 import org.apache.stormcrawler.opensearch.OpenSearchConnection;
@@ -56,6 +58,8 @@ public class DeletionBolt extends BaseRichBolt implements AsyncBulkProcessor.Lis
 
     private WaitAckCache waitAck;
 
+    private String docIdMetadataKey;
+
     public DeletionBolt() {}
 
     /** Sets the index name instead of taking it from the configuration. * */
@@ -71,6 +75,9 @@ public class DeletionBolt extends BaseRichBolt implements AsyncBulkProcessor.Lis
             indexName = ConfUtils.getString(conf, IndexerBolt.OSIndexNameParamName, "content");
         }
 
+        docIdMetadataKey =
+                ConfUtils.getString(conf, AbstractIndexerBolt.DOC_ID_METADATA_PARAM_NAME);
+
         try {
             connection = OpenSearchConnection.getConnection(conf, BOLT_TYPE, this);
         } catch (Exception e1) {
@@ -121,13 +128,23 @@ public class DeletionBolt extends BaseRichBolt implements AsyncBulkProcessor.Lis
     }
 
     /**
-     * Get the document id.
+     * Get the document id. Uses the value found in the metadata for the key configured with {@link
+     * AbstractIndexerBolt#DOC_ID_METADATA_PARAM_NAME} if present, so that the ID matches the one
+     * generated by the indexer for the same document.
      *
      * @param metadata The {@link Metadata}.
      * @param url The normalised url.
-     * @return Return the normalised url SHA-256 digest as String.
+     * @return Return the value found in the metadata, or the normalised url SHA-256 digest as
+     *     String.
      */
     protected String getDocumentID(Metadata metadata, String url) {
+        if (docIdMetadataKey != null) {
+            final String fromMetadata =
+                    StringUtils.trimToNull(metadata.getFirstValue(docIdMetadataKey));
+            if (fromMetadata != null) {
+                return fromMetadata;
+            }
+        }
         return org.apache.commons.codec.digest.DigestUtils.sha256Hex(url);
     }
 
diff --git a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
index 9295aa15..51e2c003 100644
--- a/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
+++ b/external/opensearch/src/main/java/org/apache/stormcrawler/opensearch/bolt/DeletionBolt.java
@@ -19,12 +19,14 @@ package org.apache.stormcrawler.opensearch.bolt;
 
 import java.lang.invoke.MethodHandles;
 import java.util.Map;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.storm.task.OutputCollector;
 import org.apache.storm.task.TopologyContext;
 import org.apache.storm.topology.OutputFieldsDeclarer;
 import org.apache.storm.topology.base.BaseRichBolt;
 import org.apache.storm.tuple.Tuple;
 import org.apache.stormcrawler.Metadata;
+import org.apache.stormcrawler.indexing.AbstractIndexerBolt;
 import org.apache.stormcrawler.metrics.CrawlerMetrics;
 import org.apache.stormcrawler.opensearch.OpenSearchConnection;
 import org.apache.stormcrawler.opensearch.WaitAckCache;
@@ -56,6 +58,8 @@ public class DeletionBolt extends BaseRichBolt implements Listener {
 
     private WaitAckCache waitAck;
 
+    private String docIdMetadataKey;
+
     public DeletionBolt() {}
 
     /** Sets the index name instead of taking it from the configuration. * */
@@ -71,6 +75,9 @@ public class DeletionBolt extends BaseRichBolt implements Listener {
             indexName = ConfUtils.getString(conf, IndexerBolt.OSIndexNameParamName, "content");
         }
 
+        docIdMetadataKey =
+                ConfUtils.getString(conf, AbstractIndexerBolt.DOC_ID_METADATA_PARAM_NAME);
+
         try {
             connection = OpenSearchConnection.getConnection(conf, BOLT_TYPE, this);
         } catch (Exception e1) {
@@ -119,13 +126,23 @@ public class DeletionBolt extends BaseRichBolt implements Listener {
     }
 
     /**
-     * Get the document id.
+     * Get the document id. Uses the value found in the metadata for the key configured with {@link
+     * AbstractIndexerBolt#DOC_ID_METADATA_PARAM_NAME} if present, so that the ID matches the one
+     * generated by the indexer for the same document.
      *
      * @param metadata The {@link Metadata}.
      * @param url The normalised url.
-     * @return Return the normalised url SHA-256 digest as String.
+     * @return Return the value found in the metadata, or the normalised url SHA-256 digest as
+     *     String.
      */
     protected String getDocumentID(Metadata metadata, String url) {
+        if (docIdMetadataKey != null) {
+            final String fromMetadata =
+                    StringUtils.trimToNull(metadata.getFirstValue(docIdMetadataKey));
+            if (fromMetadata != null) {
+                return fromMetadata;
+            }
+        }
         return org.apache.commons.codec.digest.DigestUtils.sha256Hex(url);
     }