You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ds...@apache.org on 2026/04/26 03:43:46 UTC
(solr) 04/04: SOLR-18189: Skip indexing duplicate docs via a content hash URP (#4263)
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit 8241c2ef0aff7da8f5c24667cdb0dad09c43e37b
Author: Francois Huaulme <fr...@gmail.com>
AuthorDate: Sat Apr 25 00:17:47 2026 -0400
SOLR-18189: Skip indexing duplicate docs via a content hash URP (#4263)
Co-authored-by: fhuaulme <fh...@salesforce.com>
Co-authored-by: David Smiley <ds...@apache.org>
(cherry picked from commit ae4f479898eff2d7b3b4c6d1c6c12535fea6f561)
---
changelog/unreleased/SOLR-18189.yml | 8 +
.../processor/ContentHashVersionProcessor.java | 179 ++++++++
.../ContentHashVersionProcessorFactory.java | 219 ++++++++++
.../test-files/solr/collection1/conf/schema16.xml | 34 ++
.../conf/solrconfig-contenthashversion.xml | 65 +++
.../ContentHashVersionProcessorFactoryTest.java | 156 +++++++
.../processor/ContentHashVersionProcessorTest.java | 452 +++++++++++++++++++++
.../pages/update-request-processors.adoc | 6 +
.../src/java/org/apache/solr/SolrTestCaseJ4.java | 8 +-
9 files changed, 1125 insertions(+), 2 deletions(-)
diff --git a/changelog/unreleased/SOLR-18189.yml b/changelog/unreleased/SOLR-18189.yml
new file mode 100644
index 00000000000..638acdf9950
--- /dev/null
+++ b/changelog/unreleased/SOLR-18189.yml
@@ -0,0 +1,8 @@
+title: New ContentHashVersionProcessor to avoid index churn when adding same-content documents.
+type: added
+authors:
+ - name: Francois Huaulme
+ - name: David Smiley
+links:
+ - name: SOLR-18189
+ url: https://issues.apache.org/jira/browse/SOLR-18189
diff --git a/solr/core/src/java/org/apache/solr/update/processor/ContentHashVersionProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/ContentHashVersionProcessor.java
new file mode 100644
index 00000000000..457dcc92e16
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/update/processor/ContentHashVersionProcessor.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update.processor;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Predicate;
+import org.apache.lucene.util.BytesRef;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.SolrInputField;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.handler.component.RealTimeGetComponent;
+import org.apache.solr.handler.component.RealTimeGetComponent.Resolution;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.update.AddUpdateCommand;
+
+/**
+ * An implementation of {@link UpdateRequestProcessor} which computes a hash of field values, and
+ * uses this hash to reject/accept document updates.
+ *
+ * <ul>
+ * <li>When no corresponding document with same id exists (create), the computed hash is added to
+ * the document.
+ * <li>When a previous document exists (update), a new hash is computed from the incoming field
+ * values and compared with the stored hash.
+ * </ul>
+ *
+ * <p>Depending on {#dropSameDocuments} value, this processor may drop or accept document updates.
+ * This implementation can be used for monitoring or dropping no-op updates (updates that do not
+ * change the Solr document content).
+ *
+ * <p>Note: the hash is computed using {@link Lookup3Signature} and must be stored in a field with
+ * docValues enabled for retrieval.
+ *
+ * @see Lookup3Signature
+ */
+public class ContentHashVersionProcessor extends UpdateRequestProcessor {
+ private final SchemaField hashField;
+ private final SolrQueryResponse rsp;
+ private final SolrCore core;
+ private final Predicate<SolrInputField> includedFields; // Matcher for included fields in hash
+ private final Predicate<SolrInputField> excludedFields; // Matcher for excluded fields from hash
+ private boolean dropSameDocuments;
+ private int sameCount = 0;
+ private int differentCount = 0;
+
+ public ContentHashVersionProcessor(
+ Predicate<SolrInputField> hashIncludedFields,
+ Predicate<SolrInputField> hashExcludedFields,
+ String hashFieldName,
+ boolean dropSameDocuments,
+ SolrQueryRequest req,
+ SolrQueryResponse rsp,
+ UpdateRequestProcessor next) {
+ super(next);
+ this.core = req.getCore();
+
+ IndexSchema schema = core.getLatestSchema();
+ this.hashField = schema.getField(hashFieldName);
+ this.dropSameDocuments = dropSameDocuments;
+ this.rsp = rsp;
+ this.includedFields = hashIncludedFields;
+ this.excludedFields = hashExcludedFields;
+ }
+
+ @Override
+ public void processAdd(AddUpdateCommand cmd) throws IOException {
+ SolrInputDocument newDoc = cmd.getSolrInputDocument();
+ byte[] newHash = computeDocHash(newDoc);
+ newDoc.setField(hashField.getName(), newHash);
+
+ if (!isHashAcceptable(cmd.getIndexedId(), newHash)) {
+ return;
+ }
+ super.processAdd(cmd);
+ }
+
+ @Override
+ public void finish() throws IOException {
+ try {
+ super.finish();
+ } finally {
+ if (sameCount + differentCount > 0) {
+ if (dropSameDocuments) {
+ rsp.addToLog("contentHash.duplicatesDropped", sameCount);
+ } else {
+ rsp.addToLog("contentHash.duplicatesDetected", sameCount);
+ }
+ }
+ }
+ }
+
+ private boolean isHashAcceptable(BytesRef indexedDocId, byte[] newHash) throws IOException {
+ assert null != indexedDocId;
+
+ Optional<byte[]> oldDocHash = getOldDocHash(indexedDocId);
+ if (oldDocHash.isPresent()) {
+ if (Arrays.equals(newHash, oldDocHash.get())) {
+ sameCount++;
+ return !dropSameDocuments;
+ } else {
+ differentCount++;
+ return true;
+ }
+ }
+ return true; // Doc not found
+ }
+
+ /** Retrieves the hash value from the old document identified by the given ID. */
+ private Optional<byte[]> getOldDocHash(BytesRef indexedDocId) throws IOException {
+ SolrInputDocument oldDoc =
+ RealTimeGetComponent.getInputDocument(
+ core, indexedDocId, indexedDocId, null, Set.of(hashField.getName()), Resolution.DOC);
+ if (oldDoc == null) {
+ return Optional.empty();
+ }
+ Object o = oldDoc.getFieldValue(hashField.getName());
+ if (o instanceof byte[]) {
+ return Optional.of((byte[]) o);
+ } else if (o instanceof ByteBuffer) {
+ var buf = (ByteBuffer) o;
+ byte[] bytes = new byte[buf.remaining()];
+ buf.duplicate().get(bytes);
+ return Optional.of(bytes);
+ }
+ return Optional.empty();
+ }
+
+ byte[] computeDocHash(SolrInputDocument doc) {
+ final Signature sig = new Lookup3Signature();
+
+ // Stream field names, filter, sort, and process in a single pass
+ doc.values().stream()
+ .filter(includedFields) // Keep fields that match 'included fields' matcher
+ .filter(excludedFields.negate()) // Exclude fields that match 'excluded fields' matcher
+ .sorted(
+ Comparator.comparing(
+ SolrInputField
+ ::getName)) // Sort to ensure consistent field order across different doc field
+ // orders
+ .forEach(
+ inputField -> {
+ sig.add(inputField.getName());
+ Object o = inputField.getValue();
+ if (o instanceof Collection) {
+ for (Object oo : (Collection<?>) o) {
+ sig.add(String.valueOf(oo));
+ }
+ } else {
+ sig.add(String.valueOf(o));
+ }
+ });
+
+ return sig.getSignature();
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/update/processor/ContentHashVersionProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/ContentHashVersionProcessorFactory.java
new file mode 100644
index 00000000000..479ce33e987
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/update/processor/ContentHashVersionProcessorFactory.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.update.processor;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputField;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.StrUtils;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+
+/**
+ * Factory for {@link ContentHashVersionProcessor} instances.
+ *
+ * <p>This processor computes a hash from the document content and sees if an existing (indexed)
+ * document also has this hash, stored in a designated field. If so, it drops the document to avoid
+ * needless index churn. Alternatively, it can be configured to merely log the fact.
+ *
+ * <h2>Configuration</h2>
+ *
+ * <ul>
+ * <li><b>hashField</b> (required): The name of the field where the computed hash will be stored.
+ * It should be of type BinaryField. This field must have docValues enabled for hash
+ * retrieval. The hash field is automatically excluded from hash computation.
+ * <li><b>includeFields</b> (optional, default="*"): Comma-separated list of fields to include in
+ * hash computation. Supports wildcard patterns (e.g., "name*"). Use "*" to include all
+ * fields.
+ * <li><b>excludeFields</b> (optional): Comma-separated list of fields to exclude from hash
+ * computation. Supports wildcard patterns. Cannot be "*" (cannot exclude all fields).
+ * <li><b>hashCompareStrategy</b> (optional, default="drop"): Controls behavior when duplicate
+ * content is detected:
+ * <ul>
+ * <li>"drop": Silently drops documents with matching hash (no-op updates)
+ * <li>"log": Logs duplicate detection but still processes the update
+ * </ul>
+ * </ul>
+ *
+ * <h2>Configuration Example</h2>
+ *
+ * <pre class="prettyprint">
+ * <processor class="solr.ContentHashVersionProcessorFactory">
+ * <str name="hashField">content_hash</str>
+ * <str name="includeFields">title,body,author</str>
+ * <str name="excludeFields">timestamp,version</str>
+ * <str name="hashCompareStrategy">drop</str>
+ * </processor>
+ * </pre>
+ *
+ * <h2>Important Considerations</h2>
+ *
+ * <ul>
+ * <li><b>In-Place Updates</b>: Fields updated via in-place (partial) updates should be excluded
+ * from hash computation using <code>excludeFields</code>, as these are updated independently
+ * and should not affect duplicate detection.
+ * </ul>
+ *
+ * <h2>Monitoring</h2>
+ *
+ * <p>The processor logs duplicate statistics in the response:
+ *
+ * <ul>
+ * <li><code>contentHash.duplicatesDropped</code>: Count of duplicates dropped (when
+ * hashCompareStrategy=drop)
+ * <li><code>contentHash.duplicatesDetected</code>: Count of duplicates detected (when
+ * hashCompareStrategy=log)
+ * </ul>
+ *
+ * @see ContentHashVersionProcessor
+ * @see Lookup3Signature
+ */
+public class ContentHashVersionProcessorFactory extends UpdateRequestProcessorFactory {
+ private static final char SEPARATOR = ','; // Separator for included/excluded fields
+ private List<String> includeFields = List.of("*"); // Included fields defaults to 'all'
+ private List<String> excludeFields = new ArrayList<>();
+ private String hashField; // Must be explicitly configured
+ private boolean dropSameDocuments = true;
+
+ public ContentHashVersionProcessorFactory() {}
+
+ @Override
+ public void init(NamedList<?> args) {
+ Object tmp = args.remove("includeFields");
+ if (tmp != null) {
+ if (!(tmp instanceof String)) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "'includeFields' must be configured as a <str>");
+ }
+ // Include fields support comma separated list of fields (e.g. "field1,field2,field3").
+ // Also supports "*" to include all fields
+ this.includeFields =
+ StrUtils.splitSmart((String) tmp, SEPARATOR).stream()
+ .map(String::trim)
+ .collect(Collectors.toList());
+ }
+ tmp = args.remove("hashField");
+ if (tmp == null) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "'hashField' is required and must be explicitly configured");
+ }
+ if (!(tmp instanceof String)) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "'hashField' must be configured as a <str>");
+ }
+ this.hashField = (String) tmp;
+
+ tmp = args.remove("excludeFields");
+ if (tmp != null) {
+ if (!(tmp instanceof String)) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "'excludeFields' must be configured as a <str>");
+ }
+ if ("*".equals(((String) tmp).trim())) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR, "'excludeFields' can't exclude all fields.");
+ }
+ // Exclude fields support comma separated list of fields (e.g.
+ // "excluded_field1,excluded_field2").
+ // Also supports "*" to exclude all fields
+ this.excludeFields =
+ StrUtils.splitSmart((String) tmp, SEPARATOR).stream()
+ .map(String::trim)
+ .collect(Collectors.toList());
+ }
+ excludeFields.add(hashField); // Hash field name is excluded from hash computation
+
+ tmp = args.remove("hashCompareStrategy");
+ if (tmp != null) {
+ if (!(tmp instanceof String)) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "'hashCompareStrategy' must be configured as a <str>");
+ }
+ String value = ((String) tmp).toLowerCase(Locale.ROOT);
+ if ("drop".equalsIgnoreCase(value)) {
+ dropSameDocuments = true;
+ } else if ("log".equalsIgnoreCase(value)) {
+ dropSameDocuments = false;
+ } else {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ "Value '"
+ + value
+ + "' is unsupported for 'hashCompareStrategy', only 'drop' and 'log' are supported.");
+ }
+ }
+
+ super.init(args);
+ }
+
+ @Override
+ public UpdateRequestProcessor getInstance(
+ SolrQueryRequest req, SolrQueryResponse rsp, UpdateRequestProcessor next) {
+ return new ContentHashVersionProcessor(
+ buildFieldMatcher(includeFields),
+ buildFieldMatcher(excludeFields),
+ hashField,
+ dropSameDocuments,
+ req,
+ rsp,
+ next);
+ }
+
+ public String getHashField() {
+ return hashField;
+ }
+
+ public List<String> getIncludeFields() {
+ return includeFields;
+ }
+
+ public List<String> getExcludeFields() {
+ return excludeFields;
+ }
+
+ public boolean dropSameDocuments() {
+ return dropSameDocuments;
+ }
+
+ static Predicate<SolrInputField> buildFieldMatcher(List<String> fieldNames) {
+ return inputField -> {
+ for (String currentFieldName : fieldNames) {
+ if ("*".equals(currentFieldName)) {
+ return true;
+ }
+ final String fieldName = inputField.getName();
+ if (fieldName.equals(currentFieldName)) {
+ return true;
+ }
+ if (currentFieldName.length() > 1
+ && currentFieldName.endsWith("*")
+ && fieldName.startsWith(currentFieldName.substring(0, currentFieldName.length() - 1))) {
+ return true;
+ }
+ }
+ return false;
+ };
+ }
+}
diff --git a/solr/core/src/test-files/solr/collection1/conf/schema16.xml b/solr/core/src/test-files/solr/collection1/conf/schema16.xml
new file mode 100644
index 00000000000..f34bfed2e4c
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/schema16.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" ?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!-- The Solr schema file, version 1.6 -->
+
+<schema name="test" version="1.6">
+ <fieldType name="string" class="solr.StrField" sortMissingLast="true"/>
+ <fieldType name="binary" class="solr.BinaryField"/>
+
+ <field name="_id" type="string" indexed="true" stored="true" multiValued="false" required="true"/>
+ <field name="_version_" type="string" indexed="true" stored="true" multiValued="false" required="true"/>
+ <field name="_hash_" type="binary" indexed="true" docValues="true" stored="false" required="true"/>
+ <field name="field1" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="field2" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="docField3" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+ <field name="field4" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
+
+ <uniqueKey>_id</uniqueKey>
+</schema>
diff --git a/solr/core/src/test-files/solr/collection1/conf/solrconfig-contenthashversion.xml b/solr/core/src/test-files/solr/collection1/conf/solrconfig-contenthashversion.xml
new file mode 100644
index 00000000000..1308c63a972
--- /dev/null
+++ b/solr/core/src/test-files/solr/collection1/conf/solrconfig-contenthashversion.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0" ?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<config>
+
+ <luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
+
+ <directoryFactory name="DirectoryFactory" class="${solr.directoryFactory:solr.MockDirectoryFactory}"/>
+ <schemaFactory class="ClassicIndexSchemaFactory"/>
+
+ <dataDir>${solr.data.dir:}</dataDir>
+
+ <xi:include href="solrconfig.snippet.randomindexconfig.xml" xmlns:xi="http://www.w3.org/2001/XInclude"/>
+
+ <updateHandler class="solr.DirectUpdateHandler2">
+ <updateLog>
+ <str name="dir">${solr.ulog.dir:}</str>
+ </updateLog>
+ </updateHandler>
+
+ <updateRequestProcessorChain name="contenthashversion-default" default="true">
+ <processor class="org.apache.solr.update.processor.ContentHashVersionProcessorFactory">
+ <str name="hashField">_hash_</str>
+ <str name="excludeFields">_id</str>
+ </processor>
+ <processor class="solr.RunUpdateProcessorFactory"/>
+ </updateRequestProcessorChain>
+
+ <updateRequestProcessorChain name="contenthashversion-log">
+ <processor class="org.apache.solr.update.processor.ContentHashVersionProcessorFactory">
+ <str name="hashField">_hash_</str>
+ <str name="excludeFields">_id</str>
+ <str name="hashCompareStrategy">log</str>
+ </processor>
+ <processor class="solr.RunUpdateProcessorFactory"/>
+ </updateRequestProcessorChain>
+
+ <updateRequestProcessorChain name="contenthashversion-drop">
+ <processor class="org.apache.solr.update.processor.ContentHashVersionProcessorFactory">
+ <str name="hashField">_hash_</str>
+ <str name="excludeFields">_id</str>
+ <str name="hashCompareStrategy">drop</str>
+ </processor>
+ <processor class="solr.RunUpdateProcessorFactory"/>
+ </updateRequestProcessorChain>
+
+ <requestHandler name="/select" class="solr.SearchHandler">
+ </requestHandler>
+</config>
diff --git a/solr/core/src/test/org/apache/solr/update/processor/ContentHashVersionProcessorFactoryTest.java b/solr/core/src/test/org/apache/solr/update/processor/ContentHashVersionProcessorFactoryTest.java
new file mode 100644
index 00000000000..59134eb1e55
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/update/processor/ContentHashVersionProcessorFactoryTest.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.update.processor;
+
+import static org.apache.solr.SolrTestCaseJ4.assumeWorkingMockito;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.NamedList;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class ContentHashVersionProcessorFactoryTest {
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ assumeWorkingMockito();
+ }
+
+ @Test
+ public void shouldHaveSensibleDefaultValues() {
+ ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory();
+ assertEquals(List.of("*"), factory.getIncludeFields());
+ assertTrue(factory.dropSameDocuments());
+ }
+
+ @Test
+ public void shouldInitWithHashFieldName() {
+ ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory();
+ NamedList<String> args = new NamedList<>();
+ args.add("hashField", "_hash_field_");
+ factory.init(args);
+
+ assertEquals("_hash_field_", factory.getHashField());
+ }
+
+ @Test
+ public void shouldInitWithAllField() {
+ ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory();
+ NamedList<String> args = new NamedList<>();
+ args.add("hashField", "content_hash");
+ args.add("includeFields", "*");
+ factory.init(args);
+
+ assertEquals(1, factory.getIncludeFields().size());
+ assertEquals("*", factory.getIncludeFields().get(0));
+ }
+
+ @Test
+ public void shouldInitWithIncludedFields() {
+ ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory();
+ NamedList<String> args = new NamedList<>();
+ args.add("hashField", "content_hash");
+ args.add("includeFields", " field1,field2 , field3 ");
+ factory.init(args);
+
+ assertEquals(3, factory.getIncludeFields().size());
+ assertEquals(List.of("field1", "field2", "field3"), factory.getIncludeFields());
+ }
+
+ @Test
+ public void shouldInitWithExcludedFields() {
+ ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory();
+ NamedList<String> args = new NamedList<>();
+ args.add("hashField", "content_hash");
+ args.add("excludeFields", " field1,field2 , field3 ");
+ factory.init(args);
+
+ assertEquals(4, factory.getExcludeFields().size());
+ assertEquals(List.of("field1", "field2", "field3", "content_hash"), factory.getExcludeFields());
+ }
+
+ @Test
+ public void shouldSelectDropStrategy() {
+ ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory();
+ NamedList<String> args = new NamedList<>();
+ args.add("hashField", "content_hash");
+ args.add("hashCompareStrategy", "drop");
+ factory.init(args);
+
+ assertTrue(factory.dropSameDocuments());
+ }
+
+ @Test
+ public void shouldSelectLogStrategy() {
+ ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory();
+ NamedList<String> args = new NamedList<>();
+ args.add("hashField", "content_hash");
+ args.add("hashCompareStrategy", "log");
+ factory.init(args);
+
+ assertFalse(factory.dropSameDocuments());
+ }
+
+ @Test(expected = SolrException.class)
+ public void shouldSelectUnsupportedStrategy() {
+ ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory();
+ NamedList<String> args = new NamedList<>();
+ args.add("hashField", "content_hash");
+ args.add("hashCompareStrategy", "unsupported value");
+ factory.init(args);
+ }
+
+ @Test(expected = SolrException.class)
+ public void shouldRejectExcludeAllFields() {
+ ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory();
+ NamedList<String> args = new NamedList<>();
+ args.add("hashField", "content_hash");
+ args.add("excludeFields", "*");
+ factory.init(args);
+ }
+
+ @Test(expected = SolrException.class)
+ public void shouldRequireExplicitHashFieldName() {
+ ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory();
+ NamedList<String> args = new NamedList<>();
+ // Intentionally not setting hashField
+ factory.init(args);
+ }
+
+ @Test
+ public void shouldAutoExcludeHashFieldFromHashComputation() {
+ ContentHashVersionProcessorFactory factory = new ContentHashVersionProcessorFactory();
+ NamedList<String> args = new NamedList<>();
+ args.add("hashField", "my_hash_field");
+ args.add("excludeFields", "field1,field2");
+ factory.init(args);
+
+ // Hash field should be automatically added to excludeFields
+ assertEquals(3, factory.getExcludeFields().size());
+ assertTrue(
+ "Should contain explicitly excluded field1", factory.getExcludeFields().contains("field1"));
+ assertTrue(
+ "Should contain explicitly excluded field2", factory.getExcludeFields().contains("field2"));
+ assertTrue(
+ "Should auto-exclude hash field name",
+ factory.getExcludeFields().contains("my_hash_field"));
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/update/processor/ContentHashVersionProcessorTest.java b/solr/core/src/test/org/apache/solr/update/processor/ContentHashVersionProcessorTest.java
new file mode 100644
index 00000000000..4a3764ed6a7
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/update/processor/ContentHashVersionProcessorTest.java
@@ -0,0 +1,452 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.update.processor;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.List;
+import java.util.UUID;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.BinaryField;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class ContentHashVersionProcessorTest extends UpdateProcessorTestBase {
+
+ private static final String ID_FIELD = "_id";
+ private static final String FIRST_FIELD = "field1";
+ private static final String SECOND_FIELD = "field2";
+ private static final String THIRD_FIELD = "docField3";
+ private static final String FOURTH_FIELD = "field4";
+
+ private static final String INITIAL_DOC_ID = "1";
+ private static final String INITIAL_FIELD1_VALUE = "Initial values used to compute initial hash";
+ private static final String INITIAL_FIELD2_VALUE =
+ "This a constant value for testing include/exclude fields";
+ private static final String INITIAL_DOC =
+ adoc(
+ ID_FIELD, INITIAL_DOC_ID,
+ FIRST_FIELD, INITIAL_FIELD1_VALUE,
+ SECOND_FIELD, INITIAL_FIELD2_VALUE);
+ private String initialDocHash;
+
+ @BeforeClass
+ public static void beforeClass() throws Exception {
+ assumeWorkingMockito();
+ initCore("solrconfig-contenthashversion.xml", "schema16.xml");
+ }
+
+ @Before
+ @Override
+ public void setUp() throws Exception {
+ super.setUp();
+ assertU(delQ("*:*"));
+ addDoc(INITIAL_DOC, "contenthashversion-default");
+ assertU(commit());
+
+ // Query for the document and extract _hash_ field value
+ initialDocHash = getHashFieldValue(INITIAL_DOC_ID);
+ }
+
+ private static String getHashFieldValue(String docId) throws Exception {
+ String response = h.query(req("q", ID_FIELD + ":" + docId, "fl", "_hash_"));
+
+ // Parse XML response to extract _hash_ field value
+ // Response format: <str name="_hash_">value</str>
+ String hashPattern = "<str name=\"_hash_\">";
+ int startIdx = response.indexOf(hashPattern);
+ if (startIdx == -1) {
+ fail("Hash field not found in document " + docId);
+ }
+ startIdx += hashPattern.length();
+ int endIdx = response.indexOf("</str>", startIdx);
+ if (endIdx == -1) {
+ fail("Hash field closing tag not found");
+ }
+ return response.substring(startIdx, endIdx);
+ }
+
+ private ContentHashVersionProcessor getContentHashVersionProcessor(
+ List<String> includedFields, List<String> excludedFields) {
+ final SolrQueryRequest req = mock(SolrQueryRequest.class);
+ final SolrCore solrCore = mock(SolrCore.class);
+ final IndexSchema indexSchema = mock(IndexSchema.class);
+ when(indexSchema.getField("_hash_")).thenReturn(new SchemaField("_hash_", new BinaryField()));
+
+ when(solrCore.getLatestSchema()).thenReturn(indexSchema);
+ when(req.getCore()).thenReturn(solrCore);
+ return new ContentHashVersionProcessor(
+ ContentHashVersionProcessorFactory.buildFieldMatcher(includedFields),
+ ContentHashVersionProcessorFactory.buildFieldMatcher(excludedFields),
+ "_hash_",
+ false,
+ req,
+ mock(SolrQueryResponse.class),
+ mock(UpdateRequestProcessor.class));
+ }
+
+ @Test
+ public void shouldUseExcludedFieldsWildcard() {
+ // Given
+ ContentHashVersionProcessor processor =
+ getContentHashVersionProcessor(List.of("*"), List.of("field*"));
+
+ // Given (doc for update)
+ SolrInputDocument inputDocument =
+ doc(
+ f(ID_FIELD, "0000000001"),
+ f(FIRST_FIELD, UUID.randomUUID().toString()),
+ f(SECOND_FIELD, UUID.randomUUID().toString()),
+ f(THIRD_FIELD, "constant to have a constant hash"),
+ f(FOURTH_FIELD, UUID.randomUUID().toString()));
+
+ // Then (only ID and THIRD_FIELD is used in hash, other fields contain random values)
+ assertArrayEquals(
+ Base64.getDecoder().decode("bwE8Zjq0aOs="),
+ processor.computeDocHash(inputDocument)); // Hash if only ID field was used
+ }
+
+ @Test
+ public void shouldUseIncludedFieldsWildcard() {
+ // Given
+ ContentHashVersionProcessor processor =
+ getContentHashVersionProcessor(List.of("field*"), List.of(THIRD_FIELD));
+
+ // Given (doc for update)
+ SolrInputDocument inputDocument =
+ doc(
+ f(ID_FIELD, "0000000001"),
+ f(FIRST_FIELD, "constant to have a constant hash for field1"),
+ f(SECOND_FIELD, "constant to have a constant hash for field2"),
+ f(THIRD_FIELD, UUID.randomUUID().toString()),
+ f(FOURTH_FIELD, "constant to have a constant hash for field4"));
+
+ // Then
+ assertArrayEquals(
+ Base64.getDecoder().decode("PozPs2qZQtw="), processor.computeDocHash(inputDocument));
+ }
+
+ @Test
+ public void shouldUseIncludedFieldsWildcard2() {
+ // Given (variant of previous shouldUseIncludedFieldsWildcard, without the excludedField config)
+ ContentHashVersionProcessor processor =
+ getContentHashVersionProcessor(List.of("field*"), List.of());
+
+ // Given (doc for update)
+ SolrInputDocument inputDocument =
+ doc(
+ f(ID_FIELD, "0000000001"),
+ f(FIRST_FIELD, "constant to have a constant hash for field1"),
+ f(SECOND_FIELD, "constant to have a constant hash for field2"),
+ f(THIRD_FIELD, UUID.randomUUID().toString()),
+ f(FOURTH_FIELD, "constant to have a constant hash for field4"));
+
+ // Then
+ assertArrayEquals(
+ Base64.getDecoder().decode("PozPs2qZQtw="), processor.computeDocHash(inputDocument));
+ }
+
+ @Test
+ public void shouldDedupIncludedFields() {
+ // Given (processor to include field1 and field2 only)
+ ContentHashVersionProcessor processorWithDuplicatedFieldName =
+ getContentHashVersionProcessor(List.of(FIRST_FIELD, FIRST_FIELD, SECOND_FIELD), List.of());
+ ContentHashVersionProcessor processorWithWildcard =
+ getContentHashVersionProcessor(
+ List.of( // Also change order of config (test reorder of field names)
+ SECOND_FIELD, FIRST_FIELD, "field1*"),
+ List.of());
+
+ // Given (doc for update)
+ SolrInputDocument inputDocument =
+ doc(
+ f(ID_FIELD, "0000000001"),
+ f(FIRST_FIELD, "constant to have a constant hash for field1"),
+ f(SECOND_FIELD, "constant to have a constant hash for field2"),
+ f(THIRD_FIELD, UUID.randomUUID().toString()),
+ f(FOURTH_FIELD, "constant to have a constant hash for field4"));
+
+ // Then
+ assertArrayEquals(
+ Base64.getDecoder().decode("XavrOYGlkXM="),
+ processorWithDuplicatedFieldName.computeDocHash(inputDocument));
+ assertArrayEquals(
+ Base64.getDecoder().decode("XavrOYGlkXM="),
+ processorWithWildcard.computeDocHash(inputDocument));
+ }
+
+ @Test
+ public void shouldCreateSignatureForNewDoc() throws Exception {
+ // When (update)
+ final String newDocId = UUID.randomUUID().toString();
+ assertU(
+ adoc(
+ ID_FIELD, newDocId,
+ FIRST_FIELD, INITIAL_FIELD1_VALUE,
+ SECOND_FIELD, INITIAL_FIELD2_VALUE));
+ assertU(commit());
+
+ // Then
+ final String hashFieldValueForNewDoc = getHashFieldValue(newDocId);
+ assertEquals(initialDocHash, hashFieldValueForNewDoc);
+ }
+
+ @Test
+ public void shouldAddToResponseLog() throws Exception {
+ // Given (command to update existing doc)
+ final String newDocId = UUID.randomUUID().toString();
+ final SolrQueryResponse update1 =
+ addDoc(
+ adoc(
+ ID_FIELD, newDocId,
+ FIRST_FIELD, INITIAL_FIELD1_VALUE,
+ SECOND_FIELD, INITIAL_FIELD2_VALUE),
+ "contenthashversion-default");
+ final SolrQueryResponse update2 =
+ addDoc(
+ adoc(
+ ID_FIELD, newDocId,
+ FIRST_FIELD, "This is a doc with values",
+ SECOND_FIELD, "that differs from stored doc, so it's considered new"),
+ "contenthashversion-default");
+ assertU(commit());
+
+ // Then
+ assertResponse(update1, -1, -1);
+ assertResponse(update2, 0, -1);
+ }
+
+ @Test
+ public void shouldKeepDuplicateDocumentsInLogMode() throws Exception {
+ // Given: Use log chain which detects but does NOT drop duplicates
+ final String docId = UUID.randomUUID().toString();
+
+ // When: Add a document
+ addDoc(
+ adoc(
+ ID_FIELD, docId,
+ FIRST_FIELD, "original value",
+ SECOND_FIELD, "original value 2"),
+ "contenthashversion-log");
+ assertU(commit());
+ String originalHash = getHashFieldValue(docId);
+
+ // When: Try to add the same content again (duplicate)
+ SolrQueryResponse duplicateResponse =
+ addDoc(
+ adoc(
+ ID_FIELD, docId,
+ FIRST_FIELD, "original value",
+ SECOND_FIELD, "original value 2"),
+ "contenthashversion-log");
+ assertU(commit());
+
+ // Then: Response should show duplicate was detected but NOT dropped
+ assertResponse(duplicateResponse, -1, 1);
+
+ // Then: Document should still exist in index
+ assertQ(req("q", ID_FIELD + ":" + docId), "//result[@numFound='1']");
+
+ // Then: Document hash should remain unchanged (duplicate was processed)
+ String currentHash = getHashFieldValue(docId);
+ assertEquals("Hash should remain unchanged for duplicate", originalHash, currentHash);
+
+ // When: Update with different content
+ SolrQueryResponse changedResponse =
+ addDoc(
+ adoc(
+ ID_FIELD, docId,
+ FIRST_FIELD, "changed value",
+ SECOND_FIELD, "changed value 2"),
+ "contenthashversion-log");
+ assertU(commit());
+
+ // Then: Response should show content changed
+ assertResponse(changedResponse, -1, 0);
+
+ // Then: Hash should be updated
+ String newHash = getHashFieldValue(docId);
+ assertNotEquals("Hash should change for different content", originalHash, newHash);
+ }
+
+ @Test
+ public void shouldExcludeFieldsUpdateSignatureForNewDoc() throws Exception {
+ // Given (update using URP chain WITHOUT drop doc (log mode))
+ final String newDocId = UUID.randomUUID().toString();
+ addDoc(
+ adoc(
+ ID_FIELD, newDocId,
+ FIRST_FIELD, INITIAL_FIELD1_VALUE,
+ SECOND_FIELD, INITIAL_FIELD2_VALUE),
+ "contenthashversion-default");
+ assertU(commit());
+
+ // Then
+ final String hashFieldValue = getHashFieldValue(newDocId);
+ assertEquals(initialDocHash, hashFieldValue);
+ }
+
+ @Test
+ public void shouldCommitWithDropModeEnabled() throws Exception {
+ // Initial document already exists from setUp()
+ // When: Try to add the same document again (duplicate content) using URP chain WITH drop doc
+ // (drop mode)
+ SolrQueryResponse solrQueryResponse =
+ addDoc(
+ adoc(
+ ID_FIELD, INITIAL_DOC_ID,
+ FIRST_FIELD, INITIAL_FIELD1_VALUE,
+ SECOND_FIELD, INITIAL_FIELD2_VALUE),
+ "contenthashversion-drop");
+ assertU(commit());
+
+ // Then: Verify response shows duplicate was dropped
+ assertResponse(solrQueryResponse, 1, -1);
+
+ // Then: Verify document was NOT actually added/updated (still only 1 doc in index)
+ assertQ(req("q", "*:*"), "//result[@numFound='1']");
+
+ // Verify the document still has the original hash
+ String currentHash = getHashFieldValue(INITIAL_DOC_ID);
+ assertEquals("Document hash should not have changed", initialDocHash, currentHash);
+ }
+
+ @Test
+ public void shouldHandleDocumentWithOnlyIdField() {
+ // Given: Document with only ID field (no other fields to hash)
+ ContentHashVersionProcessor processor =
+ getContentHashVersionProcessor(List.of("*"), List.of(ID_FIELD));
+
+ // When: Compute hash for document with only ID
+ SolrInputDocument doc = doc(f(ID_FIELD, "only-id-doc"));
+
+ // Then: Should compute hash (even if empty field set)
+ byte[] hash = processor.computeDocHash(doc);
+ assertNotNull("Hash should not be null for ID-only document", hash);
+ assertTrue("Hash should not be empty", hash.length > 0);
+ }
+
+ @Test
+ public void shouldHandleMultiValueFields() {
+ // Given: Processor that includes multi-value fields
+ ContentHashVersionProcessor processor =
+ getContentHashVersionProcessor(List.of("*"), List.of(ID_FIELD));
+
+ // When: Document with multi-value field
+ SolrInputDocument doc1 = doc(f(ID_FIELD, "doc1"), f(FIRST_FIELD, "value1", "value2", "value3"));
+
+ // Then: Should compute consistent hash
+ byte[] hash1 = processor.computeDocHash(doc1);
+ assertNotNull(hash1);
+
+ // Same values in same order should produce same hash
+ SolrInputDocument doc2 = doc(f(ID_FIELD, "doc2"), f(FIRST_FIELD, "value1", "value2", "value3"));
+ byte[] hash2 = processor.computeDocHash(doc2);
+ assertArrayEquals("Same multi-value field should produce same hash", hash1, hash2);
+
+ // Different order should produce different hash (collection order matters)
+ SolrInputDocument doc3 = doc(f(ID_FIELD, "doc3"), f(FIRST_FIELD, "value3", "value1", "value2"));
+ byte[] hash3 = processor.computeDocHash(doc3);
+ assertFalse("Different order should produce different hash", Arrays.equals(hash1, hash3));
+ }
+
+ @Test
+ public void shouldHandleNullFieldValues() {
+ // Given: Processor that handles null values
+ ContentHashVersionProcessor processor =
+ getContentHashVersionProcessor(List.of("*"), List.of(ID_FIELD));
+
+ // When: Document with null field value (represented as "null" string)
+ SolrInputDocument doc = doc(f(ID_FIELD, "null-doc"), f(FIRST_FIELD, (Object) null));
+
+ // Then: Should compute hash without error
+ byte[] hash = processor.computeDocHash(doc);
+ assertNotNull("Should handle null values", hash);
+ assertTrue("Hash should not be empty", hash.length > 0);
+ }
+
+ @Test
+ public void shouldProduceSameHashRegardlessOfFieldOrder() {
+ // Given: Documents with same fields in different order
+ ContentHashVersionProcessor processor =
+ getContentHashVersionProcessor(List.of("*"), List.of(ID_FIELD));
+
+ // When: Create docs with fields in different order
+ SolrInputDocument doc1 =
+ doc(
+ f(ID_FIELD, "doc1"),
+ f(FIRST_FIELD, "value1"),
+ f(SECOND_FIELD, "value2"),
+ f(THIRD_FIELD, "value3"));
+
+ SolrInputDocument doc2 =
+ doc(
+ f(ID_FIELD, "doc2"),
+ f(THIRD_FIELD, "value3"),
+ f(FIRST_FIELD, "value1"),
+ f(SECOND_FIELD, "value2"));
+
+ // Then: Hashes should be identical (fields are sorted before hashing)
+ byte[] hash1 = processor.computeDocHash(doc1);
+ byte[] hash2 = processor.computeDocHash(doc2);
+ assertArrayEquals("Hash should be same regardless of field order", hash1, hash2);
+ }
+
+ @Test
+ public void shouldHandleEmptyFieldValues() {
+ // Given: Document with empty string values
+ ContentHashVersionProcessor processor =
+ getContentHashVersionProcessor(List.of("*"), List.of(ID_FIELD));
+
+ SolrInputDocument doc1 = doc(f(ID_FIELD, "empty-doc"), f(FIRST_FIELD, ""), f(SECOND_FIELD, ""));
+
+ // When: Compute hash
+ byte[] hash1 = processor.computeDocHash(doc1);
+
+ // Then: Should produce valid hash
+ assertNotNull("Should handle empty values", hash1);
+ assertTrue("Hash should not be empty", hash1.length > 0);
+
+ // Empty strings should produce different hash than no fields
+ SolrInputDocument doc2 = doc(f(ID_FIELD, "empty-doc"));
+ byte[] hash2 = processor.computeDocHash(doc2);
+ assertFalse("Empty string fields should differ from no fields", Arrays.equals(hash1, hash2));
+ }
+
+ private static void assertResponse(
+ SolrQueryResponse solrQueryResponse, int droppedDocCount, int duplicateDocCount) {
+ if (droppedDocCount >= 0) {
+ assertNotNull(solrQueryResponse.getToLog().get("contentHash.duplicatesDropped"));
+ int droppedDocs = (int) solrQueryResponse.getToLog().get("contentHash.duplicatesDropped");
+ assertEquals(droppedDocCount, droppedDocs);
+ }
+ if (duplicateDocCount >= 0) {
+ assertNotNull(solrQueryResponse.getToLog().get("contentHash.duplicatesDetected"));
+ int duplicateDocs = (int) solrQueryResponse.getToLog().get("contentHash.duplicatesDetected");
+ assertEquals(duplicateDocCount, duplicateDocs);
+ }
+ }
+}
diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc
index b430c2db2c5..1368b9b5d2a 100644
--- a/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc
+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/update-request-processors.adoc
@@ -410,6 +410,12 @@ When using any of these factories, please consult the {solr-javadocs}/core/org/a
{solr-javadocs}/core/org/apache/solr/update/processor/UniqFieldsUpdateProcessorFactory.html[UniqFieldsUpdateProcessorFactory]:: Removes duplicate values found in fields matching the specified conditions.
+{solr-javadocs}/core/org/apache/solr/update/processor/ContentHashVersionProcessorFactory.html[ContentHashVersionProcessorFactory]:: Removes duplicate documents detected using a configurable hash of field values.
+[WARNING]
+====
+This URP should be configured to exclude fields that are updated with xref:indexing-guide:partial-document-updates.adoc[in-place updates] from the content hash calculation, as these fields are updated independently and should not affect duplicate detection.
+====
+
=== Update Processor Factories That Can Be Loaded as Plugins
These processors are included in Solr releases as "module", and require additional jars loaded at runtime.
diff --git a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
index 34bdcce7e6c..9841ca27077 100644
--- a/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
+++ b/solr/test-framework/src/java/org/apache/solr/SolrTestCaseJ4.java
@@ -1240,7 +1240,8 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
return out.toString();
}
- public static void addDoc(String doc, String updateRequestProcessorChain) throws Exception {
+ public static SolrQueryResponse addDoc(String doc, String updateRequestProcessorChain)
+ throws Exception {
Map<String, String[]> params = new HashMap<>();
MultiMapSolrParams mmparams = new MultiMapSolrParams(params);
params.put(UpdateParams.UPDATE_CHAIN, new String[] {updateRequestProcessorChain});
@@ -1251,8 +1252,11 @@ public abstract class SolrTestCaseJ4 extends SolrTestCase {
ArrayList<ContentStream> streams = new ArrayList<>(2);
streams.add(new ContentStreamBase.StringStream(doc));
req.setContentStreams(streams);
- handler.handleRequestBody(req, new SolrQueryResponse());
+ final SolrQueryResponse rsp = new SolrQueryResponse();
+ handler.handleRequestBody(req, rsp);
req.close();
+
+ return rsp;
}
/**