You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ds...@apache.org on 2026/07/20 18:28:32 UTC
(solr) branch branch_10x updated (e496990b613 -> 9f61d40b678)
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a change to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/solr.git
from e496990b613 Fix TestReplicationHandler.testUrlAllowListFailure
new aa4d77520bf SOLR-17231: Don't try buffering updates when there's no update log. (#3880)
new 6c252fab8d7 SOLR-18008: Remnant core files prevent Collections for working (#3904)
new e97632af65a changelog: move entry advise into template (#4111)
new 8c63c07dcae SOLR-18153: Fix File Upload in Documents Tab of Admin UI (#4204)
new b40e436cfb0 ZkTestServer: log.error -> log.warn for 'stat'
new 9f61d40b678 SOLR-18039: More tests for repeated knn parents.preFilter (#4159)
The 6 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../SOLR-18008-simulate_solr_core_remnants.yml | 9 +
changelog/unreleased/SOLR-18153.yml | 7 +
.../unreleased/{solr-18077.yml => solr-17231.yml} | 9 +-
dev-docs/changelog.adoc | 32 +--
dev-tools/scripts/validate-changelog-yaml.py | 11 +-
gradle/changelog.gradle | 30 +-
.../java/org/apache/solr/core/CoreContainer.java | 9 +-
.../apache/solr/core/CorePropertiesLocator.java | 21 ++
.../src/java/org/apache/solr/core/SolrCore.java | 9 +-
.../solr/cloud/DeleteCoreRemnantsOnCreateTest.java | 320 +++++++++++++++++++++
.../org/apache/solr/cloud/TestPullReplica.java | 22 ++
.../join/BlockJoinNestedVectorsQParserTest.java | 35 +++
.../configuration-guide/pages/core-discovery.adoc | 2 +
.../configuration-guide/pages/solr-properties.adoc | 2 +
.../pages/collection-management.adoc | 9 +
.../java/org/apache/solr/cloud/ZkTestServer.java | 2 +-
solr/webapp/web/js/angular/services.js | 6 +-
17 files changed, 495 insertions(+), 40 deletions(-)
create mode 100644 changelog/unreleased/SOLR-18008-simulate_solr_core_remnants.yml
create mode 100644 changelog/unreleased/SOLR-18153.yml
copy changelog/unreleased/{solr-18077.yml => solr-17231.yml} (54%)
create mode 100644 solr/core/src/test/org/apache/solr/cloud/DeleteCoreRemnantsOnCreateTest.java
(solr) 01/06: SOLR-17231: Don't try buffering updates when there's no update log. (#3880)
Posted by ds...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit aa4d77520bf31eb34de86c7e7235a2ef1ff0bb60
Author: Andrzej Białecki <ab...@apache.org>
AuthorDate: Mon Nov 24 12:27:52 2025 +0100
SOLR-17231: Don't try buffering updates when there's no update log. (#3880)
(cherry picked from commit 1344194099c29ff2391266c77682198a3c6f4468)
---
changelog/unreleased/solr-17231.yml | 10 ++++++++++
.../src/java/org/apache/solr/core/SolrCore.java | 9 ++++++---
.../org/apache/solr/cloud/TestPullReplica.java | 22 ++++++++++++++++++++++
3 files changed, 38 insertions(+), 3 deletions(-)
diff --git a/changelog/unreleased/solr-17231.yml b/changelog/unreleased/solr-17231.yml
new file mode 100644
index 00000000000..4f7ec2cdd08
--- /dev/null
+++ b/changelog/unreleased/solr-17231.yml
@@ -0,0 +1,10 @@
+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+title: Don't buffer updates on replicas without update log (e.g. PULL).
+type: fixed # added, changed, fixed, deprecated, removed, dependency_update, security, other
+authors:
+ - name: Andrzej Bialecki
+links:
+ - name: SOLR-17231
+ url: https://issues.apache.org/jira/browse/SOLR-17231
+issues:
+ - 17231
diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java
index 2d57377b5a2..7e9eaeb01b2 100644
--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java
+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java
@@ -83,6 +83,7 @@ import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.ResourceLoader;
import org.apache.solr.client.solrj.response.JavaBinResponseParser;
+import org.apache.solr.cloud.CloudDescriptor;
import org.apache.solr.cloud.RecoveryStrategy;
import org.apache.solr.cloud.ZkSolrResourceLoader;
import org.apache.solr.common.SolrException;
@@ -1239,10 +1240,12 @@ public class SolrCore implements SolrInfoBean, Closeable {
// ZK pre-register would have already happened so we read slice properties now
final ClusterState clusterState = coreContainer.getZkController().getClusterState();
+ final CloudDescriptor cloudDescriptor = coreDescriptor.getCloudDescriptor();
final DocCollection collection =
- clusterState.getCollection(coreDescriptor.getCloudDescriptor().getCollectionName());
- final Slice slice = collection.getSlice(coreDescriptor.getCloudDescriptor().getShardId());
- if (slice.getState() == Slice.State.CONSTRUCTION) {
+ clusterState.getCollection(cloudDescriptor.getCollectionName());
+ final Slice slice = collection.getSlice(cloudDescriptor.getShardId());
+ if (slice.getState() == Slice.State.CONSTRUCTION
+ && getUpdateHandler().getUpdateLog() != null) {
// set update log to buffer before publishing the core
getUpdateHandler().getUpdateLog().bufferUpdates();
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java b/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java
index 213b487459f..c1bece3037d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestPullReplica.java
@@ -429,6 +429,28 @@ public class TestPullReplica extends SolrCloudTestCase {
doTestNoLeader(false);
}
+ @Test
+ public void testNoBufferingInPullIfConstructing() throws Exception {
+ CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1, 0, 0)
+ .process(cluster.getSolrClient());
+ waitForState("Replica not added", collectionName, activeReplicaCount(1, 0, 0));
+
+ setSliceState(collectionName, "shard1", Slice.State.CONSTRUCTION);
+
+ CollectionAdminRequest.addReplicaToShard(collectionName, "shard1", Replica.Type.PULL)
+ .process(cluster.getSolrClient());
+ waitForState("Replica not added", collectionName, activeReplicaCount(1, 0, 1));
+ }
+
+ private void setSliceState(String collectionName, String shardId, Slice.State state)
+ throws Exception {
+ ShardTestUtil.setSliceState(cluster, collectionName, shardId, state);
+ waitForState(
+ "Expected shard " + shardId + " to be in state " + state,
+ collectionName,
+ c -> c.getSlice(shardId).getState() == state);
+ }
+
@Ignore("Ignore until I figure out a way to reliably record state transitions")
public void testPullReplicaStates() throws Exception {
// Validate that pull replicas go through the correct states when starting, stopping,
(solr) 06/06: SOLR-18039: More tests for repeated knn parents.preFilter (#4159)
Posted by ds...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit 9f61d40b67838f00d9a3c0f46d85e57c4a39e4a9
Author: Arup Chauhan <ar...@gmail.com>
AuthorDate: Sat Apr 25 23:39:16 2026 -0500
SOLR-18039: More tests for repeated knn parents.preFilter (#4159)
Add byte-path parity tests for repeated knn parents.preFilter
Signed-off-by: Arup Chauhan <ar...@gmail.com>
(cherry picked from commit f436c26fb66b1a98c738cf7cde3582678126ae59)
---
.../join/BlockJoinNestedVectorsQParserTest.java | 35 ++++++++++++++++++++++
1 file changed, 35 insertions(+)
diff --git a/solr/core/src/test/org/apache/solr/search/join/BlockJoinNestedVectorsQParserTest.java b/solr/core/src/test/org/apache/solr/search/join/BlockJoinNestedVectorsQParserTest.java
index 176aaf224c1..1243d4f7fae 100644
--- a/solr/core/src/test/org/apache/solr/search/join/BlockJoinNestedVectorsQParserTest.java
+++ b/solr/core/src/test/org/apache/solr/search/join/BlockJoinNestedVectorsQParserTest.java
@@ -294,6 +294,23 @@ public class BlockJoinNestedVectorsQParserTest extends SolrTestCaseJ4 {
"//result/doc[3]/str[@name='id'][.='2']");
}
+ @Test
+ public void parentRetrievalByte_knnChildrenWithMultipleParentFilters_shouldReturnKnnParents() {
+ assertQ(
+ req(
+ "q", "{!parent which=$allParents score=max v=$children.q}",
+ "fl", "id,score",
+ "children.q",
+ "{!knn f=vector_byte topK=3 parents.preFilter=$parentFilter1 parents.preFilter=$parentFilter2 childrenOf=$allParents}"
+ + BYTE_QUERY_VECTOR,
+ "allParents", "parent_s:[* TO *]",
+ "parentFilter1", "parent_s:(a c)",
+ "parentFilter2", "parent_s:(c e)"),
+ "//*[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='8']",
+ "//result/doc[2]/str[@name='id'][.='2']");
+ }
+
@Test
public void
parentRetrievalByte_knnChildrenWithParentFilterAndChildrenFilter_shouldReturnKnnParents() {
@@ -311,6 +328,24 @@ public class BlockJoinNestedVectorsQParserTest extends SolrTestCaseJ4 {
"//result/doc[2]/str[@name='id'][.='2']");
}
+ @Test
+ public void
+ parentRetrievalByte_knnChildrenWithMultipleParentFiltersAndChildrenFilter_shouldReturnKnnParents() {
+ assertQ(
+ req(
+ "q", "{!parent which=$allParents score=max v=$children.q}",
+ "fl", "id,score",
+ "children.q",
+ "{!knn f=vector_byte topK=3 preFilter=child_s:m parents.preFilter=$parentFilter1 parents.preFilter=$parentFilter2 childrenOf=$allParents}"
+ + BYTE_QUERY_VECTOR,
+ "allParents", "parent_s:[* TO *]",
+ "parentFilter1", "parent_s:(a c)",
+ "parentFilter2", "parent_s:(c e)"),
+ "//*[@numFound='2']",
+ "//result/doc[1]/str[@name='id'][.='8']",
+ "//result/doc[2]/str[@name='id'][.='2']");
+ }
+
@Test
public void
parentRetrievalFloat_topKWithChildTransformerWithFilter_shouldUseOriginalChildTransformerFilter() {
(solr) 03/06: changelog: move entry advise into template (#4111)
Posted by ds...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit e97632af65a44d8721fad0f49f40094a2523d3ad
Author: David Smiley <ds...@apache.org>
AuthorDate: Fri Feb 20 22:55:49 2026 -0500
changelog: move entry advise into template (#4111)
And improve/clarify advise.
Remove "security" as a type, which is orthogonal to the existing types.
Co-authored-by: Jan Høydahl <ja...@apache.org>
(cherry picked from commit 232690c9dd8c9a03bd552923c69d3fafbeade0f8)
---
dev-docs/changelog.adoc | 32 ++++++++--------------------
dev-tools/scripts/validate-changelog-yaml.py | 11 +++++++++-
gradle/changelog.gradle | 30 +++++++++++++++++++++++---
3 files changed, 46 insertions(+), 27 deletions(-)
diff --git a/dev-docs/changelog.adoc b/dev-docs/changelog.adoc
index a9f671c544a..d8b767c81d7 100644
--- a/dev-docs/changelog.adoc
+++ b/dev-docs/changelog.adoc
@@ -113,25 +113,11 @@ user.githubid=johngithub
user.asfid=johnapache
----
-TIP: Aliases for `writeChangelog` task are `changelog` and `newChangelog`.
-
-== 4. Writing Good Entries
-
-* **Audience** is end-users and administrators, not committers.
-* If the change is super minor, like a typo, don't bother adding a yaml file
-* Keep the entry short and focused on the user impact.
-* Choose the correct *type*:
-** `added` for new features
-** `changed` for improvements to existing code
-** `fixed` for bug fixes
-** `deprecated` for deprecated features
-** `removed` for code removed in major releases
-** `dependency_update` for updates to dependencies
-** `security` for security-related changes
-** `other` for anything else, like the build or documentation
-* Reference issues as `SOLR-12345` or GitHub `PR#123`.
-
-== 5. Changelog Validation in Pull Requests
+The changelog has comments to help you fill it out. Remove them after filling out the form.
+
+TIP: Aliases for the `writeChangelog` task are `changelog` and `newChangelog`.
+
+== 4. Changelog Validation in Pull Requests
The `validate-changelog` GitHub workflow automatically checks that:
@@ -140,9 +126,9 @@ The `validate-changelog` GitHub workflow automatically checks that:
If your change does not require a changelog entry, it is still possible to merge the PR.
-== 6. For release managers
+== 5. For release managers
-=== 6.1 Gradle tasks for logchange
+=== 5.1 Gradle tasks for logchange
The logchange gradle plugin offers some tasks, here are the two most important:
@@ -164,7 +150,7 @@ The `logchangeRelease` and `logchangeGenerate` tasks are used by ReleaseWizard.
These are integrated in the Release Wizard via `dev-tools/scripts/logchange.py`, which also handles RC2+ scenarios — see section 5.4.
-=== 6.2 Migration tool
+=== 5.2 Migration tool
There is a migration tool in `dev-tools/scripts/changes2logchange.py` for one-time use during the transition.
It will bulk convert the entire `solr/CHANGES.txt` file to files in the `changelog/` folder and versioned sub folders.
@@ -195,7 +181,7 @@ links:
url: https://issues.apache.org/jira/browse/SOLR-17960
----
-=== 6.3 Changelog validation tool
+=== 5.3 Changelog validation tool
There is a tool `dev-tools/scripts/validateChangelogs.py` that will do a
cross-branch validation of changelog folders. It takes no arguments and
diff --git a/dev-tools/scripts/validate-changelog-yaml.py b/dev-tools/scripts/validate-changelog-yaml.py
index 0a41586c781..fa7369ebd4c 100644
--- a/dev-tools/scripts/validate-changelog-yaml.py
+++ b/dev-tools/scripts/validate-changelog-yaml.py
@@ -30,6 +30,7 @@ Checks:
- Contains required 'type' field (one of: added, changed, fixed, deprecated, removed, dependency_update, security, other)
- Contains required 'authors' field with at least one author
- Each author has a 'name' field (non-empty string)
+- Comment block is removed
"""
import sys
@@ -40,10 +41,12 @@ import yaml
def validate_changelog_yaml(file_path):
"""Validate a changelog YAML file. Returns True if valid."""
valid_types = ['added', 'changed', 'fixed', 'deprecated', 'removed', 'dependency_update', 'security', 'other']
+ not_allowed_text = ['DELETE ALL COMMENTS UP HERE', 'Most such changes are too small']
try:
with open(file_path, 'r', encoding='utf-8') as f:
- data = yaml.safe_load(f)
+ raw_content = f.read()
+ data = yaml.safe_load(raw_content)
# Check if file contains a mapping (dictionary)
if not isinstance(data, dict):
@@ -86,6 +89,12 @@ def validate_changelog_yaml(file_path):
print(f"::error file={file_path}::Author {i} 'name' must be a non-empty string")
return False
+ # Validate that comments are removed
+ for not_allowed in not_allowed_text:
+ if not_allowed in raw_content:
+ print(f"::error file={file_path}::File still contains commented template text. Please remove the comment block at the top of the file.")
+ return False
+
# All validations passed
print(f"✓ {file_path} is valid")
print(f" Title: {data['title']}")
diff --git a/gradle/changelog.gradle b/gradle/changelog.gradle
index 661c8cfb54f..69d2d6e87f7 100644
--- a/gradle/changelog.gradle
+++ b/gradle/changelog.gradle
@@ -50,15 +50,39 @@ task writeChangelog {
def fileName = "changelog/unreleased/${sanitizedBranchName}.yml"
def file = new File(fileName)
file.parentFile.mkdirs()
- file.text = """# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+ file.text = """# (DELETE ALL COMMENTS UP HERE AFTER FILLING THIS IN
+
+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+
+# If the change is minor, don't bother adding a changelog entry.
+# For `other` type entries, the threshold to bother with a changelog entry should be even higher.
+
+# title:
+# * The audience is end-users and administrators, not committers.
+# * Be short and focused on the user impact. Multiple sentences is fine!
+# * For technical/geeky details, prefer the commit message instead of changelog.
+# * Reference JIRA issues like `SOLR-12345`, or if no JIRA but have a GitHub PR then `PR#12345`.
+
+# type:
+# `added` for new features/improvements, opt-in by the user typically documented in the ref guide
+# `changed` for improvements; not opt-in
+# `fixed` for improvements that are deemed to have fixed buggy behavior
+# `deprecated` for marking things deprecated
+# `removed` for code removed
+# `dependency_update` for updates to dependencies
+# `other` for anything else, like large/significant refactorings, build changes,
+# test infrastructure, or documentation.
+# Most such changes are too small/minor to bother with a changelog entry.
+
title: ${title}
-type: other # added, changed, fixed, deprecated, removed, dependency_update, security, other
+type:
authors:
- name: ${configuredName}${nick}${asfIdUrl}
${jiraLinks}
"""
- println "Generated file: ${fileName} -- open it"
+ println "Generated file: ${fileName} -- open and edit before committing"
+ println "Read dev-docs/changelog.adoc if you don't contribute here often."
}
}
(solr) 05/06: ZkTestServer: log.error -> log.warn for 'stat'
Posted by ds...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit b40e436cfb04ef3bb70ec265d196ec2754d94a29
Author: David Smiley <ds...@apache.org>
AuthorDate: Thu Mar 12 22:10:24 2026 -0400
ZkTestServer: log.error -> log.warn for 'stat'
Wasn't error-worthy
(cherry picked from commit 7b0208db9bb3fdabc7c68eda6dd7c4df5be96960)
---
solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java b/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java
index 373a2c8eb4d..8be5bec00db 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java
@@ -732,7 +732,7 @@ public class ZkTestServer {
if (!FourLetterCommands.isEnabled(stat)) {
final String original = System.getProperty(ZK_WHITELIST_PROPERTY);
try {
- log.error(
+ log.warn(
"ZkTestServer requires the 'stat' command, temporarily manipulating your whitelist");
System.setProperty(ZK_WHITELIST_PROPERTY, "*");
FourLetterCommands.resetWhiteList();
(solr) 04/06: SOLR-18153: Fix File Upload in Documents Tab of Admin UI (#4204)
Posted by ds...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit 8c63c07dcaee324913299385d8d621987919df51
Author: Eric Pugh <ep...@opensourceconnections.com>
AuthorDate: Thu Mar 12 15:27:58 2026 -0400
SOLR-18153: Fix File Upload in Documents Tab of Admin UI (#4204)
(cherry picked from commit 4abc924e0ce9c0638a41e8c46bd231b6d4f5eafd)
---
changelog/unreleased/SOLR-18153.yml | 7 +++++++
solr/webapp/web/js/angular/services.js | 6 +++++-
2 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/changelog/unreleased/SOLR-18153.yml b/changelog/unreleased/SOLR-18153.yml
new file mode 100644
index 00000000000..536c6b47f2d
--- /dev/null
+++ b/changelog/unreleased/SOLR-18153.yml
@@ -0,0 +1,7 @@
+title: Fixed file upload in the Documents tab of Admin UI.
+type: fixed
+authors:
+ - name: Eric Pugh
+links:
+ - name: SOLR-18153
+ url: https://issues.apache.org/jira/browse/SOLR-18153
diff --git a/solr/webapp/web/js/angular/services.js b/solr/webapp/web/js/angular/services.js
index 9c6ea9fb799..98e7e37d9ba 100644
--- a/solr/webapp/web/js/angular/services.js
+++ b/solr/webapp/web/js/angular/services.js
@@ -166,7 +166,11 @@ solrAdminServices.factory('System',
$http.post(url, fd, {
transformRequest: angular.identity,
headers: {'Content-Type': undefined}
- }).success(success).error(error);
+ }).then(function(response) {
+ success(response.data);
+ }, function(response) {
+ error(response.data);
+ });
}
})
.filter('splitByComma', function() {
(solr) 02/06: SOLR-18008: Remnant core files prevent Collections for working (#3904)
Posted by ds...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dsmiley pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/solr.git
commit 6c252fab8d7b167a15f93b9cd903415e767bda39
Author: Eric Pugh <ep...@opensourceconnections.com>
AuthorDate: Mon Jan 19 17:59:30 2026 -0500
SOLR-18008: Remnant core files prevent Collections for working (#3904)
Add solr.cloud.delete.unknown.cores.enabled setting for removing unknown but existing core data when a core is created in SolrCloud mode, either during a new collection or when adding replicas. This shouldn't happen, but it does occasionally, so this is documented as an expert level feature for those experiencing it.
(cherry picked from commit 2b70060ac3ed60785e7a660665bd835e3b5ac584)
---
.../SOLR-18008-simulate_solr_core_remnants.yml | 9 +
.../java/org/apache/solr/core/CoreContainer.java | 9 +-
.../apache/solr/core/CorePropertiesLocator.java | 21 ++
.../solr/cloud/DeleteCoreRemnantsOnCreateTest.java | 320 +++++++++++++++++++++
.../configuration-guide/pages/core-discovery.adoc | 2 +
.../configuration-guide/pages/solr-properties.adoc | 2 +
.../pages/collection-management.adoc | 9 +
7 files changed, 368 insertions(+), 4 deletions(-)
diff --git a/changelog/unreleased/SOLR-18008-simulate_solr_core_remnants.yml b/changelog/unreleased/SOLR-18008-simulate_solr_core_remnants.yml
new file mode 100644
index 00000000000..45d804cccd6
--- /dev/null
+++ b/changelog/unreleased/SOLR-18008-simulate_solr_core_remnants.yml
@@ -0,0 +1,9 @@
+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+title: Add solr.cloud.delete.unknown.cores.enabled setting for removing unknown but existing core data when a core is created in SolrCloud mode.
+type: changed # added, changed, fixed, deprecated, removed, dependency_update, security, other
+authors:
+ - name: Eric Pugh
+ - name: David Smiley
+links:
+- name: SOLR-18008
+ url: https://issues.apache.org/jira/browse/SOLR-18008
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index d7c67ecf3af..186e319eae8 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -94,6 +94,7 @@ import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.util.CollectionUtil;
+import org.apache.solr.common.util.EnvUtils;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.common.util.ObjectCache;
@@ -1688,11 +1689,11 @@ public class CoreContainer {
} catch (Exception e) {
coreInitFailures.put(dcore.getName(), new CoreLoadFailure(dcore, e));
if (e instanceof ZkController.NotInClusterStateException && !newCollection) {
- // this mostly happens when the core is deleted when this node is down
+ // this mostly happens when the core is deleted when this node is down,
// but it can also happen if connecting to the wrong zookeeper
final boolean deleteUnknownCores =
- Boolean.parseBoolean(
- System.getProperty("solr.cloud.startup.delete.unknown.cores.enabled", "false"));
+ EnvUtils.getPropertyAsBool("solr.cloud.startup.delete.unknown.cores.enabled", false);
+
log.error(
"SolrCore {} in {} is not in cluster state.{}",
dcore.getName(),
@@ -1700,7 +1701,7 @@ public class CoreContainer {
(deleteUnknownCores
? " It will be deleted. See SOLR-13396 for more information."
: ""));
- // We alreday have an ongoing CoreOp, so do not wait to start another one
+ // We already have an ongoing CoreOp, so do not wait to start another one
unloadWithoutCoreOp(
dcore.getName(), deleteUnknownCores, deleteUnknownCores, deleteUnknownCores);
throw e;
diff --git a/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java b/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
index c060fc5d3cd..2fe356dfca8 100644
--- a/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
+++ b/solr/core/src/java/org/apache/solr/core/CorePropertiesLocator.java
@@ -39,6 +39,7 @@ import java.util.Properties;
import java.util.Set;
import java.util.stream.Collectors;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.util.EnvUtils;
import org.apache.solr.util.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,6 +66,7 @@ public class CorePropertiesLocator implements CoresLocator {
@Override
public void create(CoreContainer cc, CoreDescriptor... coreDescriptors) {
for (CoreDescriptor cd : coreDescriptors) {
+ checkForExistingCore(cd);
Path propertiesFile = cd.getInstanceDir().resolve(PROPERTIES_FILENAME);
if (Files.exists(propertiesFile))
throw new SolrException(
@@ -240,4 +242,23 @@ public class CorePropertiesLocator implements CoresLocator {
p.putAll(cd.getPersistableUserProperties());
return p;
}
+
+ protected void checkForExistingCore(CoreDescriptor cd) {
+ if (cd.getCloudDescriptor() != null && Files.exists(cd.getInstanceDir())) {
+ final boolean deleteUnknownCores =
+ EnvUtils.getPropertyAsBool("solr.cloud.delete.unknown.cores.enabled", false);
+ if (deleteUnknownCores) {
+ log.warn(
+ "Automatically deleting existing directory at [{}] for core [{}] because solr.cloud.delete.unknown.cores.enabled is true",
+ cd.getInstanceDir().toAbsolutePath(),
+ cd.getName());
+ SolrCore.deleteUnloadedCore(cd, true, true);
+ } else {
+ log.warn(
+ "Directory at [{}] for core[{}] already exists may prevent create operation. Set solr.cloud.delete.unknown.cores.enabled=true to delete directory. (SOLR-18008)",
+ cd.getInstanceDir().toAbsolutePath(),
+ cd.getName());
+ }
+ }
+ }
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteCoreRemnantsOnCreateTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteCoreRemnantsOnCreateTest.java
new file mode 100644
index 00000000000..57cea18d24e
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/DeleteCoreRemnantsOnCreateTest.java
@@ -0,0 +1,320 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cloud;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.Optional;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.CoreAdminRequest;
+import org.apache.solr.client.solrj.response.json.JsonMapResponseParser;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.embedded.JettySolrRunner;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test "solr.cloud.delete.unknown.cores.enabled" property that can be used if Solr has an
+ * inconsistent state with its cores lifecycle where remnant files are left on disk after various
+ * operations that delete a core.
+ */
+public class DeleteCoreRemnantsOnCreateTest extends SolrCloudTestCase {
+ private static final String DELETE_UNKNOWN_CORES_PROP = "solr.cloud.delete.unknown.cores.enabled";
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(1).addConfig("conf", configset("cloud-minimal")).configure();
+ }
+
+ /**
+ * Shared setup for testing collection creation with remnants. Creates a collection, deletes it,
+ * and then leaves behind a remnant directory.
+ */
+ private String setupCollectionRemnant(String collectionName) throws Exception {
+ List<JettySolrRunner> jettys = cluster.getJettySolrRunners();
+ String primaryNode = jettys.getFirst().getNodeName();
+
+ CollectionAdminRequest.Create createRequest =
+ CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1);
+ createRequest.process(cluster.getSolrClient());
+
+ waitForState(
+ "Expected collection to be fully active",
+ collectionName,
+ (n, c) -> SolrCloudTestCase.replicasForCollectionAreFullyActive(n, c, 1, 1));
+
+ Replica primaryReplica = getReplicaOnNode(collectionName, "shard1", primaryNode);
+ JettySolrRunner primaryJetty = cluster.getReplicaJetty(primaryReplica);
+ String originalCoreName = primaryReplica.getCoreName();
+ Path remnantInstanceDir;
+ try (SolrCore core = primaryJetty.getCoreContainer().getCore(originalCoreName)) {
+ CoreDescriptor cd = core.getCoreDescriptor();
+ remnantInstanceDir = cd.getInstanceDir();
+ }
+
+ CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());
+ waitForState("Expected collection deletion", collectionName, (n, c) -> c == null);
+
+ // Simulate a core remnant still exists by creating the directory and core.properties
+ Files.createDirectories(remnantInstanceDir);
+ String propertiesContent = "";
+ Files.writeString(
+ remnantInstanceDir.resolve("core.properties"), propertiesContent, StandardCharsets.UTF_8);
+
+ return originalCoreName;
+ }
+
+ /**
+ * Shared setup for testing replica addition with remnants. Creates a collection, then simulates a
+ * remnant directory on the single node that will impact the next addReplica command.
+ */
+ private void setupReplicaRemnant(String collectionName) throws Exception {
+ List<JettySolrRunner> jettys = cluster.getJettySolrRunners();
+ String primaryNode = jettys.getFirst().getNodeName();
+
+ CollectionAdminRequest.Create createRequest =
+ CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1);
+ createRequest.process(cluster.getSolrClient());
+
+ waitForState(
+ "Expected collection to be fully active",
+ collectionName,
+ (n, c) -> SolrCloudTestCase.replicasForCollectionAreFullyActive(n, c, 1, 1));
+
+ int nextReplicaIndex = 3; // Yep, from 1 to 3 due to how we count in ZK and setup.
+ String expectedNewReplicaName = collectionName + "_shard1_replica_n" + nextReplicaIndex;
+
+ // Simulate a core remnant on the single node adjacent to the existing replica instance path
+ Replica existing = getReplicaOnNode(collectionName, "shard1", primaryNode);
+ try (SolrCore core =
+ cluster.getReplicaJetty(existing).getCoreContainer().getCore(existing.getCoreName())) {
+ Path siblingDir = core.getInstancePath().getParent().resolve(expectedNewReplicaName);
+ Files.createDirectories(siblingDir);
+ Files.writeString(
+ siblingDir.resolve("core.properties"),
+ "name="
+ + expectedNewReplicaName
+ + "_remnant\n"
+ + "collection="
+ + collectionName
+ + "_remnant\n"
+ + "shard=shard1\n"
+ + "coreNodeName=core_node_remnant\n",
+ StandardCharsets.UTF_8);
+ }
+ }
+
+ /**
+ * Shared setup for testing DeleteCore admin API with remnants. Creates a collection, deletes it,
+ * and then leaves behind a remnant core directory.
+ */
+ private String setupCoreRemnantForUnloadCoreOperation(String collectionName) throws Exception {
+ List<JettySolrRunner> jettys = cluster.getJettySolrRunners();
+ String primaryNode = jettys.getFirst().getNodeName();
+
+ CollectionAdminRequest.Create createRequest =
+ CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1);
+ createRequest.process(cluster.getSolrClient());
+
+ waitForState(
+ "Expected collection to be fully active",
+ collectionName,
+ (n, c) -> SolrCloudTestCase.replicasForCollectionAreFullyActive(n, c, 1, 1));
+
+ Replica primaryReplica = getReplicaOnNode(collectionName, "shard1", primaryNode);
+ JettySolrRunner primaryJetty = cluster.getReplicaJetty(primaryReplica);
+ String originalCoreName = primaryReplica.getCoreName();
+ Path remnantInstanceDir;
+ try (SolrCore core = primaryJetty.getCoreContainer().getCore(originalCoreName)) {
+ CoreDescriptor cd = core.getCoreDescriptor();
+ remnantInstanceDir = cd.getInstanceDir();
+ }
+
+ CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());
+ waitForState("Expected collection deletion", collectionName, (n, c) -> c == null);
+
+ // Simulate a core remnant still exists by creating the directory and core.properties
+ Files.createDirectories(remnantInstanceDir);
+ Files.writeString(
+ remnantInstanceDir.resolve("core.properties"),
+ "name=" + originalCoreName + "\n",
+ StandardCharsets.UTF_8);
+
+ return originalCoreName;
+ }
+
+ @Test
+ public void testCreateCollectionWithRemnantsFailsWithoutSetting() throws Exception {
+ assertNull(
+ "Property should not be set by default", System.getProperty(DELETE_UNKNOWN_CORES_PROP));
+
+ String collectionName = "coreRemnantCreateNoSetting";
+ setupCollectionRemnant(collectionName);
+
+ // Try to create the collection again - this demonstrates the behavior without the setting
+ // In typical environments, this might fail, but behavior depends on configuration
+ CollectionAdminRequest.Create recreateRequest =
+ CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1);
+
+ // The request to create a collection SHOULD fail based on the remnant file, if it does not it
+ // means we've changed Solr's behavior when creating a core and
+ // remnants exist, and therefore we should rethink the utility of this setting.
+ Exception e =
+ assertThrows(
+ "This request to recreate the collection should have failed due to remnant files.",
+ Exception.class,
+ () -> recreateRequest.process(cluster.getSolrClient()));
+
+ assertTrue(
+ "Verify the exception was due to core creation failed.",
+ e.getMessage().contains("Underlying core creation failed"));
+ }
+
+ @Test
+ public void testCreateCollectionWithRemnantsWithSetting() throws Exception {
+ System.setProperty(DELETE_UNKNOWN_CORES_PROP, "true");
+
+ String collectionName = "coreRemnantCreateWithSetting";
+ setupCollectionRemnant(collectionName);
+
+ // With the setting enabled, collection creation should succeed despite remnants
+ CollectionAdminRequest.Create recreateRequest =
+ CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1);
+ List<JettySolrRunner> jettys = cluster.getJettySolrRunners();
+ recreateRequest.process(cluster.getSolrClient());
+
+ waitForState(
+ "Expected recreated collection to be fully active",
+ collectionName,
+ (n, c) -> SolrCloudTestCase.replicasForCollectionAreFullyActive(n, c, 1, 1));
+
+ // Verify collection was created successfully
+ DocCollection collection = getCollectionState(collectionName);
+ assertNotNull("Collection should exist", collection);
+ assertEquals("Should have 1 replica", 1, collection.getReplicas().size());
+
+ // Verify replica on the node where we had the remnant is active
+ Replica recreatedReplica =
+ getReplicaOnNode(collectionName, "shard1", jettys.getFirst().getNodeName());
+ assertNotNull("Should have a replica on the primary node", recreatedReplica);
+ assertEquals("Replica should be active", Replica.State.ACTIVE, recreatedReplica.getState());
+ }
+
+ @Test
+ public void testAddReplicaWithRemnantFailsWithoutSetting() throws Exception {
+ assertNull(
+ "Property should not be set by default", System.getProperty(DELETE_UNKNOWN_CORES_PROP));
+
+ String collectionName = "coreRemnantAddReplicaNoSetting";
+ setupReplicaRemnant(collectionName);
+
+ List<JettySolrRunner> jettys = cluster.getJettySolrRunners();
+ String primaryNode = jettys.getFirst().getNodeName();
+
+ // Try to add a new replica - this demonstrates the behavior without the setting
+ CollectionAdminRequest.AddReplica addReplicaRequest =
+ CollectionAdminRequest.addReplicaToShard(collectionName, "shard1");
+ addReplicaRequest.setNode(primaryNode);
+
+ Exception e =
+ assertThrows(
+ "This request to add a replica to the collection should have failed due to remnant files.",
+ Exception.class,
+ () -> addReplicaRequest.process(cluster.getSolrClient()));
+
+ assertTrue(
+ "Verify the exception was due to core creation failed.",
+ e.getMessage().contains("ADDREPLICA failed to create replica"));
+ }
+
+ @Test
+ public void testAddReplicaWithRemnantWithSetting() throws Exception {
+ System.setProperty(DELETE_UNKNOWN_CORES_PROP, "true");
+
+ String collectionName = "coreRemnantAddReplicaWithSetting";
+ setupReplicaRemnant(collectionName);
+
+ List<JettySolrRunner> jettys = cluster.getJettySolrRunners();
+ String primaryNode = jettys.getFirst().getNodeName();
+
+ // With the setting enabled, replica addition should succeed despite remnants
+ CollectionAdminRequest.AddReplica addReplicaRequest =
+ CollectionAdminRequest.addReplicaToShard(collectionName, "shard1");
+ addReplicaRequest.setNode(primaryNode);
+ addReplicaRequest.process(cluster.getSolrClient());
+
+ waitForState(
+ "Expected replica addition to finish",
+ collectionName,
+ (n, c) -> SolrCloudTestCase.replicasForCollectionAreFullyActive(n, c, 1, 2));
+
+ // Verify collection now has 2 replicas
+ DocCollection collection = getCollectionState(collectionName);
+ assertNotNull("Collection should exist", collection);
+ assertEquals("Should have 2 replicas after adding", 2, collection.getReplicas().size());
+
+ // Verify the replica was added on the single node and is active
+ Replica addedReplica = getReplicaOnNode(collectionName, "shard1", primaryNode);
+ assertNotNull("Should have added a replica on the primary node", addedReplica);
+ assertEquals("Added replica should be active", Replica.State.ACTIVE, addedReplica.getState());
+ }
+
+ /**
+ * This test demonstrates that you can't call the direct core unload admin operation to get rid of
+ * a remnant core, not because of the existence of the remnant, but because the core no longer has
+ * a CoreDescriptor record in ZooKeeper.
+ */
+ @Test
+ public void testDeleteCoreFailsWhenUnknown() throws Exception {
+
+ String collectionName = "coreRemnantDelete";
+ String coreName = setupCollectionRemnant(collectionName);
+
+ // Try to delete a core that only exists as a remnant - and has no record in ZooKeeper
+ CoreAdminRequest.Unload unloadRequest = new CoreAdminRequest.Unload(true);
+ unloadRequest.setDeleteIndex(true);
+ unloadRequest.setDeleteDataDir(true);
+ unloadRequest.setDeleteInstanceDir(true);
+ unloadRequest.setCoreName(coreName);
+ unloadRequest.setResponseParser(new JsonMapResponseParser());
+
+ Exception e =
+ assertThrows(
+ "Expected request to fail.",
+ Exception.class,
+ () -> cluster.getSolrClient().request(unloadRequest));
+
+ assertTrue(
+ "Verify the exception was due to ZK not knowing about the core existence.",
+ e.getMessage().contains("Cannot unload non-existent core [" + coreName + "]"));
+ }
+
+ private Replica getReplicaOnNode(String collectionName, String shard, String nodeName) {
+ DocCollection collectionState = getCollectionState(collectionName);
+ Slice slice = collectionState.getSlice(shard);
+ Optional<Replica> replica =
+ slice.getReplicas().stream().filter(r -> nodeName.equals(r.getNodeName())).findFirst();
+ return replica.orElseThrow(
+ () -> new AssertionError("No replica found on node " + nodeName + " for " + shard));
+ }
+}
diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/core-discovery.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/core-discovery.adoc
index 04ce53ad47e..f1d89ad6c38 100644
--- a/solr/solr-ref-guide/modules/configuration-guide/pages/core-discovery.adoc
+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/core-discovery.adoc
@@ -18,6 +18,8 @@
Core discovery means that creating a core is as simple as a `core.properties` file located on disk.
+TIP: If you are running in SolrCloud mode, you are shielded from this complexity.
+
== The core.properties File
In Solr, the term _core_ is used to refer to a single index and associated transaction log and configuration files (including the `solrconfig.xml` and schema files, among others).
diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/solr-properties.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/solr-properties.adoc
index dab7bdd5b69..b9ecaddb100 100644
--- a/solr/solr-ref-guide/modules/configuration-guide/pages/solr-properties.adoc
+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/solr-properties.adoc
@@ -44,6 +44,8 @@ NOTE: Properties marked with "!" indicate inverted meaning between pre Solr 10 a
|solr.cloud.startup.delete.unknown.cores.enabled|solr.deleteUnknownCores|false|Controls whether unknown cores are deleted at startup in SolrCloud mode.
+|solr.cloud.delete.unknown.cores.enabled| |false|Controls deletion of preexisting core data when a core is created in SolrCloud mode.
+
|solr.cloud.wait.for.zk.seconds|waitForZk|30|Specifies the number of seconds to wait for ZooKeeper connection in SolrCloud mode.
|solr.configset.bootstrap.confdir|bootstrap_confdir||Defines the directory of a configuration to be loaded as a configset when starting Solr. Often ends in `/conf`.
diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc
index f34f700e814..46452619f7c 100644
--- a/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc
+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/collection-management.adoc
@@ -292,6 +292,15 @@ Request ID to track this action which will be xref:configuration-guide:collectio
Collections are first created in read-write mode but can be put in `readOnly`
mode using the xref:collection-management.adoc#modifycollection[MODIFYCOLLECTION] action.
+Solr can occasionally enter an inconsistent state where remnant core files remain on disk after previous collection deletion operations fail.
+
+The system property `solr.cloud.delete.unknown.cores.enabled` is an expert-level setting designed to handle this situation.
+When enabled, Solr automatically deletes any remnant core data on startup that lacks a record in ZooKeeper.
+During core creation (whether for new collections or replicas), Solr also deletes preexisting remnant files, allowing operations to complete.
+Without this setting, these files would cause new core creation to fail.
+
+Enable this feature with caution— it indicates an underlying problem in your Solr setup that should be investigated.
+
=== CREATE Response
The response will include the status of the request and the new core names.