You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by md...@apache.org on 2020/08/21 20:22:17 UTC
[lucene-solr] branch branch_8x updated: SOLR-13438: on collection
delete, also delete .AUTOCREATED config set (#1759)
This is an automated email from the ASF dual-hosted git repository.
mdrob pushed a commit to branch branch_8x
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/branch_8x by this push:
new fc5e07a SOLR-13438: on collection delete, also delete .AUTOCREATED config set (#1759)
fc5e07a is described below
commit fc5e07ac38520162aa5af394f95978b0478cda8b
Author: Anderson Dorow <an...@gmail.com>
AuthorDate: Fri Aug 21 21:43:16 2020 +0200
SOLR-13438: on collection delete, also delete .AUTOCREATED config set (#1759)
Co-authored-by: Anderson Dorow <An...@zooplus.com>
---
solr/CHANGES.txt | 2 +
.../cloud/api/collections/DeleteCollectionCmd.java | 28 +++
.../solr/handler/admin/ConfigSetsHandlerApi.java | 4 +
.../SimpleCollectionCreateDeleteTest.java | 201 ++++++++++++++-------
4 files changed, 172 insertions(+), 63 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 08c0a5e..611459b 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -58,6 +58,8 @@ Optimizations
* SOLR-13132: JSON Facet perf improvements to support "sweeping" collection of "relatedness()"
(hossman, Michael Gibney)
+* SOLR-13438: On deleting a collection, its config set will also be deleted iff it has been auto-created, and not used by any other collection (Anderson Dorow)
+
Bug Fixes
---------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
index 70d8d2b..352d9e9 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/DeleteCollectionCmd.java
@@ -45,6 +45,7 @@ import org.apache.solr.common.util.TimeSource;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.SolrInfoBean;
import org.apache.solr.core.snapshots.SolrSnapshotManager;
+import org.apache.solr.handler.admin.ConfigSetsHandlerApi;
import org.apache.solr.handler.admin.MetricsHistoryHandler;
import org.apache.solr.metrics.SolrMetricManager;
import org.apache.zookeeper.KeeperException;
@@ -160,6 +161,33 @@ public class DeleteCollectionCmd implements OverseerCollectionMessageHandler.Cmd
});
}
+ // delete related config set iff: it is auto generated AND not related to any other collection
+ String configSetName = zkStateReader.readConfigName(collection);
+
+ if (ConfigSetsHandlerApi.isAutoGeneratedConfigSet(configSetName)) {
+ boolean configSetIsUsedByOtherCollection = false;
+
+ // make sure the configSet is not shared with other collections
+ // Similar to what happens in: OverseerConfigSetMessageHandler::deleteConfigSet
+ for (Map.Entry<String, DocCollection> entry : zkStateReader.getClusterState().getCollectionsMap().entrySet()) {
+ String otherConfigSetName = null;
+ try {
+ otherConfigSetName = zkStateReader.readConfigName(entry.getKey());
+ } catch (KeeperException ex) {
+ // ignore 'no config found' errors
+ }
+ if (configSetName.equals(otherConfigSetName)) {
+ configSetIsUsedByOtherCollection = true;
+ break;
+ }
+ }
+
+ if (!configSetIsUsedByOtherCollection) {
+ // delete the config set
+ zkStateReader.getConfigManager().deleteConfigDir(configSetName);
+ }
+ }
+
// TimeOut timeout = new TimeOut(60, TimeUnit.SECONDS, timeSource);
// boolean removed = false;
// while (! timeout.hasTimedOut()) {
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandlerApi.java b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandlerApi.java
index 1a5f6f3..30450a8 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandlerApi.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandlerApi.java
@@ -42,6 +42,10 @@ public class ConfigSetsHandlerApi extends BaseHandlerApiSupport {
return configName + AUTOCREATED_CONFIGSET_SUFFIX;
}
+ public static boolean isAutoGeneratedConfigSet(String configName) {
+ return configName != null && configName.endsWith(AUTOCREATED_CONFIGSET_SUFFIX);
+ }
+
private static Collection<ApiCommand> createMapping() {
Map<ConfigSetMeta, ApiCommand> result = new EnumMap<>(ConfigSetMeta.class);
diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/SimpleCollectionCreateDeleteTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/SimpleCollectionCreateDeleteTest.java
index 4270cfe..9a8b865 100644
--- a/solr/core/src/test/org/apache/solr/cloud/api/collections/SimpleCollectionCreateDeleteTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/SimpleCollectionCreateDeleteTest.java
@@ -16,10 +16,6 @@
*/
package org.apache.solr.cloud.api.collections;
-import java.util.Collection;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
@@ -32,71 +28,150 @@ import org.apache.solr.core.SolrCore;
import org.apache.solr.util.TimeOut;
import org.junit.Test;
+import java.util.Collection;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
public class SimpleCollectionCreateDeleteTest extends AbstractFullDistribZkTestBase {
- public SimpleCollectionCreateDeleteTest() {
- sliceCount = 1;
- }
-
- @Test
- @ShardsFixed(num = 1)
- public void test() throws Exception {
- String overseerNode = OverseerCollectionConfigSetProcessor.getLeaderNode(cloudClient.getZkStateReader().getZkClient());
- String notOverseerNode = null;
- for (CloudJettyRunner cloudJetty : cloudJettys) {
- if (!overseerNode.equals(cloudJetty.nodeName)) {
- notOverseerNode = cloudJetty.nodeName;
- break;
- }
+ public SimpleCollectionCreateDeleteTest() {
+ sliceCount = 1;
}
- String collectionName = "SimpleCollectionCreateDeleteTest";
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,1,1)
- .setCreateNodeSet(overseerNode)
- .setStateFormat(2);
-
- NamedList<Object> request = create.process(cloudClient).getResponse();
-
- if (request.get("success") != null) {
- assertTrue(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, false));
-
- @SuppressWarnings({"rawtypes"})
- CollectionAdminRequest delete = CollectionAdminRequest.deleteCollection(collectionName);
- cloudClient.request(delete);
-
- assertFalse(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, false));
-
- // currently, removing a collection does not wait for cores to be unloaded
- TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- while (true) {
-
- if( timeout.hasTimedOut() ) {
- throw new TimeoutException("Timed out waiting for all collections to be fully removed.");
+
+ @Test
+ @ShardsFixed(num = 1)
+ public void testCreateAndDeleteThenCreateAgain() throws Exception {
+ String overseerNode = OverseerCollectionConfigSetProcessor.getLeaderNode(cloudClient.getZkStateReader().getZkClient());
+ String notOverseerNode = null;
+ for (CloudJettyRunner cloudJetty : cloudJettys) {
+ if (!overseerNode.equals(cloudJetty.nodeName)) {
+ notOverseerNode = cloudJetty.nodeName;
+ break;
+ }
}
-
- boolean allContainersEmpty = true;
- for(JettySolrRunner jetty : jettys) {
-
- Collection<SolrCore> cores = jetty.getCoreContainer().getCores();
- for (SolrCore core : cores) {
- CoreDescriptor cd = core.getCoreDescriptor();
- if (cd != null) {
- if (cd.getCloudDescriptor().getCollectionName().equals(collectionName)) {
- allContainersEmpty = false;
- }
+ String collectionName = "SimpleCollectionCreateDeleteTest";
+ CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, 1, 1)
+ .setCreateNodeSet(overseerNode);
+
+ NamedList<Object> request = create.process(cloudClient).getResponse();
+
+ if (request.get("success") != null) {
+ assertTrue(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, false));
+
+ CollectionAdminRequest.Delete delete = CollectionAdminRequest.deleteCollection(collectionName);
+ cloudClient.request(delete);
+
+ assertFalse(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, false));
+
+ // currently, removing a collection does not wait for cores to be unloaded
+ TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
+ while (true) {
+
+ if (timeout.hasTimedOut()) {
+ throw new TimeoutException("Timed out waiting for all collections to be fully removed.");
+ }
+
+ boolean allContainersEmpty = true;
+ for (JettySolrRunner jetty : jettys) {
+
+ Collection<SolrCore> cores = jetty.getCoreContainer().getCores();
+ for (SolrCore core : cores) {
+ CoreDescriptor cd = core.getCoreDescriptor();
+ if (cd != null) {
+ if (cd.getCloudDescriptor().getCollectionName().equals(collectionName)) {
+ allContainersEmpty = false;
+ }
+ }
+ }
+ }
+ if (allContainersEmpty) {
+ break;
+ }
}
- }
+
+ // create collection again on a node other than the overseer leader
+ create = CollectionAdminRequest.createCollection(collectionName, 1, 1)
+ .setCreateNodeSet(notOverseerNode);
+ request = create.process(cloudClient).getResponse();
+ assertTrue("Collection creation should not have failed", request.get("success") != null);
+ }
+ }
+
+ @Test
+ @ShardsFixed(num = 1)
+ public void testDeleteAlsoDeletesAutocreatedConfigSet() throws Exception {
+ String collectionName = "SimpleCollectionCreateDeleteTest.testDeleteAlsoDeletesAutocreatedConfigSet";
+ CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName, 1, 1);
+
+ NamedList<Object> request = create.process(cloudClient).getResponse();
+
+ if (request.get("success") != null) {
+ // collection exists now
+ assertTrue(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, false));
+
+ String configName = cloudClient.getZkStateReader().readConfigName(collectionName);
+
+ // config for this collection is '.AUTOCREATED', and exists globally
+ assertTrue(configName.endsWith(".AUTOCREATED"));
+ assertTrue(cloudClient.getZkStateReader().getConfigManager().listConfigs().contains(configName));
+
+ CollectionAdminRequest.Delete delete = CollectionAdminRequest.deleteCollection(collectionName);
+ cloudClient.request(delete);
+
+ // collection has been deleted
+ assertFalse(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, false));
+ // ... and so has its autocreated config set
+ assertFalse("The auto-created config set should have been deleted with its collection", cloudClient.getZkStateReader().getConfigManager().listConfigs().contains(configName));
}
- if (allContainersEmpty) {
- break;
+ }
+
+ @Test
+ @ShardsFixed(num = 1)
+ public void testDeleteDoesNotDeleteSharedAutocreatedConfigSet() throws Exception {
+ String collectionNameInitial = "SimpleCollectionCreateDeleteTest.initialCollection";
+ CollectionAdminRequest.Create createInitial = CollectionAdminRequest.createCollection(collectionNameInitial, 1, 1);
+
+ NamedList<Object> requestInitial = createInitial.process(cloudClient).getResponse();
+
+ if (requestInitial.get("success") != null) {
+ // collection exists now
+ assertTrue(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionNameInitial, false));
+
+ String configName = cloudClient.getZkStateReader().readConfigName(collectionNameInitial);
+
+ // config for this collection is '.AUTOCREATED', and exists globally
+ assertTrue(configName.endsWith(".AUTOCREATED"));
+ assertTrue(cloudClient.getZkStateReader().getConfigManager().listConfigs().contains(configName));
+
+ // create a second collection, sharing the same configSet
+ String collectionNameWithSharedConfig = "SimpleCollectionCreateDeleteTest.collectionSharingAutocreatedConfigSet";
+ CollectionAdminRequest.Create createWithSharedConfig = CollectionAdminRequest.createCollection(collectionNameWithSharedConfig, configName, 1, 1);
+
+ NamedList<Object> requestWithSharedConfig = createWithSharedConfig.process(cloudClient).getResponse();
+ assertTrue("The collection with shared config set should have been created", requestWithSharedConfig.get("success") != null);
+ assertTrue("The new collection should exist after a successful creation", cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionNameWithSharedConfig, false));
+
+ String configNameOfSecondCollection = cloudClient.getZkStateReader().readConfigName(collectionNameWithSharedConfig);
+
+ assertEquals("Both collections should be using the same config", configName, configNameOfSecondCollection);
+
+ // delete the initial collection - the config set should stay, since it is shared with the other collection
+ CollectionAdminRequest.Delete deleteInitialCollection = CollectionAdminRequest.deleteCollection(collectionNameInitial);
+ cloudClient.request(deleteInitialCollection);
+
+ // initial collection has been deleted
+ assertFalse(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionNameInitial, false));
+ // ... but not its autocreated config set, since it is shared with another collection
+ assertTrue("The auto-created config set should NOT have been deleted. Another collection is using it.", cloudClient.getZkStateReader().getConfigManager().listConfigs().contains(configName));
+
+ // delete the second collection - the config set should now be deleted, since it is no longer shared any other collection
+ CollectionAdminRequest.Delete deleteSecondCollection = CollectionAdminRequest.deleteCollection(collectionNameWithSharedConfig);
+ cloudClient.request(deleteSecondCollection);
+
+ // the collection has been deleted
+ assertFalse(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionNameWithSharedConfig, false));
+ // ... and the config set is now also deleted - once it doesn't get referenced by any collection
+ assertFalse("The auto-created config set should have been deleted now. No collection is referencing it.", cloudClient.getZkStateReader().getConfigManager().listConfigs().contains(configName));
}
- }
-
- // create collection again on a node other than the overseer leader
- create = CollectionAdminRequest.createCollection(collectionName,1,1)
- .setCreateNodeSet(notOverseerNode)
- .setStateFormat(2);
- request = create.process(cloudClient).getResponse();
- assertTrue("Collection creation should not have failed", request.get("success") != null);
}
- }
}