You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by kr...@apache.org on 2022/05/05 17:04:40 UTC
[solr] branch branch_9x updated: SOLR-16110: Using Schema/Config API breaks the File-Upload of Config Set File (#831)
This is an automated email from the ASF dual-hosted git repository.
krisden pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new e0bf3d4a4bd SOLR-16110: Using Schema/Config API breaks the File-Upload of Config Set File (#831)
e0bf3d4a4bd is described below
commit e0bf3d4a4bdea9c26e2d774529f36533a48b1411
Author: Kevin Risden <ri...@users.noreply.github.com>
AuthorDate: Thu May 5 13:01:09 2022 -0400
SOLR-16110: Using Schema/Config API breaks the File-Upload of Config Set File (#831)
Co-authored-by: Steffen Moldenhauer <s....@intershop.de>
---
.../java/org/apache/solr/cloud/ZkController.java | 10 +++-
.../org/apache/solr/cloud/TestConfigSetsAPI.java | 61 +++++++++++++++++++++-
.../org/apache/solr/cloud/ZkControllerTest.java | 60 +++++++++++++++++++++
.../apache/solr/schema/TestManagedSchemaAPI.java | 6 +++
4 files changed, 134 insertions(+), 3 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkController.java b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
index d026e855c91..d796d2b73eb 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkController.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkController.java
@@ -135,6 +135,8 @@ public class ZkController implements Closeable {
public static final String COLLECTION_PARAM_PREFIX = "collection.";
public static final String CONFIGNAME_PROP = "configName";
+ public static final byte[] TOUCHED_ZNODE_DATA = "{}".getBytes(StandardCharsets.UTF_8);
+
static class ContextKey {
private String collection;
@@ -2625,13 +2627,17 @@ public class ZkController implements Closeable {
public static void touchConfDir(ZkSolrResourceLoader zkLoader) {
SolrZkClient zkClient = zkLoader.getZkController().getZkClient();
+ String configSetZkPath = zkLoader.getConfigSetZkPath();
try {
- zkClient.setData(zkLoader.getConfigSetZkPath(), new byte[] {0}, true);
+ // Ensure that version gets updated by replacing data with itself.
+ // If there is no existing data then set it to byte[] {0}.
+ // This should trigger any watchers if necessary as well.
+ zkClient.atomicUpdate(configSetZkPath, bytes -> bytes == null ? TOUCHED_ZNODE_DATA : bytes);
} catch (Exception e) {
if (e instanceof InterruptedException) {
Thread.currentThread().interrupt(); // Restore the interrupted status
}
- final String msg = "Error 'touching' conf location " + zkLoader.getConfigSetZkPath();
+ final String msg = "Error 'touching' conf location " + configSetZkPath;
log.error(msg, e);
throw new SolrException(ErrorCode.SERVER_ERROR, msg, e);
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
index 6ab945fbdcc..740f234e88b 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
@@ -43,6 +43,7 @@ import java.util.Collections;
import java.util.Deque;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
@@ -76,8 +77,10 @@ import org.apache.solr.client.solrj.request.ConfigSetAdminRequest.Create;
import org.apache.solr.client.solrj.request.ConfigSetAdminRequest.Delete;
import org.apache.solr.client.solrj.request.ConfigSetAdminRequest.Upload;
import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.request.schema.SchemaRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.ConfigSetAdminResponse;
+import org.apache.solr.client.solrj.response.schema.SchemaResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.SolrZkClient;
@@ -117,11 +120,14 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
@BeforeClass
public static void setUpClass() throws Exception {
+ System.setProperty("managed.schema.mutable", "true");
configureCluster(1).withSecurityJson(getSecurityJson()).configure();
}
@AfterClass
- public static void tearDownClass() throws Exception {}
+ public static void tearDownClass() {
+ System.clearProperty("managed.schema.mutable");
+ }
private static ConfigSetService getConfigSetService() {
return cluster.getOpenOverseer().getCoreContainer().getConfigSetService();
@@ -833,6 +839,59 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
}
}
+ @Test
+ public void testNewSingleFileAfterSchemaAPIV1() throws Exception {
+ testNewSingleFileAfterSchemaAPI(false);
+ }
+
+ @Test
+ public void testNewSingleFileAfterSchemaAPIV2() throws Exception {
+ testNewSingleFileAfterSchemaAPI(true);
+ }
+
+ private void addStringField(String fieldName, String collection, CloudSolrClient cloudClient)
+ throws IOException, SolrServerException {
+ Map<String, Object> fieldAttributes = new LinkedHashMap<>();
+ fieldAttributes.put("name", fieldName);
+ fieldAttributes.put("type", "string");
+ SchemaRequest.AddField addFieldUpdateSchemaRequest =
+ new SchemaRequest.AddField(fieldAttributes);
+ SchemaResponse.UpdateResponse addFieldResponse =
+ addFieldUpdateSchemaRequest.process(cloudClient, collection);
+ assertEquals(0, addFieldResponse.getStatus());
+ assertNull(addFieldResponse.getResponse().get("errors"));
+
+ log.info("added new field={}", fieldName);
+ }
+
+ private void testNewSingleFileAfterSchemaAPI(boolean v2) throws Exception {
+ String collectionName = "newcollection";
+ String configsetName = "regular";
+ String configsetSuffix = "testSinglePathNew-1-" + v2;
+ createConfigSet(null, configsetName + configsetSuffix, null, cluster.getSolrClient(), "solr");
+ createCollection(
+ collectionName, configsetName + configsetSuffix, 1, 1, cluster.getSolrClient());
+ addStringField("newField", collectionName, cluster.getSolrClient());
+
+ assertEquals(
+ 0,
+ uploadSingleConfigSetFile(
+ configsetName,
+ configsetSuffix,
+ "solr",
+ "solr/configsets/upload/regular/solrconfig.xml",
+ "/test/upload/path/solrconfig.xml",
+ false,
+ false,
+ v2));
+ SolrZkClient zkClient = cluster.getZkServer().getZkClient();
+ assertEquals(
+ "Expecting first version of new file",
+ 0,
+ getConfigZNodeVersion(
+ zkClient, configsetName, configsetSuffix, "test/upload/path/solrconfig.xml"));
+ }
+
@Test
public void testSingleWithCleanupV1() throws Exception {
testSingleWithCleanup(false);
diff --git a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
index 4b82516b065..4da6d64a2b7 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ZkControllerTest.java
@@ -21,6 +21,7 @@ import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.mockito.Mockito.mock;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.Collections;
import java.util.HashMap;
@@ -44,8 +45,10 @@ import org.apache.solr.update.UpdateShardHandler;
import org.apache.solr.update.UpdateShardHandlerConfig;
import org.apache.solr.util.LogLevel;
import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.data.Stat;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Test;
@Slow
@SolrTestCaseJ4.SuppressSSL
@@ -337,6 +340,63 @@ public class ZkControllerTest extends SolrTestCaseJ4 {
}
}
+ @Test
+ public void testTouchConfDir() throws Exception {
+ Path zkDir = createTempDir("zkData");
+ ZkTestServer server = new ZkTestServer(zkDir);
+ try {
+ server.run();
+ try (SolrZkClient zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT)) {
+ CoreContainer cc = getCoreContainer();
+ try {
+ CloudConfig cloudConfig =
+ new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983, "solr").build();
+ try (ZkController zkController =
+ new ZkController(cc, server.getZkAddress(), TIMEOUT, cloudConfig, () -> null)) {
+ final Path dir = createTempDir();
+ final String configsetName = "testconfigset";
+ try (ZkSolrResourceLoader loader =
+ new ZkSolrResourceLoader(dir, configsetName, null, zkController)) {
+ String zkpath = "/configs/" + configsetName;
+
+ // touchConfDir doesn't make the znode
+ Stat s = new Stat();
+ assertFalse(zkClient.exists(zkpath, true));
+ zkClient.makePath(zkpath, true);
+ assertTrue(zkClient.exists(zkpath, true));
+ assertNull(zkClient.getData(zkpath, null, s, true));
+ assertEquals(0, s.getVersion());
+
+ // touchConfDir should only set the data to new byte[] {0}
+ ZkController.touchConfDir(loader);
+ assertTrue(zkClient.exists(zkpath, true));
+ assertArrayEquals(
+ ZkController.TOUCHED_ZNODE_DATA, zkClient.getData(zkpath, null, s, true));
+ assertEquals(1, s.getVersion());
+
+ // set new data to check if touchConfDir overwrites later
+ byte[] data = "{\"key\", \"new data\"".getBytes(StandardCharsets.UTF_8);
+ s = zkClient.setData(zkpath, data, true);
+ assertEquals(2, s.getVersion());
+
+ // make sure touchConfDir doesn't overwrite existing data.
+ // touchConfDir should update version.
+ assertTrue(zkClient.exists(zkpath, true));
+ ZkController.touchConfDir(loader);
+ assertTrue(zkClient.exists(zkpath, true));
+ assertArrayEquals(data, zkClient.getData(zkpath, null, s, true));
+ assertEquals(3, s.getVersion());
+ }
+ }
+ } finally {
+ cc.shutdown();
+ }
+ }
+ } finally {
+ server.shutdown();
+ }
+ }
+
private CoreContainer getCoreContainer() {
return new MockCoreContainer();
}
diff --git a/solr/core/src/test/org/apache/solr/schema/TestManagedSchemaAPI.java b/solr/core/src/test/org/apache/solr/schema/TestManagedSchemaAPI.java
index 50eba14edc2..e2a1cbe7029 100644
--- a/solr/core/src/test/org/apache/solr/schema/TestManagedSchemaAPI.java
+++ b/solr/core/src/test/org/apache/solr/schema/TestManagedSchemaAPI.java
@@ -30,6 +30,7 @@ import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.schema.SchemaResponse;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.SolrInputDocument;
+import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
@@ -47,6 +48,11 @@ public class TestManagedSchemaAPI extends SolrCloudTestCase {
.configure();
}
+ @AfterClass
+ public static void tearDownClass() {
+ System.clearProperty("managed.schema.mutable");
+ }
+
@Test
public void test() throws Exception {
String collection = "testschemaapi";