You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ho...@apache.org on 2023/01/10 00:25:48 UTC
[solr] branch main updated: SOLR-16480: Add overridable allow-list for ConfigSet file types
This is an automated email from the ASF dual-hosted git repository.
houston pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 28d6b016331 SOLR-16480: Add overridable allow-list for ConfigSet file types
28d6b016331 is described below
commit 28d6b0163316376ef3b5429b3554c5041b47b5be
Author: Houston Putman <ho...@apache.org>
AuthorDate: Mon Jan 9 16:59:23 2023 -0500
SOLR-16480: Add overridable allow-list for ConfigSet file types
---
solr/CHANGES.txt | 2 +
.../org/apache/solr/cloud/ZkConfigSetService.java | 24 ++-
.../solr/core/FileSystemConfigSetService.java | 21 ++-
.../org/apache/solr/core/backup/BackupManager.java | 31 ++--
.../backup/repository/BackupRepositoryFactory.java | 1 -
.../handler/configsets/UploadConfigSetFileAPI.java | 7 +
.../org/apache/solr/cloud/TestConfigSetsAPI.java | 162 +++++++++++++++++++--
.../configuration-guide/pages/config-sets.adoc | 19 +++
.../configuration-guide/pages/configsets-api.adoc | 2 +
.../pages/major-changes-in-solr-9.adoc | 2 +
.../solr/common/cloud/ZkMaintenanceUtils.java | 44 +++++-
11 files changed, 277 insertions(+), 38 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index d9a8ff7768e..b34910562f5 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -228,6 +228,8 @@ Bug Fixes
* SOLR-16588: Fixed problem with default knn algorithm (Elia Porciani via Alessandro Benedetti)
+* SOLR-16480: ConfigSets now have an overridable allow-list for filetypes. (Houston Putman)
+
================== 9.1.0 ==================
New Features
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
index b67d4e7cc50..88f6a7f9f91 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
@@ -199,8 +199,12 @@ public class ZkConfigSetService extends ConfigSetService {
throws IOException {
String filePath = CONFIGS_ZKNODE + "/" + configName + "/" + fileName;
try {
- // if overwriteOnExists is true then zkClient#makePath failOnExists is set to false
- zkClient.makePath(filePath, data, CreateMode.PERSISTENT, null, !overwriteOnExists, true);
+ if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {
+ log.warn("Not including uploading file to config, as it is a forbidden type: {}", fileName);
+ } else {
+ // if overwriteOnExists is true then zkClient#makePath failOnExists is set to false
+ zkClient.makePath(filePath, data, CreateMode.PERSISTENT, null, !overwriteOnExists, true);
+ }
} catch (KeeperException.NodeExistsException nodeExistsException) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
@@ -327,9 +331,19 @@ public class ZkConfigSetService extends ConfigSetService {
private void copyData(String fromZkFilePath, String toZkFilePath)
throws KeeperException, InterruptedException {
- log.debug("Copying zk node {} to {}", fromZkFilePath, toZkFilePath);
- byte[] data = zkClient.getData(fromZkFilePath, null, null, true);
- zkClient.makePath(toZkFilePath, data, true);
+ if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fromZkFilePath)) {
+ log.warn(
+ "Skipping copy of file in ZK, as the source file is a forbidden type: {}",
+ fromZkFilePath);
+ } else if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(toZkFilePath)) {
+ log.warn(
+ "Skipping download of file from ZK, as the target file is a forbidden type: {}",
+ toZkFilePath);
+ } else {
+ log.debug("Copying zk node {} to {}", fromZkFilePath, toZkFilePath);
+ byte[] data = zkClient.getData(fromZkFilePath, null, null, true);
+ zkClient.makePath(toZkFilePath, data, true);
+ }
}
public SolrCloudManager getSolrCloudManager() {
diff --git a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
index 75d8e288834..5ada2f99cfb 100644
--- a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
+++ b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
@@ -35,6 +35,7 @@ import java.util.Objects;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ZkMaintenanceUtils;
import org.apache.solr.common.util.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -146,9 +147,13 @@ public class FileSystemConfigSetService extends ConfigSetService {
public void uploadFileToConfig(
String configName, String fileName, byte[] data, boolean overwriteOnExists)
throws IOException {
- Path filePath = getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));
- if (!Files.exists(filePath) || overwriteOnExists) {
- Files.write(filePath, data);
+ if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {
+ log.warn("Not including uploading file to config, as it is a forbidden type: {}", fileName);
+ } else {
+ Path filePath = getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));
+ if (!Files.exists(filePath) || overwriteOnExists) {
+ Files.write(filePath, data);
+ }
}
}
@@ -195,8 +200,14 @@ public class FileSystemConfigSetService extends ConfigSetService {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
- Files.copy(
- file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);
+ if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(file.getFileName().toString())) {
+ log.warn(
+ "Not including uploading file to config, as it is a forbidden type: {}",
+ file.getFileName());
+ } else {
+ Files.copy(
+ file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);
+ }
return FileVisitResult.CONTINUE;
}
});
diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
index 0d6cd77f3df..cd50bb5deef 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
@@ -36,6 +36,7 @@ import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkMaintenanceUtils;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.ConfigSetService;
@@ -337,11 +338,16 @@ public class BackupManager {
// checking for '/' is correct for a directory since ConfigSetService#getAllConfigFiles
// always separates file paths with '/'
if (!filePath.endsWith("/")) {
- log.debug("Writing file {}", filePath);
- // ConfigSetService#downloadFileFromConfig requires '/' in fle path separator
- byte[] data = configSetService.downloadFileFromConfig(configName, filePath);
- try (OutputStream os = repository.createOutput(uri)) {
- os.write(data);
+ if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(filePath)) {
+ log.warn(
+ "Not including zookeeper file in backup, as it is a forbidden type: {}", filePath);
+ } else {
+ log.debug("Writing file {}", filePath);
+ // ConfigSetService#downloadFileFromConfig requires '/' in fle path separator
+ byte[] data = configSetService.downloadFileFromConfig(configName, filePath);
+ try (OutputStream os = repository.createOutput(uri)) {
+ os.write(data);
+ }
}
} else {
if (!repository.exists(uri)) {
@@ -361,11 +367,16 @@ public class BackupManager {
switch (t) {
case FILE:
{
- try (IndexInput is = repository.openInput(sourceDir, file, IOContext.DEFAULT)) {
- // probably ok since the config file should be small.
- byte[] arr = new byte[(int) is.length()];
- is.readBytes(arr, 0, (int) is.length());
- configSetService.uploadFileToConfig(configName, filePath, arr, false);
+ if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(filePath)) {
+ log.warn(
+ "Not including zookeeper file in restore, as it is a forbidden type: {}", file);
+ } else {
+ try (IndexInput is = repository.openInput(sourceDir, file, IOContext.DEFAULT)) {
+ // probably ok since the config file should be small.
+ byte[] arr = new byte[(int) is.length()];
+ is.readBytes(arr, 0, (int) is.length());
+ configSetService.uploadFileToConfig(configName, filePath, arr, false);
+ }
}
break;
}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java b/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java
index 6a5ae42f165..17fc9b4bd76 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/repository/BackupRepositoryFactory.java
@@ -95,7 +95,6 @@ public class BackupRepositoryFactory {
if (defaultBackupRepoPlugin != null) {
return newInstance(loader, defaultBackupRepoPlugin.name);
}
-
LocalFileSystemRepository repo = new LocalFileSystemRepository();
repo.init(new NamedList<>());
return repo;
diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
index d44f3c5ff31..df877e2b8f8 100644
--- a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
+++ b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
@@ -23,6 +23,7 @@ import java.io.InputStream;
import org.apache.commons.io.IOUtils;
import org.apache.solr.api.EndPoint;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ZkMaintenanceUtils;
import org.apache.solr.common.params.ConfigSetParams;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.request.SolrQueryRequest;
@@ -71,6 +72,12 @@ public class UploadConfigSetFileAPI extends ConfigSetAPIBase {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"The file path provided for upload, '" + singleFilePath + "', is not valid.");
+ } else if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fixedSingleFilePath)) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ "The file type provided for upload, '"
+ + singleFilePath
+ + "', is forbidden for use in configSets.");
} else if (cleanup) {
// Cleanup is not allowed while using singleFilePath upload
throw new SolrException(
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
index f66d6e97349..48f651862e1 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
@@ -82,6 +82,7 @@ import org.apache.solr.client.solrj.response.schema.SchemaResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkMaintenanceUtils;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.common.params.ConfigSetParams;
import org.apache.solr.common.params.ConfigSetParams.ConfigSetAction;
@@ -578,13 +579,14 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
assertEquals(
"Can't overwrite an existing configset unless the overwrite parameter is set",
400,
- uploadConfigSet(configsetName, configsetSuffix, null, false, false, v2));
+ uploadConfigSet(configsetName, configsetSuffix, null, false, false, v2, false));
unIgnoreException("The configuration regulartestOverwrite-1 already exists in zookeeper");
assertEquals(
"Expecting version to remain equal",
solrconfigZkVersion,
getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"));
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2));
+ assertEquals(
+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -619,13 +621,14 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
for (String f : extraFiles) {
zkClient.makePath(f, true);
}
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2));
+ assertEquals(
+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));
for (String f : extraFiles) {
assertTrue(
"Expecting file " + f + " to exist in ConfigSet but it's gone",
zkClient.exists(f, true));
}
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2));
+ assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false));
for (String f : extraFiles) {
assertFalse(
"Expecting file " + f + " to be deleted from ConfigSet but it wasn't",
@@ -635,6 +638,34 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
}
}
+ @Test
+ public void testOverwriteWithForbiddenFilesV1() throws Exception {
+ testOverwriteWithForbiddenFiles(false);
+ }
+
+ @Test
+ public void testOverwriteWithForbiddenFilesV2() throws Exception {
+ testOverwriteWithForbiddenFiles(true);
+ }
+
+ public void testOverwriteWithForbiddenFiles(boolean v2) throws Exception {
+ String configsetName = "regular";
+ String configsetSuffix = "testOverwriteWithForbiddenFiles-1-" + v2;
+ uploadConfigSetWithAssertions(configsetName, configsetSuffix, null);
+ try (SolrZkClient zkClient =
+ new SolrZkClient(
+ cluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, 45000, null)) {
+ String configPath = "/configs/" + configsetName + configsetSuffix;
+ assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, true));
+ for (String fileEnding : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
+ String f = configPath + "/test." + fileEnding;
+ assertFalse(
+ "Expecting file " + f + " to not exist, because it has a forbidden file type",
+ zkClient.exists(f, true));
+ }
+ }
+ }
+
@Test
public void testOverwriteWithTrustV1() throws Exception {
testOverwriteWithTrust(false);
@@ -656,7 +687,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
int solrconfigZkVersion =
getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml");
// Was untrusted, overwrite with untrusted
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2));
+ assertEquals(
+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -666,7 +698,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml");
// Was untrusted, overwrite with trusted but no cleanup
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2));
+ assertEquals(
+ 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -691,7 +724,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
"Either empty zipped data, or non-zipped data was passed. In order to upload a configSet, you must zip a non-empty directory to upload.");
// Was untrusted, overwrite with trusted with cleanup
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2));
+ assertEquals(
+ 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -705,7 +739,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
assertEquals(
"Can't upload a trusted configset with an untrusted request",
400,
- uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2));
+ uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));
assertEquals(
"Expecting version to remain equal",
solrconfigZkVersion,
@@ -717,7 +751,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
assertEquals(
"Can't upload a trusted configset with an untrusted request",
400,
- uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2));
+ uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false));
assertEquals(
"Expecting version to remain equal",
solrconfigZkVersion,
@@ -726,7 +760,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
unIgnoreException("Trying to make an unstrusted ConfigSet update on a trusted configSet");
// Was trusted, overwrite with trusted no cleanup
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2));
+ assertEquals(
+ 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -736,7 +771,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml");
// Was trusted, overwrite with trusted with cleanup
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2));
+ assertEquals(
+ 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -1014,6 +1050,51 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
}
}
+ @Test
+ public void testSingleFileForbiddenTypeV1() throws Exception {
+ testSingleFileForbiddenType(false);
+ }
+
+ @Test
+ public void testSingleFileForbiddenTypeV2() throws Exception {
+ testSingleFileForbiddenType(true);
+ }
+
+ public void testSingleFileForbiddenType(boolean v2) throws Exception {
+ String configsetName = "regular";
+ String configsetSuffix = "testSingleFileForbiddenType-1-" + v2;
+ uploadConfigSetWithAssertions(configsetName, configsetSuffix, "solr");
+ try (SolrZkClient zkClient =
+ new SolrZkClient(
+ cluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, 45000, null)) {
+ for (String fileType : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
+ ignoreException("is forbidden for use in configSets");
+ assertEquals(
+ "Can't upload a configset file with a forbidden type: " + fileType,
+ 400,
+ uploadSingleConfigSetFile(
+ configsetName,
+ configsetSuffix,
+ "solr",
+ "solr/configsets/upload/regular/solrconfig.xml",
+ "/test/different/path/solrconfig." + fileType,
+ false,
+ false,
+ v2));
+ assertFalse(
+ "New file should not exist, since the filetype is forbidden: " + fileType,
+ zkClient.exists(
+ "/configs/"
+ + configsetName
+ + configsetSuffix
+ + "/test/different/path/solrconfig."
+ + fileType,
+ true));
+ unIgnoreException("is forbidden for use in configSets");
+ }
+ }
+ }
+
@Test
public void testSingleFileUntrustedV1() throws Exception {
testSingleFileUntrusted(false);
@@ -1387,7 +1468,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
String configSetName, String suffix, String username, SolrZkClient zkClient, boolean v2)
throws IOException {
assertFalse(getConfigSetService().checkConfigExists(configSetName + suffix));
- return uploadConfigSet(configSetName, suffix, username, false, false, v2);
+ return uploadConfigSet(configSetName, suffix, username, false, false, v2, false);
}
private long uploadConfigSet(
@@ -1396,12 +1477,16 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
String username,
boolean overwrite,
boolean cleanup,
- boolean v2)
+ boolean v2,
+ boolean forbiddenTypes)
throws IOException {
// Read zipped sample config
return uploadGivenConfigSet(
- createTempZipFile("solr/configsets/upload/" + configSetName),
+ forbiddenTypes
+ ? createTempZipFileWithForbiddenTypes(
+ "solr/configsets/upload/" + configSetName + "/solrconfig.xml")
+ : createTempZipFile("solr/configsets/upload/" + configSetName),
configSetName,
suffix,
username,
@@ -1553,6 +1638,55 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
}
}
+ /**
+ * Create a zip file (in the temp directory) containing a file with all forbidden types (named
+ * "test.fileType")
+ */
+ private File createTempZipFileWithForbiddenTypes(String file) {
+ try {
+ final File zipFile = createTempFile("configset", "zip").toFile();
+ final File directory = SolrTestCaseJ4.getFile(file);
+ if (log.isInfoEnabled()) {
+ log.info("Directory: {}", directory.getAbsolutePath());
+ }
+ zipWithForbiddenEndings(directory, zipFile);
+ if (log.isInfoEnabled()) {
+ log.info("Zipfile: {}", zipFile.getAbsolutePath());
+ }
+ return zipFile;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static void zipWithForbiddenEndings(File file, File zipfile) throws IOException {
+ OutputStream out = new FileOutputStream(zipfile);
+ ZipOutputStream zout = new ZipOutputStream(out);
+ try {
+ for (String fileType : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
+ zout.putNextEntry(new ZipEntry("test." + fileType));
+
+ InputStream in = new FileInputStream(file);
+ try {
+ byte[] buffer = new byte[1024];
+ while (true) {
+ int readCount = in.read(buffer);
+ if (readCount < 0) {
+ break;
+ }
+ zout.write(buffer, 0, readCount);
+ }
+ } finally {
+ in.close();
+ }
+
+ zout.closeEntry();
+ }
+ } finally {
+ zout.close();
+ }
+ }
+
private static void zip(File directory, File zipfile) throws IOException {
URI base = directory.toURI();
Deque<File> queue = new ArrayDeque<>();
diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/config-sets.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/config-sets.adoc
index 782aeab1e61..a04d6e41d18 100644
--- a/solr/solr-ref-guide/modules/configuration-guide/pages/config-sets.adoc
+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/config-sets.adoc
@@ -100,3 +100,22 @@ To upload a file to a configset already stored on ZooKeeper, you can use xref:de
CAUTION: By default, ZooKeeper's file size limit is 1MB.
If your files are larger than this, you'll need to either xref:deployment-guide:zookeeper-ensemble.adoc#increasing-the-file-size-limit[increase the ZooKeeper file size limit] or store them xref:libs.adoc#lib-directives-in-solrconfig[on the filesystem] of every node in a cluster.
+
+=== Forbidden File Types
+
+Solr does not accept all file types when uploading or downloading configSets.
+By default the excluded file types are:
+
+- `class`
+- `java`
+- `jar`
+- `tgz`
+- `zip`
+- `tar`
+- `gz`
+
+However, users can impose stricter or looser limits on their systems by providing a comma separated list of file types
+(without the preceding dot, e.g. `jar,class,csv`), to either of the following settings:
+
+- System Property: `-DsolrConfigSetForbiddenFileTypes`
+- Environment Variable: `SOLR_CONFIG_SET_FORBIDDEN_FILE_TYPES`
diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/configsets-api.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/configsets-api.adoc
index 6b8d78be67c..46087834555 100644
--- a/solr/solr-ref-guide/modules/configuration-guide/pages/configsets-api.adoc
+++ b/solr/solr-ref-guide/modules/configuration-guide/pages/configsets-api.adoc
@@ -103,6 +103,8 @@ Upon creation of a collection using an "untrusted" configset, the following func
If you use any of these parameters or features, you must have enabled security features in your Solr installation and you must upload the configset as an authenticated user.
+Not all file types are supported for use in configSets. Please see xref:configuration-guide:config-sets.adoc#forbidden-file-types[] for more information.
+
The `upload` command takes the following parameters:
`name`::
diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
index 11f6b1c1f40..d110ffdf33b 100644
--- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
+++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc
@@ -70,6 +70,8 @@ Due to changes in Lucene 9, that isn't possible any more.
This is an improvement to the binary release artifact, but Jetty does not allow web-apps (Solr) to share these libraries by default.
The `server/contexts/solr-jetty-context.xml` now explicitly removes these restrictions, allowing Solr to share these "server" jars which now live in `server/lib/ext`.
+* Solr no longer accepts all file types for configSets. Please see xref:configuration-guide:config-sets.adoc#forbidden-file-types[ConfigSet Forbidden File Types] for more information.
+
=== Tracing
* A new `opentelemetry` module is added, with support for OTEL tracing in `OTLP` format using gRPC.
At the same time, the `jaegertracer-configurator` module is deprecated for removal in Solr 10.
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
index 40da28d255d..4b12cbea817 100644
--- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
@@ -30,6 +30,7 @@ import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Locale;
+import java.util.Set;
import java.util.function.Predicate;
import java.util.regex.Pattern;
import org.apache.solr.client.solrj.SolrServerException;
@@ -328,13 +329,20 @@ public class ZkMaintenanceUtils {
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
throws IOException {
String filename = file.getFileName().toString();
- if (filenameExclusions != null && filenameExclusions.matcher(filename).matches()) {
+ if ((filenameExclusions != null && filenameExclusions.matcher(filename).matches())) {
log.info(
"uploadToZK skipping '{}' due to filenameExclusions '{}'",
filename,
filenameExclusions);
return FileVisitResult.CONTINUE;
}
+ if (isFileForbiddenInConfigSets(filename)) {
+ log.info(
+ "uploadToZK skipping '{}' due to forbidden file types '{}'",
+ filename,
+ USE_FORBIDDEN_FILE_TYPES);
+ return FileVisitResult.CONTINUE;
+ }
String zkNode = createZkNodeName(zkPath, rootPath, file);
try {
// if the path exists (and presumably we're uploading data to it) just set its data
@@ -421,8 +429,12 @@ public class ZkMaintenanceUtils {
if (children.size() == 0) {
// If we didn't copy data down, then we also didn't create the file. But we still need a
// marker on the local disk so create an empty file.
- if (copyDataDown(zkClient, zkPath, file) == 0) {
- Files.createFile(file);
+ if (isFileForbiddenInConfigSets(zkPath)) {
+ log.warn("Skipping download of file from ZK, as it is a forbidden type: {}", zkPath);
+ } else {
+ if (copyDataDown(zkClient, zkPath, file) == 0) {
+ Files.createFile(file);
+ }
}
} else {
Files.createDirectories(file); // Make parent dir.
@@ -548,6 +560,32 @@ public class ZkMaintenanceUtils {
}
return ret;
}
+
+ public static final String FORBIDDEN_FILE_TYPES_PROP = "solrConfigSetForbiddenFileTypes";
+ public static final String FORBIDDEN_FILE_TYPES_ENV = "SOLR_CONFIG_SET_FORBIDDEN_FILE_TYPES";
+ public static final Set<String> DEFAULT_FORBIDDEN_FILE_TYPES =
+ Set.of("class", "java", "jar", "tgz", "zip", "tar", "gz");
+ private static volatile Set<String> USE_FORBIDDEN_FILE_TYPES = null;
+
+ public static boolean isFileForbiddenInConfigSets(String filePath) {
+ // Try to set the forbidden file types just once, since it is set by SysProp/EnvVar
+ if (USE_FORBIDDEN_FILE_TYPES == null) {
+ synchronized (DEFAULT_FORBIDDEN_FILE_TYPES) {
+ if (USE_FORBIDDEN_FILE_TYPES == null) {
+ String userForbiddenFileTypes =
+ System.getProperty(
+ FORBIDDEN_FILE_TYPES_PROP, System.getenv(FORBIDDEN_FILE_TYPES_ENV));
+ if (StringUtils.isEmpty(userForbiddenFileTypes)) {
+ USE_FORBIDDEN_FILE_TYPES = DEFAULT_FORBIDDEN_FILE_TYPES;
+ } else {
+ USE_FORBIDDEN_FILE_TYPES = Set.of(userForbiddenFileTypes.split(","));
+ }
+ }
+ }
+ }
+ int lastDot = filePath.lastIndexOf(".");
+ return lastDot >= 0 && USE_FORBIDDEN_FILE_TYPES.contains(filePath.substring(lastDot + 1));
+ }
}
class ZkCopier implements ZkMaintenanceUtils.ZkVisitor {