You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ja...@apache.org on 2023/12/13 22:08:04 UTC
(solr) branch main updated: SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets
This is an automated email from the ASF dual-hosted git repository.
janhoy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/main by this push:
new 15534754f49 SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets
15534754f49 is described below
commit 15534754f492079e52288dd11abaf1c4261b3ea4
Author: Jan Høydahl <ja...@apache.org>
AuthorDate: Wed Dec 13 22:49:23 2023 +0100
SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets
---
solr/CHANGES.txt | 2 +
solr/core/build.gradle | 2 +
.../org/apache/solr/cli/ConfigSetUploadTool.java | 2 +
.../org/apache/solr/cloud/ZkConfigSetService.java | 21 ++-
.../solr/core/FileSystemConfigSetService.java | 28 +++-
.../org/apache/solr/core/backup/BackupManager.java | 23 ++-
.../handler/configsets/UploadConfigSetFileAPI.java | 8 +-
.../org/apache/solr/util/FileTypeMagicUtil.java | 166 +++++++++++++++++++++
solr/core/src/resources/magic/executables | 74 +++++++++
solr/core/src/test-files/magic/HelloWorld.java.txt | 5 +
.../test-files/magic/HelloWorldJavaClass.class.bin | Bin 0 -> 426 bytes
solr/core/src/test-files/magic/README.md | 29 ++++
solr/core/src/test-files/magic/hello.tar.bin | Bin 0 -> 4096 bytes
solr/core/src/test-files/magic/plain.txt | 1 +
solr/core/src/test-files/magic/shell.sh.txt | 2 +
.../org/apache/solr/cloud/TestConfigSetsAPI.java | 141 +++++++++++------
.../apache/solr/util/FileTypeMagicUtilTest.java | 54 +++++++
solr/licenses/simplemagic-1.17.jar.sha1 | 1 +
solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt | 15 ++
solr/licenses/simplemagic-NOTICE.txt | 0
.../solr/common/cloud/ZkMaintenanceUtils.java | 2 +
versions.lock | 1 +
versions.props | 1 +
23 files changed, 522 insertions(+), 56 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 40da8c435ac..2fd94304a24 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -169,6 +169,8 @@ Other Changes
* SOLR-17091: dev tools script cloud.sh became broken after changes in 9.3 added a new -slim.tgz file it was not expecting
cloud.sh has been updated to ignore the -slim.tgz version of the tarball.
+* SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets (janhoy, Houston Putman)
+
================== 9.4.0 ==================
New Features
---------------------
diff --git a/solr/core/build.gradle b/solr/core/build.gradle
index 61ecd1713af..ed2c8a370ae 100644
--- a/solr/core/build.gradle
+++ b/solr/core/build.gradle
@@ -159,6 +159,8 @@ dependencies {
compileOnly 'com.github.stephenc.jcip:jcip-annotations'
+ implementation 'com.j256.simplemagic:simplemagic'
+
// -- Test Dependencies
testRuntimeOnly 'org.slf4j:jcl-over-slf4j'
diff --git a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
index 5fd4a538bd7..6576742a195 100644
--- a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
@@ -27,6 +27,7 @@ import org.apache.solr.client.solrj.impl.SolrZkClientTimeout;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkMaintenanceUtils;
import org.apache.solr.core.ConfigSetService;
+import org.apache.solr.util.FileTypeMagicUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -101,6 +102,7 @@ public class ConfigSetUploadTool extends ToolBase {
+ cli.getOptionValue("confname")
+ " to ZooKeeper at "
+ zkHost);
+ FileTypeMagicUtil.assertConfigSetFolderLegal(confPath);
ZkMaintenanceUtils.uploadToZK(
zkClient,
confPath,
diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
index f02404d636d..9abde098e1c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
+++ b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
@@ -22,6 +22,7 @@ import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import org.apache.solr.client.solrj.cloud.SolrCloudManager;
@@ -39,6 +40,7 @@ import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.core.SolrConfig;
import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.util.FileTypeMagicUtil;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
@@ -199,6 +201,15 @@ public class ZkConfigSetService extends ConfigSetService {
try {
if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {
log.warn("Not including uploading file to config, as it is a forbidden type: {}", fileName);
+ } else if (FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
+ String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ String.format(
+ Locale.ROOT,
+ "Not uploading file %s to config, as it matched the MAGIC signature of a forbidden mime type %s",
+ fileName,
+ mimeType));
} else {
// if overwriteOnExists is true then zkClient#makePath failOnExists is set to false
zkClient.makePath(filePath, data, CreateMode.PERSISTENT, null, !overwriteOnExists, true);
@@ -340,7 +351,15 @@ public class ZkConfigSetService extends ConfigSetService {
} else {
log.debug("Copying zk node {} to {}", fromZkFilePath, toZkFilePath);
byte[] data = zkClient.getData(fromZkFilePath, null, null, true);
- zkClient.makePath(toZkFilePath, data, true);
+ if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
+ zkClient.makePath(toZkFilePath, data, true);
+ } else {
+ String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
+ log.warn(
+ "Skipping copy of file {} in ZK, as it matched the MAGIC signature of a forbidden mime type {}",
+ fromZkFilePath,
+ mimeType);
+ }
}
}
diff --git a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
index 1f1a42b15e1..4b041252211 100644
--- a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
+++ b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
@@ -37,6 +37,7 @@ import java.util.stream.Stream;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ZkMaintenanceUtils;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.util.FileTypeMagicUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -150,9 +151,17 @@ public class FileSystemConfigSetService extends ConfigSetService {
if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {
log.warn("Not including uploading file to config, as it is a forbidden type: {}", fileName);
} else {
- Path filePath = getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));
- if (!Files.exists(filePath) || overwriteOnExists) {
- Files.write(filePath, data);
+ if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
+ Path filePath = getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));
+ if (!Files.exists(filePath) || overwriteOnExists) {
+ Files.write(filePath, data);
+ }
+ } else {
+ String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
+ log.warn(
+ "Not including uploading file {}, as it matched the MAGIC signature of a forbidden mime type {}",
+ fileName,
+ mimeType);
}
}
}
@@ -205,8 +214,17 @@ public class FileSystemConfigSetService extends ConfigSetService {
"Not including uploading file to config, as it is a forbidden type: {}",
file.getFileName());
} else {
- Files.copy(
- file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);
+ if (!FileTypeMagicUtil.isFileForbiddenInConfigset(Files.newInputStream(file))) {
+ Files.copy(
+ file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);
+ } else {
+ String mimeType =
+ FileTypeMagicUtil.INSTANCE.guessMimeType(Files.newInputStream(file));
+ log.warn(
+ "Not copying file {}, as it matched the MAGIC signature of a forbidden mime type {}",
+ file.getFileName(),
+ mimeType);
+ }
}
return FileVisitResult.CONTINUE;
}
diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
index be6a1a83c2f..8ff78b27e08 100644
--- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
+++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
@@ -40,6 +40,7 @@ import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.ConfigSetService;
import org.apache.solr.core.backup.repository.BackupRepository;
+import org.apache.solr.util.FileTypeMagicUtil;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.slf4j.Logger;
@@ -349,8 +350,16 @@ public class BackupManager {
if (data == null) {
data = new byte[0];
}
- try (OutputStream os = repository.createOutput(uri)) {
- os.write(data);
+ if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
+ try (OutputStream os = repository.createOutput(uri)) {
+ os.write(data);
+ }
+ } else {
+ String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
+ log.warn(
+ "Not including zookeeper file {} in backup, as it matched the MAGIC signature of a forbidden mime type {}",
+ filePath,
+ mimeType);
}
}
} else {
@@ -379,7 +388,15 @@ public class BackupManager {
// probably ok since the config file should be small.
byte[] arr = new byte[(int) is.length()];
is.readBytes(arr, 0, (int) is.length());
- configSetService.uploadFileToConfig(configName, filePath, arr, false);
+ if (!FileTypeMagicUtil.isFileForbiddenInConfigset(arr)) {
+ configSetService.uploadFileToConfig(configName, filePath, arr, false);
+ } else {
+ String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(arr);
+ log.warn(
+ "Not including zookeeper file {} in restore, as it matched the MAGIC signature of a forbidden mime type {}",
+ filePath,
+ mimeType);
+ }
}
}
break;
diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
index 98889aff563..2380a79a92b 100644
--- a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
+++ b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
@@ -27,6 +27,7 @@ import org.apache.solr.common.params.ConfigSetParams;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.util.FileTypeMagicUtil;
/**
* V2 API for adding or updating a single file within a configset.
@@ -67,11 +68,13 @@ public class UploadConfigSetFileAPI extends ConfigSetAPIBase {
if (fixedSingleFilePath.charAt(0) == '/') {
fixedSingleFilePath = fixedSingleFilePath.substring(1);
}
+ byte[] data = inputStream.readAllBytes();
if (fixedSingleFilePath.isEmpty()) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"The file path provided for upload, '" + singleFilePath + "', is not valid.");
- } else if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fixedSingleFilePath)) {
+ } else if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fixedSingleFilePath)
+ || FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"The file type provided for upload, '"
@@ -87,8 +90,7 @@ public class UploadConfigSetFileAPI extends ConfigSetAPIBase {
// For creating the baseNode, the cleanup parameter is only allowed to be true when
// singleFilePath is not passed.
createBaseNode(configSetService, overwritesExisting, requestIsTrusted, configSetName);
- configSetService.uploadFileToConfig(
- configSetName, fixedSingleFilePath, inputStream.readAllBytes(), allowOverwrite);
+ configSetService.uploadFileToConfig(configSetName, fixedSingleFilePath, data, allowOverwrite);
}
}
}
diff --git a/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java b/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java
new file mode 100644
index 00000000000..cfb6c9fa0af
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util;
+
+import com.j256.simplemagic.ContentInfo;
+import com.j256.simplemagic.ContentInfoUtil;
+import com.j256.simplemagic.ContentType;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Locale;
+import java.util.Set;
+import org.apache.solr.common.SolrException;
+
+/** Utility class to guess the mime type of file based on its magic number. */
+public class FileTypeMagicUtil implements ContentInfoUtil.ErrorCallBack {
+ private final ContentInfoUtil util;
+ private static final Set<String> SKIP_FOLDERS = new HashSet<>(Arrays.asList(".", ".."));
+
+ public static FileTypeMagicUtil INSTANCE = new FileTypeMagicUtil();
+
+ FileTypeMagicUtil() {
+ try {
+ util = new ContentInfoUtil("/magic/executables", this);
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error parsing magic file", e);
+ }
+ }
+
+ /**
+ * Asserts that an entire configset folder is legal to upload.
+ *
+ * @param confPath the path to the folder
+ * @throws SolrException if an illegal file is found in the folder structure
+ */
+ public static void assertConfigSetFolderLegal(Path confPath) throws IOException {
+ Files.walkFileTree(
+ confPath,
+ new SimpleFileVisitor<Path>() {
+ @Override
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
+ throws IOException {
+ // Read first 100 bytes of the file to determine the mime type
+ try (InputStream fileStream = Files.newInputStream(file)) {
+ byte[] bytes = new byte[100];
+ fileStream.read(bytes);
+ if (FileTypeMagicUtil.isFileForbiddenInConfigset(bytes)) {
+ throw new SolrException(
+ SolrException.ErrorCode.BAD_REQUEST,
+ String.format(
+ Locale.ROOT,
+ "Not uploading file %s to configset, as it matched the MAGIC signature of a forbidden mime type %s",
+ file,
+ FileTypeMagicUtil.INSTANCE.guessMimeType(bytes)));
+ }
+ return FileVisitResult.CONTINUE;
+ }
+ }
+
+ @Override
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
+ throws IOException {
+ if (SKIP_FOLDERS.contains(dir.getFileName().toString()))
+ return FileVisitResult.SKIP_SUBTREE;
+
+ return FileVisitResult.CONTINUE;
+ }
+ });
+ }
+
+ /**
+ * Guess the mime type of file based on its magic number.
+ *
+ * @param stream input stream of the file
+ * @return string with content-type or "application/octet-stream" if unknown
+ */
+ public String guessMimeType(InputStream stream) {
+ try {
+ ContentInfo info = util.findMatch(stream);
+ if (info == null) {
+ return ContentType.OTHER.getMimeType();
+ }
+ return info.getContentType().getMimeType();
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+ }
+ }
+
+ /**
+ * Guess the mime type of file bytes based on its magic number.
+ *
+ * @param bytes the first bytes at start of the file
+ * @return string with content-type or "application/octet-stream" if unknown
+ */
+ public String guessMimeType(byte[] bytes) {
+ return guessMimeType(new ByteArrayInputStream(bytes));
+ }
+
+ @Override
+ public void error(String line, String details, Exception e) {
+ throw new SolrException(
+ SolrException.ErrorCode.SERVER_ERROR,
+ String.format(Locale.ROOT, "%s: %s", line, details),
+ e);
+ }
+
+ /**
+ * Determine forbidden file type based on magic bytes matching of the file itself. Forbidden types
+ * are:
+ *
+ * <ul>
+ * <li><code>application/x-java-applet</code>: java class file
+ * <li><code>application/zip</code>: jar or zip archives
+ * <li><code>application/x-tar</code>: tar archives
+ * <li><code>text/x-shellscript</code>: shell or bash script
+ * </ul>
+ *
+ * @param fileStream stream from the file content
+ * @return true if file is among the forbidden mime-types
+ */
+ public static boolean isFileForbiddenInConfigset(InputStream fileStream) {
+ return forbiddenTypes.contains(FileTypeMagicUtil.INSTANCE.guessMimeType(fileStream));
+ }
+
+ /**
+ * Determine forbidden file type based on magic bytes matching of the first bytes of the file.
+ *
+ * @param bytes byte array of the file content
+ * @return true if file is among the forbidden mime-types
+ */
+ public static boolean isFileForbiddenInConfigset(byte[] bytes) {
+ if (bytes == null || bytes.length == 0)
+ return false; // A ZK znode may be a folder with no content
+ return isFileForbiddenInConfigset(new ByteArrayInputStream(bytes));
+ }
+
+ private static final Set<String> forbiddenTypes =
+ new HashSet<>(
+ Arrays.asList(
+ System.getProperty(
+ "solr.configset.upload.mimetypes.forbidden",
+ "application/x-java-applet,application/zip,application/x-tar,text/x-shellscript")
+ .split(",")));
+}
diff --git a/solr/core/src/resources/magic/executables b/solr/core/src/resources/magic/executables
new file mode 100644
index 00000000000..04094eaf797
--- /dev/null
+++ b/solr/core/src/resources/magic/executables
@@ -0,0 +1,74 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# POSIX tar archives
+# URL: https://en.wikipedia.org/wiki/Tar_(computing)
+# Reference: https://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5&manpath=FreeBSD+8-current
+# header mainly padded with nul bytes
+500 quad 0
+!:strength /2
+# filename or extended attribute printable strings in range space null til umlaut ue
+>0 ubeshort >0x1F00
+>>0 ubeshort <0xFCFD
+# last 4 header bytes often null but tar\0 in gtarfail2.tar gtarfail.tar-bad
+# at https://sourceforge.net/projects/s-tar/files/testscripts/
+>>>508 ubelong&0x8B9E8DFF 0
+# nul, space or ascii digit 0-7 at start of mode
+>>>>100 ubyte&0xC8 =0
+>>>>>101 ubyte&0xC8 =0
+# nul, space at end of check sum
+>>>>>>155 ubyte&0xDF =0
+# space or ascii digit 0 at start of check sum
+>>>>>>>148 ubyte&0xEF =0x20
+# check for specific 1st member name that indicates other mime type and file name suffix
+>>>>>>>>0 string TpmEmuTpms/permall
+!:mime application/x-tar
+!:ext tar
+# other stuff in padding
+# some implementations add new fields to the blank area at the end of the header record
+# created for example by DOS TAR 3.20g 1994 Tim V.Shapore with -j option
+>>257 ulong !0 tar archive (old)
+!:mime application/x-tar
+!:ext tar
+# magic in newer, GNU, posix variants
+>257 string =ustar
+# 2 last char of magic and UStar version because string expression does not work
+# 2 space characters followed by a null for GNU variant
+>>261 ubelong =0x72202000 POSIX tar archive (GNU)
+!:mime application/x-gtar
+!:ext tar/gtar
+
+
+# Zip archives (Greg Roelofs, c/o zip-bugs@wkuvx1.wku.edu)
+0 string PK\005\006 Zip archive data (empty)
+0 string PK\003\004 Zip archive data
+!:strength +1
+!:mime application/zip
+!:ext zip/cbz
+
+
+# JAVA
+0 belong 0xcafebabe
+>4 ubelong >30 compiled Java class data,
+!:mime application/x-java-applet
+#!:mime application/java-byte-code
+!:ext class
+
+
+# SHELL scripts
+#0 string/w : shell archive or script for antique kernel text
+0 regex \^#!\\s?(/bin/|/usr/) POSIX shell script text executable
+!:mime text/x-shellscript
+!:ext sh/bash
\ No newline at end of file
diff --git a/solr/core/src/test-files/magic/HelloWorld.java.txt b/solr/core/src/test-files/magic/HelloWorld.java.txt
new file mode 100644
index 00000000000..ca9518d2afa
--- /dev/null
+++ b/solr/core/src/test-files/magic/HelloWorld.java.txt
@@ -0,0 +1,5 @@
+class HelloWorld {
+ public static void main(String[] args) {
+ System.out.println("Hellow world");
+ }
+}
\ No newline at end of file
diff --git a/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin b/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin
new file mode 100644
index 00000000000..e15d0a6c5b9
Binary files /dev/null and b/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin differ
diff --git a/solr/core/src/test-files/magic/README.md b/solr/core/src/test-files/magic/README.md
new file mode 100644
index 00000000000..6e499a2f711
--- /dev/null
+++ b/solr/core/src/test-files/magic/README.md
@@ -0,0 +1,29 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+The two binary files were created by the following commands:
+
+```bash
+echo "Hello" > hello.txt && \
+ tar -cvf hello.tar.bin hello.txt && \
+ rm hello.txt
+
+cp HelloWorld.java.txt HelloWorld.java && \
+ javac HelloWorld.java && \
+ mv HelloWorld.class HelloWorldJavaClass.class.bin && \
+ rm HelloWorld.java
+```
\ No newline at end of file
diff --git a/solr/core/src/test-files/magic/hello.tar.bin b/solr/core/src/test-files/magic/hello.tar.bin
new file mode 100644
index 00000000000..68ca23c362a
Binary files /dev/null and b/solr/core/src/test-files/magic/hello.tar.bin differ
diff --git a/solr/core/src/test-files/magic/plain.txt b/solr/core/src/test-files/magic/plain.txt
new file mode 100644
index 00000000000..70c379b63ff
--- /dev/null
+++ b/solr/core/src/test-files/magic/plain.txt
@@ -0,0 +1 @@
+Hello world
\ No newline at end of file
diff --git a/solr/core/src/test-files/magic/shell.sh.txt b/solr/core/src/test-files/magic/shell.sh.txt
new file mode 100644
index 00000000000..9ea411e111d
--- /dev/null
+++ b/solr/core/src/test-files/magic/shell.sh.txt
@@ -0,0 +1,2 @@
+#! /usr/bin/env bash
+echo Hello
\ No newline at end of file
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
index 1da60d85ff9..f7c9431c296 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
@@ -45,6 +45,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Objects;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -592,14 +593,14 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
assertEquals(
"Can't overwrite an existing configset unless the overwrite parameter is set",
400,
- uploadConfigSet(configsetName, configsetSuffix, null, false, false, v2, false));
+ uploadConfigSet(configsetName, configsetSuffix, null, false, false, v2, false, false));
unIgnoreException("The configuration regulartestOverwrite-1 already exists in zookeeper");
assertEquals(
"Expecting version to remain equal",
solrconfigZkVersion,
getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"));
assertEquals(
- 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));
+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -638,13 +639,14 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
zkClient.makePath(f, true);
}
assertEquals(
- 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));
+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false));
for (String f : extraFiles) {
assertTrue(
"Expecting file " + f + " to exist in ConfigSet but it's gone",
zkClient.exists(f, true));
}
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false));
+ assertEquals(
+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false, false));
for (String f : extraFiles) {
assertFalse(
"Expecting file " + f + " to be deleted from ConfigSet but it wasn't",
@@ -675,7 +677,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
.withConnTimeOut(45000, TimeUnit.MILLISECONDS)
.build()) {
String configPath = "/configs/" + configsetName + configsetSuffix;
- assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, true));
+ assertEquals(
+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, true, false));
for (String fileEnding : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
String f = configPath + "/test." + fileEnding;
assertFalse(
@@ -710,7 +713,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml");
// Was untrusted, overwrite with untrusted
assertEquals(
- 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));
+ 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -721,7 +724,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
// Was untrusted, overwrite with trusted but no cleanup
assertEquals(
- 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false));
+ 0,
+ uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -747,7 +751,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
// Was untrusted, overwrite with trusted with cleanup
assertEquals(
- 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false));
+ 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -761,7 +765,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
assertEquals(
"Can't upload a trusted configset with an untrusted request",
400,
- uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));
+ uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false));
assertEquals(
"Expecting version to remain equal",
solrconfigZkVersion,
@@ -773,7 +777,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
assertEquals(
"Can't upload a trusted configset with an untrusted request",
400,
- uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false));
+ uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false, false));
assertEquals(
"Expecting version to remain equal",
solrconfigZkVersion,
@@ -783,7 +787,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
// Was trusted, overwrite with trusted no cleanup
assertEquals(
- 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false));
+ 0,
+ uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -794,7 +799,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
// Was trusted, overwrite with trusted with cleanup
assertEquals(
- 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false));
+ 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false, false));
assertTrue(
"Expecting version bump",
solrconfigZkVersion
@@ -1457,6 +1462,13 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
.get("id"));
}
+ @Test
+ public void testUploadWithForbiddenContent() throws Exception {
+ // Uploads a config set containing a script, a class file and jar file, will return 400 error
+ long res = uploadConfigSet("forbidden", "suffix", "foo", true, false, true, false, true);
+ assertEquals(400, res);
+ }
+
private static String getSecurityJson() {
return "{\n"
+ " 'authentication':{\n"
@@ -1511,7 +1523,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
String configSetName, String suffix, String username, SolrZkClient zkClient, boolean v2)
throws IOException {
assertFalse(getConfigSetService().checkConfigExists(configSetName + suffix));
- return uploadConfigSet(configSetName, suffix, username, false, false, v2, false);
+ return uploadConfigSet(configSetName, suffix, username, false, false, v2, false, false);
}
private long uploadConfigSet(
@@ -1521,21 +1533,25 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
boolean overwrite,
boolean cleanup,
boolean v2,
- boolean forbiddenTypes)
+ boolean forbiddenTypes,
+ boolean forbiddenContent)
throws IOException {
+ File zipFile;
+ if (forbiddenTypes) {
+ log.info("Uploading configset with forbidden file endings");
+ zipFile =
+ createTempZipFileWithForbiddenTypes(
+ "solr/configsets/upload/" + configSetName + "/solrconfig.xml");
+ } else if (forbiddenContent) {
+ log.info("Uploading configset with forbidden file content");
+ zipFile = createTempZipFileWithForbiddenContent("magic");
+ } else {
+ zipFile = createTempZipFile("solr/configsets/upload/" + configSetName);
+ }
+
// Read zipped sample config
- return uploadGivenConfigSet(
- forbiddenTypes
- ? createTempZipFileWithForbiddenTypes(
- "solr/configsets/upload/" + configSetName + "/solrconfig.xml")
- : createTempZipFile("solr/configsets/upload/" + configSetName),
- configSetName,
- suffix,
- username,
- overwrite,
- cleanup,
- v2);
+ return uploadGivenConfigSet(zipFile, configSetName, suffix, username, overwrite, cleanup, v2);
}
private long uploadBadConfigSet(String configSetName, String suffix, String username, boolean v2)
@@ -1702,31 +1718,68 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
}
}
- private static void zipWithForbiddenEndings(File file, File zipfile) throws IOException {
- OutputStream out = new FileOutputStream(zipfile);
- ZipOutputStream zout = new ZipOutputStream(out);
+ /** Create a zip file (in the temp directory) containing files with forbidden content */
+ private File createTempZipFileWithForbiddenContent(String resourcePath) {
try {
- for (String fileType : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
- zout.putNextEntry(new ZipEntry("test." + fileType));
+ final File zipFile = createTempFile("configset", "zip").toFile();
+ final File directory = SolrTestCaseJ4.getFile(resourcePath);
+ if (log.isInfoEnabled()) {
+ log.info("Directory: {}", directory.getAbsolutePath());
+ }
+ zipWithForbiddenContent(directory, zipFile);
+ if (log.isInfoEnabled()) {
+ log.info("Zipfile: {}", zipFile.getAbsolutePath());
+ }
+ return zipFile;
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
- InputStream in = new FileInputStream(file);
- try {
- byte[] buffer = new byte[1024];
- while (true) {
- int readCount = in.read(buffer);
- if (readCount < 0) {
- break;
+ private static void zipWithForbiddenContent(File directory, File zipfile) throws IOException {
+ OutputStream out = Files.newOutputStream(zipfile.toPath());
+ assertTrue(directory.isDirectory());
+ try (ZipOutputStream zout = new ZipOutputStream(out)) {
+ // Copy in all files from the directory
+ for (File file : Objects.requireNonNull(directory.listFiles())) {
+ zout.putNextEntry(new ZipEntry(file.getName()));
+ zout.write(Files.readAllBytes(file.toPath()));
+ zout.closeEntry();
+ }
+ }
+ }
+
+ private static void zipWithForbiddenEndings(File fileOrDirectory, File zipfile)
+ throws IOException {
+ OutputStream out = new FileOutputStream(zipfile);
+ try (ZipOutputStream zout = new ZipOutputStream(out)) {
+ if (fileOrDirectory.isFile()) {
+ // Create entries with given file, one for each forbidden endding
+ for (String fileType : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
+ zout.putNextEntry(new ZipEntry("test." + fileType));
+
+ try (InputStream in = new FileInputStream(fileOrDirectory)) {
+ byte[] buffer = new byte[1024];
+ while (true) {
+ int readCount = in.read(buffer);
+ if (readCount < 0) {
+ break;
+ }
+ zout.write(buffer, 0, readCount);
}
- zout.write(buffer, 0, readCount);
}
- } finally {
- in.close();
- }
- zout.closeEntry();
+ zout.closeEntry();
+ }
+ }
+ if (fileOrDirectory.isDirectory()) {
+ // Copy in all files from the directory
+ for (File file : Objects.requireNonNull(fileOrDirectory.listFiles())) {
+ zout.putNextEntry(new ZipEntry(file.getName()));
+ zout.write(Files.readAllBytes(file.toPath()));
+ zout.closeEntry();
+ }
}
- } finally {
- zout.close();
}
}
diff --git a/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java b/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java
new file mode 100644
index 00000000000..b8e9a35a3d9
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.util;
+
+import org.apache.solr.SolrTestCaseJ4;
+
+public class FileTypeMagicUtilTest extends SolrTestCaseJ4 {
+ public void testGuessMimeType() {
+ assertEquals(
+ "application/x-java-applet",
+ FileTypeMagicUtil.INSTANCE.guessMimeType(
+ FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin")));
+ assertEquals(
+ "application/zip",
+ FileTypeMagicUtil.INSTANCE.guessMimeType(
+ FileTypeMagicUtil.class.getResourceAsStream(
+ "/runtimecode/containerplugin.v.1.jar.bin")));
+ assertEquals(
+ "application/x-tar",
+ FileTypeMagicUtil.INSTANCE.guessMimeType(
+ FileTypeMagicUtil.class.getResourceAsStream("/magic/hello.tar.bin")));
+ assertEquals(
+ "text/x-shellscript",
+ FileTypeMagicUtil.INSTANCE.guessMimeType(
+ FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt")));
+ }
+
+ public void testIsFileForbiddenInConfigset() {
+ assertTrue(
+ FileTypeMagicUtil.isFileForbiddenInConfigset(
+ FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin")));
+ assertTrue(
+ FileTypeMagicUtil.isFileForbiddenInConfigset(
+ FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt")));
+ assertFalse(
+ FileTypeMagicUtil.isFileForbiddenInConfigset(
+ FileTypeMagicUtil.class.getResourceAsStream("/magic/plain.txt")));
+ }
+}
diff --git a/solr/licenses/simplemagic-1.17.jar.sha1 b/solr/licenses/simplemagic-1.17.jar.sha1
new file mode 100644
index 00000000000..cf101094cc8
--- /dev/null
+++ b/solr/licenses/simplemagic-1.17.jar.sha1
@@ -0,0 +1 @@
+b6e2d1e47d7172e57fa858a2e3940c09a590e61e
diff --git a/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt b/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt
new file mode 100644
index 00000000000..9228230f933
--- /dev/null
+++ b/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt
@@ -0,0 +1,15 @@
+ISC License (https://opensource.org/licenses/ISC)
+
+Copyright 2021, Gray Watson
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
\ No newline at end of file
diff --git a/solr/licenses/simplemagic-NOTICE.txt b/solr/licenses/simplemagic-NOTICE.txt
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
index d571339880c..e40294a6683 100644
--- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
+++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
@@ -348,6 +348,7 @@ public class ZkMaintenanceUtils {
USE_FORBIDDEN_FILE_TYPES);
return FileVisitResult.CONTINUE;
}
+ // TODO: Cannot check MAGIC header for file since FileTypeGuesser is in core
String zkNode = createZkNodeName(zkPath, rootPath, file);
try {
// if the path exists (and presumably we're uploading data to it) just set its data
@@ -437,6 +438,7 @@ public class ZkMaintenanceUtils {
if (isFileForbiddenInConfigSets(zkPath)) {
log.warn("Skipping download of file from ZK, as it is a forbidden type: {}", zkPath);
} else {
+ // TODO: Cannot check MAGIC header for file since FileTypeGuesser is in core
if (copyDataDown(zkClient, zkPath, file) == 0) {
Files.createFile(file);
}
diff --git a/versions.lock b/versions.lock
index 9c0f9ef53d4..6b6ea467e0d 100644
--- a/versions.lock
+++ b/versions.lock
@@ -62,6 +62,7 @@ com.googlecode.plist:dd-plist:1.24 (1 constraints: 300c84f5)
com.healthmarketscience.jackcess:jackcess:4.0.2 (1 constraints: 5d0cf201)
com.healthmarketscience.jackcess:jackcess-encrypt:4.0.1 (1 constraints: 5c0cf101)
com.ibm.icu:icu4j:70.1 (1 constraints: a90f1784)
+com.j256.simplemagic:simplemagic:1.17 (1 constraints: dd04f830)
com.jayway.jsonpath:json-path:2.8.0 (2 constraints: 6c12952c)
com.lmax:disruptor:3.4.4 (1 constraints: 0d050a36)
com.mchange:c3p0:0.9.5.5 (1 constraints: c80c571b)
diff --git a/versions.props b/versions.props
index 10b583c43e2..cfa127e1094 100644
--- a/versions.props
+++ b/versions.props
@@ -13,6 +13,7 @@ com.google.cloud:google-cloud-bom=0.204.0
com.google.errorprone:*=2.23.0
com.google.guava:guava=32.1.3-jre
com.google.re2j:re2j=1.7
+com.j256.simplemagic:simplemagic=1.17
com.jayway.jsonpath:json-path=2.8.0
com.lmax:disruptor=3.4.4
com.tdunning:t-digest=3.1
Re: (solr) branch main updated: SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets
Posted by Jan Høydahl <ja...@cominvent.com>.
Sorry 'bout that, I'm on it.
Jan
> 15. des. 2023 kl. 23:32 skrev Chris Hostetter <ho...@fucit.org>:
>
>
> This change, and it's associated backports, seem to have broken
> TestConfigSetService (regardless of seed?) due to leaked files.
>
> https://ci-builds.apache.org/job/Solr/job/Solr-Check-main/8576/consoleText
> https://jenkins.thetaphi.de/view/Solr/job/Solr-main-Windows/3706/consoleText
> https://ci-builds.apache.org/job/Solr/job/Solr-Check-9.x/39/consoleText
> https://ci-builds.apache.org/job/Solr/job/Solr-check-9.4/1208/consoleText
>
>
>
> : Date: Wed, 13 Dec 2023 22:08:04 +0000
> : From: janhoy@apache.org
> : Reply-To: dev@solr.apache.org
> : To: "commits@solr.apache.org" <co...@solr.apache.org>
> : Subject: (solr) branch main updated: SOLR-16949: Restrict certain file types
> : from being uploaded to or downloaded from Config Sets
> :
> : This is an automated email from the ASF dual-hosted git repository.
> :
> : janhoy pushed a commit to branch main
> : in repository https://gitbox.apache.org/repos/asf/solr.git
> :
> :
> : The following commit(s) were added to refs/heads/main by this push:
> : new 15534754f49 SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets
> : 15534754f49 is described below
> :
> : commit 15534754f492079e52288dd11abaf1c4261b3ea4
> : Author: Jan Høydahl <ja...@apache.org>
> : AuthorDate: Wed Dec 13 22:49:23 2023 +0100
> :
> : SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets
> : ---
> : solr/CHANGES.txt | 2 +
> : solr/core/build.gradle | 2 +
> : .../org/apache/solr/cli/ConfigSetUploadTool.java | 2 +
> : .../org/apache/solr/cloud/ZkConfigSetService.java | 21 ++-
> : .../solr/core/FileSystemConfigSetService.java | 28 +++-
> : .../org/apache/solr/core/backup/BackupManager.java | 23 ++-
> : .../handler/configsets/UploadConfigSetFileAPI.java | 8 +-
> : .../org/apache/solr/util/FileTypeMagicUtil.java | 166 +++++++++++++++++++++
> : solr/core/src/resources/magic/executables | 74 +++++++++
> : solr/core/src/test-files/magic/HelloWorld.java.txt | 5 +
> : .../test-files/magic/HelloWorldJavaClass.class.bin | Bin 0 -> 426 bytes
> : solr/core/src/test-files/magic/README.md | 29 ++++
> : solr/core/src/test-files/magic/hello.tar.bin | Bin 0 -> 4096 bytes
> : solr/core/src/test-files/magic/plain.txt | 1 +
> : solr/core/src/test-files/magic/shell.sh.txt | 2 +
> : .../org/apache/solr/cloud/TestConfigSetsAPI.java | 141 +++++++++++------
> : .../apache/solr/util/FileTypeMagicUtilTest.java | 54 +++++++
> : solr/licenses/simplemagic-1.17.jar.sha1 | 1 +
> : solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt | 15 ++
> : solr/licenses/simplemagic-NOTICE.txt | 0
> : .../solr/common/cloud/ZkMaintenanceUtils.java | 2 +
> : versions.lock | 1 +
> : versions.props | 1 +
> : 23 files changed, 522 insertions(+), 56 deletions(-)
> :
> : diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
> : index 40da8c435ac..2fd94304a24 100644
> : --- a/solr/CHANGES.txt
> : +++ b/solr/CHANGES.txt
> : @@ -169,6 +169,8 @@ Other Changes
> : * SOLR-17091: dev tools script cloud.sh became broken after changes in 9.3 added a new -slim.tgz file it was not expecting
> : cloud.sh has been updated to ignore the -slim.tgz version of the tarball.
> :
> : +* SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets (janhoy, Houston Putman)
> : +
> : ================== 9.4.0 ==================
> : New Features
> : ---------------------
> : diff --git a/solr/core/build.gradle b/solr/core/build.gradle
> : index 61ecd1713af..ed2c8a370ae 100644
> : --- a/solr/core/build.gradle
> : +++ b/solr/core/build.gradle
> : @@ -159,6 +159,8 @@ dependencies {
> :
> : compileOnly 'com.github.stephenc.jcip:jcip-annotations'
> :
> : + implementation 'com.j256.simplemagic:simplemagic'
> : +
> : // -- Test Dependencies
> :
> : testRuntimeOnly 'org.slf4j:jcl-over-slf4j'
> : diff --git a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
> : index 5fd4a538bd7..6576742a195 100644
> : --- a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
> : +++ b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
> : @@ -27,6 +27,7 @@ import org.apache.solr.client.solrj.impl.SolrZkClientTimeout;
> : import org.apache.solr.common.cloud.SolrZkClient;
> : import org.apache.solr.common.cloud.ZkMaintenanceUtils;
> : import org.apache.solr.core.ConfigSetService;
> : +import org.apache.solr.util.FileTypeMagicUtil;
> : import org.slf4j.Logger;
> : import org.slf4j.LoggerFactory;
> :
> : @@ -101,6 +102,7 @@ public class ConfigSetUploadTool extends ToolBase {
> : + cli.getOptionValue("confname")
> : + " to ZooKeeper at "
> : + zkHost);
> : + FileTypeMagicUtil.assertConfigSetFolderLegal(confPath);
> : ZkMaintenanceUtils.uploadToZK(
> : zkClient,
> : confPath,
> : diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
> : index f02404d636d..9abde098e1c 100644
> : --- a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
> : +++ b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
> : @@ -22,6 +22,7 @@ import java.nio.file.Path;
> : import java.util.ArrayList;
> : import java.util.Collections;
> : import java.util.List;
> : +import java.util.Locale;
> : import java.util.Map;
> : import java.util.Objects;
> : import org.apache.solr.client.solrj.cloud.SolrCloudManager;
> : @@ -39,6 +40,7 @@ import org.apache.solr.core.CoreContainer;
> : import org.apache.solr.core.CoreDescriptor;
> : import org.apache.solr.core.SolrConfig;
> : import org.apache.solr.core.SolrResourceLoader;
> : +import org.apache.solr.util.FileTypeMagicUtil;
> : import org.apache.zookeeper.CreateMode;
> : import org.apache.zookeeper.KeeperException;
> : import org.apache.zookeeper.data.Stat;
> : @@ -199,6 +201,15 @@ public class ZkConfigSetService extends ConfigSetService {
> : try {
> : if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {
> : log.warn("Not including uploading file to config, as it is a forbidden type: {}", fileName);
> : + } else if (FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
> : + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
> : + throw new SolrException(
> : + SolrException.ErrorCode.BAD_REQUEST,
> : + String.format(
> : + Locale.ROOT,
> : + "Not uploading file %s to config, as it matched the MAGIC signature of a forbidden mime type %s",
> : + fileName,
> : + mimeType));
> : } else {
> : // if overwriteOnExists is true then zkClient#makePath failOnExists is set to false
> : zkClient.makePath(filePath, data, CreateMode.PERSISTENT, null, !overwriteOnExists, true);
> : @@ -340,7 +351,15 @@ public class ZkConfigSetService extends ConfigSetService {
> : } else {
> : log.debug("Copying zk node {} to {}", fromZkFilePath, toZkFilePath);
> : byte[] data = zkClient.getData(fromZkFilePath, null, null, true);
> : - zkClient.makePath(toZkFilePath, data, true);
> : + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
> : + zkClient.makePath(toZkFilePath, data, true);
> : + } else {
> : + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
> : + log.warn(
> : + "Skipping copy of file {} in ZK, as it matched the MAGIC signature of a forbidden mime type {}",
> : + fromZkFilePath,
> : + mimeType);
> : + }
> : }
> : }
> :
> : diff --git a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
> : index 1f1a42b15e1..4b041252211 100644
> : --- a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
> : +++ b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
> : @@ -37,6 +37,7 @@ import java.util.stream.Stream;
> : import org.apache.solr.common.SolrException;
> : import org.apache.solr.common.cloud.ZkMaintenanceUtils;
> : import org.apache.solr.common.util.Utils;
> : +import org.apache.solr.util.FileTypeMagicUtil;
> : import org.slf4j.Logger;
> : import org.slf4j.LoggerFactory;
> :
> : @@ -150,9 +151,17 @@ public class FileSystemConfigSetService extends ConfigSetService {
> : if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {
> : log.warn("Not including uploading file to config, as it is a forbidden type: {}", fileName);
> : } else {
> : - Path filePath = getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));
> : - if (!Files.exists(filePath) || overwriteOnExists) {
> : - Files.write(filePath, data);
> : + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
> : + Path filePath = getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));
> : + if (!Files.exists(filePath) || overwriteOnExists) {
> : + Files.write(filePath, data);
> : + }
> : + } else {
> : + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
> : + log.warn(
> : + "Not including uploading file {}, as it matched the MAGIC signature of a forbidden mime type {}",
> : + fileName,
> : + mimeType);
> : }
> : }
> : }
> : @@ -205,8 +214,17 @@ public class FileSystemConfigSetService extends ConfigSetService {
> : "Not including uploading file to config, as it is a forbidden type: {}",
> : file.getFileName());
> : } else {
> : - Files.copy(
> : - file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);
> : + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(Files.newInputStream(file))) {
> : + Files.copy(
> : + file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);
> : + } else {
> : + String mimeType =
> : + FileTypeMagicUtil.INSTANCE.guessMimeType(Files.newInputStream(file));
> : + log.warn(
> : + "Not copying file {}, as it matched the MAGIC signature of a forbidden mime type {}",
> : + file.getFileName(),
> : + mimeType);
> : + }
> : }
> : return FileVisitResult.CONTINUE;
> : }
> : diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
> : index be6a1a83c2f..8ff78b27e08 100644
> : --- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
> : +++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
> : @@ -40,6 +40,7 @@ import org.apache.solr.common.cloud.ZkStateReader;
> : import org.apache.solr.common.util.Utils;
> : import org.apache.solr.core.ConfigSetService;
> : import org.apache.solr.core.backup.repository.BackupRepository;
> : +import org.apache.solr.util.FileTypeMagicUtil;
> : import org.apache.zookeeper.CreateMode;
> : import org.apache.zookeeper.KeeperException;
> : import org.slf4j.Logger;
> : @@ -349,8 +350,16 @@ public class BackupManager {
> : if (data == null) {
> : data = new byte[0];
> : }
> : - try (OutputStream os = repository.createOutput(uri)) {
> : - os.write(data);
> : + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
> : + try (OutputStream os = repository.createOutput(uri)) {
> : + os.write(data);
> : + }
> : + } else {
> : + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
> : + log.warn(
> : + "Not including zookeeper file {} in backup, as it matched the MAGIC signature of a forbidden mime type {}",
> : + filePath,
> : + mimeType);
> : }
> : }
> : } else {
> : @@ -379,7 +388,15 @@ public class BackupManager {
> : // probably ok since the config file should be small.
> : byte[] arr = new byte[(int) is.length()];
> : is.readBytes(arr, 0, (int) is.length());
> : - configSetService.uploadFileToConfig(configName, filePath, arr, false);
> : + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(arr)) {
> : + configSetService.uploadFileToConfig(configName, filePath, arr, false);
> : + } else {
> : + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(arr);
> : + log.warn(
> : + "Not including zookeeper file {} in restore, as it matched the MAGIC signature of a forbidden mime type {}",
> : + filePath,
> : + mimeType);
> : + }
> : }
> : }
> : break;
> : diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
> : index 98889aff563..2380a79a92b 100644
> : --- a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
> : +++ b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
> : @@ -27,6 +27,7 @@ import org.apache.solr.common.params.ConfigSetParams;
> : import org.apache.solr.core.CoreContainer;
> : import org.apache.solr.request.SolrQueryRequest;
> : import org.apache.solr.response.SolrQueryResponse;
> : +import org.apache.solr.util.FileTypeMagicUtil;
> :
> : /**
> : * V2 API for adding or updating a single file within a configset.
> : @@ -67,11 +68,13 @@ public class UploadConfigSetFileAPI extends ConfigSetAPIBase {
> : if (fixedSingleFilePath.charAt(0) == '/') {
> : fixedSingleFilePath = fixedSingleFilePath.substring(1);
> : }
> : + byte[] data = inputStream.readAllBytes();
> : if (fixedSingleFilePath.isEmpty()) {
> : throw new SolrException(
> : SolrException.ErrorCode.BAD_REQUEST,
> : "The file path provided for upload, '" + singleFilePath + "', is not valid.");
> : - } else if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fixedSingleFilePath)) {
> : + } else if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fixedSingleFilePath)
> : + || FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
> : throw new SolrException(
> : SolrException.ErrorCode.BAD_REQUEST,
> : "The file type provided for upload, '"
> : @@ -87,8 +90,7 @@ public class UploadConfigSetFileAPI extends ConfigSetAPIBase {
> : // For creating the baseNode, the cleanup parameter is only allowed to be true when
> : // singleFilePath is not passed.
> : createBaseNode(configSetService, overwritesExisting, requestIsTrusted, configSetName);
> : - configSetService.uploadFileToConfig(
> : - configSetName, fixedSingleFilePath, inputStream.readAllBytes(), allowOverwrite);
> : + configSetService.uploadFileToConfig(configSetName, fixedSingleFilePath, data, allowOverwrite);
> : }
> : }
> : }
> : diff --git a/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java b/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java
> : new file mode 100644
> : index 00000000000..cfb6c9fa0af
> : --- /dev/null
> : +++ b/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java
> : @@ -0,0 +1,166 @@
> : +/*
> : + * Licensed to the Apache Software Foundation (ASF) under one or more
> : + * contributor license agreements. See the NOTICE file distributed with
> : + * this work for additional information regarding copyright ownership.
> : + * The ASF licenses this file to You under the Apache License, Version 2.0
> : + * (the "License"); you may not use this file except in compliance with
> : + * the License. You may obtain a copy of the License at
> : + *
> : + * http://www.apache.org/licenses/LICENSE-2.0
> : + *
> : + * Unless required by applicable law or agreed to in writing, software
> : + * distributed under the License is distributed on an "AS IS" BASIS,
> : + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> : + * See the License for the specific language governing permissions and
> : + * limitations under the License.
> : + */
> : +
> : +package org.apache.solr.util;
> : +
> : +import com.j256.simplemagic.ContentInfo;
> : +import com.j256.simplemagic.ContentInfoUtil;
> : +import com.j256.simplemagic.ContentType;
> : +import java.io.ByteArrayInputStream;
> : +import java.io.IOException;
> : +import java.io.InputStream;
> : +import java.nio.file.FileVisitResult;
> : +import java.nio.file.Files;
> : +import java.nio.file.Path;
> : +import java.nio.file.SimpleFileVisitor;
> : +import java.nio.file.attribute.BasicFileAttributes;
> : +import java.util.Arrays;
> : +import java.util.HashSet;
> : +import java.util.Locale;
> : +import java.util.Set;
> : +import org.apache.solr.common.SolrException;
> : +
> : +/** Utility class to guess the mime type of file based on its magic number. */
> : +public class FileTypeMagicUtil implements ContentInfoUtil.ErrorCallBack {
> : + private final ContentInfoUtil util;
> : + private static final Set<String> SKIP_FOLDERS = new HashSet<>(Arrays.asList(".", ".."));
> : +
> : + public static FileTypeMagicUtil INSTANCE = new FileTypeMagicUtil();
> : +
> : + FileTypeMagicUtil() {
> : + try {
> : + util = new ContentInfoUtil("/magic/executables", this);
> : + } catch (IOException e) {
> : + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error parsing magic file", e);
> : + }
> : + }
> : +
> : + /**
> : + * Asserts that an entire configset folder is legal to upload.
> : + *
> : + * @param confPath the path to the folder
> : + * @throws SolrException if an illegal file is found in the folder structure
> : + */
> : + public static void assertConfigSetFolderLegal(Path confPath) throws IOException {
> : + Files.walkFileTree(
> : + confPath,
> : + new SimpleFileVisitor<Path>() {
> : + @Override
> : + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
> : + throws IOException {
> : + // Read first 100 bytes of the file to determine the mime type
> : + try (InputStream fileStream = Files.newInputStream(file)) {
> : + byte[] bytes = new byte[100];
> : + fileStream.read(bytes);
> : + if (FileTypeMagicUtil.isFileForbiddenInConfigset(bytes)) {
> : + throw new SolrException(
> : + SolrException.ErrorCode.BAD_REQUEST,
> : + String.format(
> : + Locale.ROOT,
> : + "Not uploading file %s to configset, as it matched the MAGIC signature of a forbidden mime type %s",
> : + file,
> : + FileTypeMagicUtil.INSTANCE.guessMimeType(bytes)));
> : + }
> : + return FileVisitResult.CONTINUE;
> : + }
> : + }
> : +
> : + @Override
> : + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
> : + throws IOException {
> : + if (SKIP_FOLDERS.contains(dir.getFileName().toString()))
> : + return FileVisitResult.SKIP_SUBTREE;
> : +
> : + return FileVisitResult.CONTINUE;
> : + }
> : + });
> : + }
> : +
> : + /**
> : + * Guess the mime type of file based on its magic number.
> : + *
> : + * @param stream input stream of the file
> : + * @return string with content-type or "application/octet-stream" if unknown
> : + */
> : + public String guessMimeType(InputStream stream) {
> : + try {
> : + ContentInfo info = util.findMatch(stream);
> : + if (info == null) {
> : + return ContentType.OTHER.getMimeType();
> : + }
> : + return info.getContentType().getMimeType();
> : + } catch (IOException e) {
> : + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
> : + }
> : + }
> : +
> : + /**
> : + * Guess the mime type of file bytes based on its magic number.
> : + *
> : + * @param bytes the first bytes at start of the file
> : + * @return string with content-type or "application/octet-stream" if unknown
> : + */
> : + public String guessMimeType(byte[] bytes) {
> : + return guessMimeType(new ByteArrayInputStream(bytes));
> : + }
> : +
> : + @Override
> : + public void error(String line, String details, Exception e) {
> : + throw new SolrException(
> : + SolrException.ErrorCode.SERVER_ERROR,
> : + String.format(Locale.ROOT, "%s: %s", line, details),
> : + e);
> : + }
> : +
> : + /**
> : + * Determine forbidden file type based on magic bytes matching of the file itself. Forbidden types
> : + * are:
> : + *
> : + * <ul>
> : + * <li><code>application/x-java-applet</code>: java class file
> : + * <li><code>application/zip</code>: jar or zip archives
> : + * <li><code>application/x-tar</code>: tar archives
> : + * <li><code>text/x-shellscript</code>: shell or bash script
> : + * </ul>
> : + *
> : + * @param fileStream stream from the file content
> : + * @return true if file is among the forbidden mime-types
> : + */
> : + public static boolean isFileForbiddenInConfigset(InputStream fileStream) {
> : + return forbiddenTypes.contains(FileTypeMagicUtil.INSTANCE.guessMimeType(fileStream));
> : + }
> : +
> : + /**
> : + * Determine forbidden file type based on magic bytes matching of the first bytes of the file.
> : + *
> : + * @param bytes byte array of the file content
> : + * @return true if file is among the forbidden mime-types
> : + */
> : + public static boolean isFileForbiddenInConfigset(byte[] bytes) {
> : + if (bytes == null || bytes.length == 0)
> : + return false; // A ZK znode may be a folder with no content
> : + return isFileForbiddenInConfigset(new ByteArrayInputStream(bytes));
> : + }
> : +
> : + private static final Set<String> forbiddenTypes =
> : + new HashSet<>(
> : + Arrays.asList(
> : + System.getProperty(
> : + "solr.configset.upload.mimetypes.forbidden",
> : + "application/x-java-applet,application/zip,application/x-tar,text/x-shellscript")
> : + .split(",")));
> : +}
> : diff --git a/solr/core/src/resources/magic/executables b/solr/core/src/resources/magic/executables
> : new file mode 100644
> : index 00000000000..04094eaf797
> : --- /dev/null
> : +++ b/solr/core/src/resources/magic/executables
> : @@ -0,0 +1,74 @@
> : +# Licensed to the Apache Software Foundation (ASF) under one or more
> : +# contributor license agreements. See the NOTICE file distributed with
> : +# this work for additional information regarding copyright ownership.
> : +# The ASF licenses this file to You under the Apache License, Version 2.0
> : +# (the "License"); you may not use this file except in compliance with
> : +# the License. You may obtain a copy of the License at
> : +#
> : +# http://www.apache.org/licenses/LICENSE-2.0
> : +#
> : +# Unless required by applicable law or agreed to in writing, software
> : +# distributed under the License is distributed on an "AS IS" BASIS,
> : +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> : +# See the License for the specific language governing permissions and
> : +# limitations under the License.
> : +
> : +# POSIX tar archives
> : +# URL: https://en.wikipedia.org/wiki/Tar_(computing)
> : +# Reference: https://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5&manpath=FreeBSD+8-current
> : +# header mainly padded with nul bytes
> : +500 quad 0
> : +!:strength /2
> : +# filename or extended attribute printable strings in range space null til umlaut ue
> : +>0 ubeshort >0x1F00
> : +>>0 ubeshort <0xFCFD
> : +# last 4 header bytes often null but tar\0 in gtarfail2.tar gtarfail.tar-bad
> : +# at https://sourceforge.net/projects/s-tar/files/testscripts/
> : +>>>508 ubelong&0x8B9E8DFF 0
> : +# nul, space or ascii digit 0-7 at start of mode
> : +>>>>100 ubyte&0xC8 =0
> : +>>>>>101 ubyte&0xC8 =0
> : +# nul, space at end of check sum
> : +>>>>>>155 ubyte&0xDF =0
> : +# space or ascii digit 0 at start of check sum
> : +>>>>>>>148 ubyte&0xEF =0x20
> : +# check for specific 1st member name that indicates other mime type and file name suffix
> : +>>>>>>>>0 string TpmEmuTpms/permall
> : +!:mime application/x-tar
> : +!:ext tar
> : +# other stuff in padding
> : +# some implementations add new fields to the blank area at the end of the header record
> : +# created for example by DOS TAR 3.20g 1994 Tim V.Shapore with -j option
> : +>>257 ulong !0 tar archive (old)
> : +!:mime application/x-tar
> : +!:ext tar
> : +# magic in newer, GNU, posix variants
> : +>257 string =ustar
> : +# 2 last char of magic and UStar version because string expression does not work
> : +# 2 space characters followed by a null for GNU variant
> : +>>261 ubelong =0x72202000 POSIX tar archive (GNU)
> : +!:mime application/x-gtar
> : +!:ext tar/gtar
> : +
> : +
> : +# Zip archives (Greg Roelofs, c/o zip-bugs@wkuvx1.wku.edu)
> : +0 string PK\005\006 Zip archive data (empty)
> : +0 string PK\003\004 Zip archive data
> : +!:strength +1
> : +!:mime application/zip
> : +!:ext zip/cbz
> : +
> : +
> : +# JAVA
> : +0 belong 0xcafebabe
> : +>4 ubelong >30 compiled Java class data,
> : +!:mime application/x-java-applet
> : +#!:mime application/java-byte-code
> : +!:ext class
> : +
> : +
> : +# SHELL scripts
> : +#0 string/w : shell archive or script for antique kernel text
> : +0 regex \^#!\\s?(/bin/|/usr/) POSIX shell script text executable
> : +!:mime text/x-shellscript
> : +!:ext sh/bash
> : \ No newline at end of file
> : diff --git a/solr/core/src/test-files/magic/HelloWorld.java.txt b/solr/core/src/test-files/magic/HelloWorld.java.txt
> : new file mode 100644
> : index 00000000000..ca9518d2afa
> : --- /dev/null
> : +++ b/solr/core/src/test-files/magic/HelloWorld.java.txt
> : @@ -0,0 +1,5 @@
> : +class HelloWorld {
> : + public static void main(String[] args) {
> : + System.out.println("Hellow world");
> : + }
> : +}
> : \ No newline at end of file
> : diff --git a/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin b/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin
> : new file mode 100644
> : index 00000000000..e15d0a6c5b9
> : Binary files /dev/null and b/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin differ
> : diff --git a/solr/core/src/test-files/magic/README.md b/solr/core/src/test-files/magic/README.md
> : new file mode 100644
> : index 00000000000..6e499a2f711
> : --- /dev/null
> : +++ b/solr/core/src/test-files/magic/README.md
> : @@ -0,0 +1,29 @@
> : +<!--
> : + Licensed to the Apache Software Foundation (ASF) under one or more
> : + contributor license agreements. See the NOTICE file distributed with
> : + this work for additional information regarding copyright ownership.
> : + The ASF licenses this file to You under the Apache License, Version 2.0
> : + the "License"); you may not use this file except in compliance with
> : + the License. You may obtain a copy of the License at
> : +
> : + http://www.apache.org/licenses/LICENSE-2.0
> : +
> : + Unless required by applicable law or agreed to in writing, software
> : + distributed under the License is distributed on an "AS IS" BASIS,
> : + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> : + See the License for the specific language governing permissions and
> : + limitations under the License.
> : + -->
> : +
> : +The two binary files were created by the following commands:
> : +
> : +```bash
> : +echo "Hello" > hello.txt && \
> : + tar -cvf hello.tar.bin hello.txt && \
> : + rm hello.txt
> : +
> : +cp HelloWorld.java.txt HelloWorld.java && \
> : + javac HelloWorld.java && \
> : + mv HelloWorld.class HelloWorldJavaClass.class.bin && \
> : + rm HelloWorld.java
> : +```
> : \ No newline at end of file
> : diff --git a/solr/core/src/test-files/magic/hello.tar.bin b/solr/core/src/test-files/magic/hello.tar.bin
> : new file mode 100644
> : index 00000000000..68ca23c362a
> : Binary files /dev/null and b/solr/core/src/test-files/magic/hello.tar.bin differ
> : diff --git a/solr/core/src/test-files/magic/plain.txt b/solr/core/src/test-files/magic/plain.txt
> : new file mode 100644
> : index 00000000000..70c379b63ff
> : --- /dev/null
> : +++ b/solr/core/src/test-files/magic/plain.txt
> : @@ -0,0 +1 @@
> : +Hello world
> : \ No newline at end of file
> : diff --git a/solr/core/src/test-files/magic/shell.sh.txt b/solr/core/src/test-files/magic/shell.sh.txt
> : new file mode 100644
> : index 00000000000..9ea411e111d
> : --- /dev/null
> : +++ b/solr/core/src/test-files/magic/shell.sh.txt
> : @@ -0,0 +1,2 @@
> : +#! /usr/bin/env bash
> : +echo Hello
> : \ No newline at end of file
> : diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
> : index 1da60d85ff9..f7c9431c296 100644
> : --- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
> : +++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
> : @@ -45,6 +45,7 @@ import java.util.LinkedHashMap;
> : import java.util.List;
> : import java.util.Locale;
> : import java.util.Map;
> : +import java.util.Objects;
> : import java.util.Properties;
> : import java.util.Set;
> : import java.util.concurrent.TimeUnit;
> : @@ -592,14 +593,14 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
> : assertEquals(
> : "Can't overwrite an existing configset unless the overwrite parameter is set",
> : 400,
> : - uploadConfigSet(configsetName, configsetSuffix, null, false, false, v2, false));
> : + uploadConfigSet(configsetName, configsetSuffix, null, false, false, v2, false, false));
> : unIgnoreException("The configuration regulartestOverwrite-1 already exists in zookeeper");
> : assertEquals(
> : "Expecting version to remain equal",
> : solrconfigZkVersion,
> : getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"));
> : assertEquals(
> : - 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));
> : + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false));
> : assertTrue(
> : "Expecting version bump",
> : solrconfigZkVersion
> : @@ -638,13 +639,14 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
> : zkClient.makePath(f, true);
> : }
> : assertEquals(
> : - 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));
> : + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false));
> : for (String f : extraFiles) {
> : assertTrue(
> : "Expecting file " + f + " to exist in ConfigSet but it's gone",
> : zkClient.exists(f, true));
> : }
> : - assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false));
> : + assertEquals(
> : + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false, false));
> : for (String f : extraFiles) {
> : assertFalse(
> : "Expecting file " + f + " to be deleted from ConfigSet but it wasn't",
> : @@ -675,7 +677,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
> : .withConnTimeOut(45000, TimeUnit.MILLISECONDS)
> : .build()) {
> : String configPath = "/configs/" + configsetName + configsetSuffix;
> : - assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, true));
> : + assertEquals(
> : + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, true, false));
> : for (String fileEnding : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
> : String f = configPath + "/test." + fileEnding;
> : assertFalse(
> : @@ -710,7 +713,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
> : getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml");
> : // Was untrusted, overwrite with untrusted
> : assertEquals(
> : - 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));
> : + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false));
> : assertTrue(
> : "Expecting version bump",
> : solrconfigZkVersion
> : @@ -721,7 +724,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
> :
> : // Was untrusted, overwrite with trusted but no cleanup
> : assertEquals(
> : - 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false));
> : + 0,
> : + uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false, false));
> : assertTrue(
> : "Expecting version bump",
> : solrconfigZkVersion
> : @@ -747,7 +751,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
> :
> : // Was untrusted, overwrite with trusted with cleanup
> : assertEquals(
> : - 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false));
> : + 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false, false));
> : assertTrue(
> : "Expecting version bump",
> : solrconfigZkVersion
> : @@ -761,7 +765,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
> : assertEquals(
> : "Can't upload a trusted configset with an untrusted request",
> : 400,
> : - uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));
> : + uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false));
> : assertEquals(
> : "Expecting version to remain equal",
> : solrconfigZkVersion,
> : @@ -773,7 +777,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
> : assertEquals(
> : "Can't upload a trusted configset with an untrusted request",
> : 400,
> : - uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false));
> : + uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false, false));
> : assertEquals(
> : "Expecting version to remain equal",
> : solrconfigZkVersion,
> : @@ -783,7 +787,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
> :
> : // Was trusted, overwrite with trusted no cleanup
> : assertEquals(
> : - 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false));
> : + 0,
> : + uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false, false));
> : assertTrue(
> : "Expecting version bump",
> : solrconfigZkVersion
> : @@ -794,7 +799,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
> :
> : // Was trusted, overwrite with trusted with cleanup
> : assertEquals(
> : - 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false));
> : + 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false, false));
> : assertTrue(
> : "Expecting version bump",
> : solrconfigZkVersion
> : @@ -1457,6 +1462,13 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
> : .get("id"));
> : }
> :
> : + @Test
> : + public void testUploadWithForbiddenContent() throws Exception {
> : + // Uploads a config set containing a script, a class file and jar file, will return 400 error
> : + long res = uploadConfigSet("forbidden", "suffix", "foo", true, false, true, false, true);
> : + assertEquals(400, res);
> : + }
> : +
> : private static String getSecurityJson() {
> : return "{\n"
> : + " 'authentication':{\n"
> : @@ -1511,7 +1523,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
> : String configSetName, String suffix, String username, SolrZkClient zkClient, boolean v2)
> : throws IOException {
> : assertFalse(getConfigSetService().checkConfigExists(configSetName + suffix));
> : - return uploadConfigSet(configSetName, suffix, username, false, false, v2, false);
> : + return uploadConfigSet(configSetName, suffix, username, false, false, v2, false, false);
> : }
> :
> : private long uploadConfigSet(
> : @@ -1521,21 +1533,25 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
> : boolean overwrite,
> : boolean cleanup,
> : boolean v2,
> : - boolean forbiddenTypes)
> : + boolean forbiddenTypes,
> : + boolean forbiddenContent)
> : throws IOException {
> :
> : + File zipFile;
> : + if (forbiddenTypes) {
> : + log.info("Uploading configset with forbidden file endings");
> : + zipFile =
> : + createTempZipFileWithForbiddenTypes(
> : + "solr/configsets/upload/" + configSetName + "/solrconfig.xml");
> : + } else if (forbiddenContent) {
> : + log.info("Uploading configset with forbidden file content");
> : + zipFile = createTempZipFileWithForbiddenContent("magic");
> : + } else {
> : + zipFile = createTempZipFile("solr/configsets/upload/" + configSetName);
> : + }
> : +
> : // Read zipped sample config
> : - return uploadGivenConfigSet(
> : - forbiddenTypes
> : - ? createTempZipFileWithForbiddenTypes(
> : - "solr/configsets/upload/" + configSetName + "/solrconfig.xml")
> : - : createTempZipFile("solr/configsets/upload/" + configSetName),
> : - configSetName,
> : - suffix,
> : - username,
> : - overwrite,
> : - cleanup,
> : - v2);
> : + return uploadGivenConfigSet(zipFile, configSetName, suffix, username, overwrite, cleanup, v2);
> : }
> :
> : private long uploadBadConfigSet(String configSetName, String suffix, String username, boolean v2)
> : @@ -1702,31 +1718,68 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
> : }
> : }
> :
> : - private static void zipWithForbiddenEndings(File file, File zipfile) throws IOException {
> : - OutputStream out = new FileOutputStream(zipfile);
> : - ZipOutputStream zout = new ZipOutputStream(out);
> : + /** Create a zip file (in the temp directory) containing files with forbidden content */
> : + private File createTempZipFileWithForbiddenContent(String resourcePath) {
> : try {
> : - for (String fileType : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
> : - zout.putNextEntry(new ZipEntry("test." + fileType));
> : + final File zipFile = createTempFile("configset", "zip").toFile();
> : + final File directory = SolrTestCaseJ4.getFile(resourcePath);
> : + if (log.isInfoEnabled()) {
> : + log.info("Directory: {}", directory.getAbsolutePath());
> : + }
> : + zipWithForbiddenContent(directory, zipFile);
> : + if (log.isInfoEnabled()) {
> : + log.info("Zipfile: {}", zipFile.getAbsolutePath());
> : + }
> : + return zipFile;
> : + } catch (IOException e) {
> : + throw new RuntimeException(e);
> : + }
> : + }
> :
> : - InputStream in = new FileInputStream(file);
> : - try {
> : - byte[] buffer = new byte[1024];
> : - while (true) {
> : - int readCount = in.read(buffer);
> : - if (readCount < 0) {
> : - break;
> : + private static void zipWithForbiddenContent(File directory, File zipfile) throws IOException {
> : + OutputStream out = Files.newOutputStream(zipfile.toPath());
> : + assertTrue(directory.isDirectory());
> : + try (ZipOutputStream zout = new ZipOutputStream(out)) {
> : + // Copy in all files from the directory
> : + for (File file : Objects.requireNonNull(directory.listFiles())) {
> : + zout.putNextEntry(new ZipEntry(file.getName()));
> : + zout.write(Files.readAllBytes(file.toPath()));
> : + zout.closeEntry();
> : + }
> : + }
> : + }
> : +
> : + private static void zipWithForbiddenEndings(File fileOrDirectory, File zipfile)
> : + throws IOException {
> : + OutputStream out = new FileOutputStream(zipfile);
> : + try (ZipOutputStream zout = new ZipOutputStream(out)) {
> : + if (fileOrDirectory.isFile()) {
> : + // Create entries with given file, one for each forbidden endding
> : + for (String fileType : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
> : + zout.putNextEntry(new ZipEntry("test." + fileType));
> : +
> : + try (InputStream in = new FileInputStream(fileOrDirectory)) {
> : + byte[] buffer = new byte[1024];
> : + while (true) {
> : + int readCount = in.read(buffer);
> : + if (readCount < 0) {
> : + break;
> : + }
> : + zout.write(buffer, 0, readCount);
> : }
> : - zout.write(buffer, 0, readCount);
> : }
> : - } finally {
> : - in.close();
> : - }
> :
> : - zout.closeEntry();
> : + zout.closeEntry();
> : + }
> : + }
> : + if (fileOrDirectory.isDirectory()) {
> : + // Copy in all files from the directory
> : + for (File file : Objects.requireNonNull(fileOrDirectory.listFiles())) {
> : + zout.putNextEntry(new ZipEntry(file.getName()));
> : + zout.write(Files.readAllBytes(file.toPath()));
> : + zout.closeEntry();
> : + }
> : }
> : - } finally {
> : - zout.close();
> : }
> : }
> :
> : diff --git a/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java b/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java
> : new file mode 100644
> : index 00000000000..b8e9a35a3d9
> : --- /dev/null
> : +++ b/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java
> : @@ -0,0 +1,54 @@
> : +/*
> : + * Licensed to the Apache Software Foundation (ASF) under one or more
> : + * contributor license agreements. See the NOTICE file distributed with
> : + * this work for additional information regarding copyright ownership.
> : + * The ASF licenses this file to You under the Apache License, Version 2.0
> : + * (the "License"); you may not use this file except in compliance with
> : + * the License. You may obtain a copy of the License at
> : + *
> : + * http://www.apache.org/licenses/LICENSE-2.0
> : + *
> : + * Unless required by applicable law or agreed to in writing, software
> : + * distributed under the License is distributed on an "AS IS" BASIS,
> : + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> : + * See the License for the specific language governing permissions and
> : + * limitations under the License.
> : + */
> : +
> : +package org.apache.solr.util;
> : +
> : +import org.apache.solr.SolrTestCaseJ4;
> : +
> : +public class FileTypeMagicUtilTest extends SolrTestCaseJ4 {
> : + public void testGuessMimeType() {
> : + assertEquals(
> : + "application/x-java-applet",
> : + FileTypeMagicUtil.INSTANCE.guessMimeType(
> : + FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin")));
> : + assertEquals(
> : + "application/zip",
> : + FileTypeMagicUtil.INSTANCE.guessMimeType(
> : + FileTypeMagicUtil.class.getResourceAsStream(
> : + "/runtimecode/containerplugin.v.1.jar.bin")));
> : + assertEquals(
> : + "application/x-tar",
> : + FileTypeMagicUtil.INSTANCE.guessMimeType(
> : + FileTypeMagicUtil.class.getResourceAsStream("/magic/hello.tar.bin")));
> : + assertEquals(
> : + "text/x-shellscript",
> : + FileTypeMagicUtil.INSTANCE.guessMimeType(
> : + FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt")));
> : + }
> : +
> : + public void testIsFileForbiddenInConfigset() {
> : + assertTrue(
> : + FileTypeMagicUtil.isFileForbiddenInConfigset(
> : + FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin")));
> : + assertTrue(
> : + FileTypeMagicUtil.isFileForbiddenInConfigset(
> : + FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt")));
> : + assertFalse(
> : + FileTypeMagicUtil.isFileForbiddenInConfigset(
> : + FileTypeMagicUtil.class.getResourceAsStream("/magic/plain.txt")));
> : + }
> : +}
> : diff --git a/solr/licenses/simplemagic-1.17.jar.sha1 b/solr/licenses/simplemagic-1.17.jar.sha1
> : new file mode 100644
> : index 00000000000..cf101094cc8
> : --- /dev/null
> : +++ b/solr/licenses/simplemagic-1.17.jar.sha1
> : @@ -0,0 +1 @@
> : +b6e2d1e47d7172e57fa858a2e3940c09a590e61e
> : diff --git a/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt b/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt
> : new file mode 100644
> : index 00000000000..9228230f933
> : --- /dev/null
> : +++ b/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt
> : @@ -0,0 +1,15 @@
> : +ISC License (https://opensource.org/licenses/ISC)
> : +
> : +Copyright 2021, Gray Watson
> : +
> : +Permission to use, copy, modify, and/or distribute this software for any
> : +purpose with or without fee is hereby granted, provided that the above
> : +copyright notice and this permission notice appear in all copies.
> : +
> : +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
> : +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
> : +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
> : +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
> : +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
> : +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
> : +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
> : \ No newline at end of file
> : diff --git a/solr/licenses/simplemagic-NOTICE.txt b/solr/licenses/simplemagic-NOTICE.txt
> : new file mode 100644
> : index 00000000000..e69de29bb2d
> : diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
> : index d571339880c..e40294a6683 100644
> : --- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
> : +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
> : @@ -348,6 +348,7 @@ public class ZkMaintenanceUtils {
> : USE_FORBIDDEN_FILE_TYPES);
> : return FileVisitResult.CONTINUE;
> : }
> : + // TODO: Cannot check MAGIC header for file since FileTypeGuesser is in core
> : String zkNode = createZkNodeName(zkPath, rootPath, file);
> : try {
> : // if the path exists (and presumably we're uploading data to it) just set its data
> : @@ -437,6 +438,7 @@ public class ZkMaintenanceUtils {
> : if (isFileForbiddenInConfigSets(zkPath)) {
> : log.warn("Skipping download of file from ZK, as it is a forbidden type: {}", zkPath);
> : } else {
> : + // TODO: Cannot check MAGIC header for file since FileTypeGuesser is in core
> : if (copyDataDown(zkClient, zkPath, file) == 0) {
> : Files.createFile(file);
> : }
> : diff --git a/versions.lock b/versions.lock
> : index 9c0f9ef53d4..6b6ea467e0d 100644
> : --- a/versions.lock
> : +++ b/versions.lock
> : @@ -62,6 +62,7 @@ com.googlecode.plist:dd-plist:1.24 (1 constraints: 300c84f5)
> : com.healthmarketscience.jackcess:jackcess:4.0.2 (1 constraints: 5d0cf201)
> : com.healthmarketscience.jackcess:jackcess-encrypt:4.0.1 (1 constraints: 5c0cf101)
> : com.ibm.icu:icu4j:70.1 (1 constraints: a90f1784)
> : +com.j256.simplemagic:simplemagic:1.17 (1 constraints: dd04f830)
> : com.jayway.jsonpath:json-path:2.8.0 (2 constraints: 6c12952c)
> : com.lmax:disruptor:3.4.4 (1 constraints: 0d050a36)
> : com.mchange:c3p0:0.9.5.5 (1 constraints: c80c571b)
> : diff --git a/versions.props b/versions.props
> : index 10b583c43e2..cfa127e1094 100644
> : --- a/versions.props
> : +++ b/versions.props
> : @@ -13,6 +13,7 @@ com.google.cloud:google-cloud-bom=0.204.0
> : com.google.errorprone:*=2.23.0
> : com.google.guava:guava=32.1.3-jre
> : com.google.re2j:re2j=1.7
> : +com.j256.simplemagic:simplemagic=1.17
> : com.jayway.jsonpath:json-path=2.8.0
> : com.lmax:disruptor=3.4.4
> : com.tdunning:t-digest=3.1
> :
> :
>
> -Hoss
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@solr.apache.org
> For additional commands, e-mail: dev-help@solr.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@solr.apache.org
For additional commands, e-mail: dev-help@solr.apache.org
Re: (solr) branch main updated: SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets
Posted by David Smiley <ds...@apache.org>.
Have we discussed Jenkins/CI configuration to notify who it thinks may have
broken the build? Even if I *might* have broken the build, I want it to
let me know!
On Fri, Dec 15, 2023 at 7:22 PM Chris Hostetter <ho...@fucit.org> wrote:
>
> This change, and it's associated backports, seem to have broken
> TestConfigSetService (regardless of seed?) due to leaked files.
>
> https://ci-builds.apache.org/job/Solr/job/Solr-Check-main/8576/consoleText
>
> https://jenkins.thetaphi.de/view/Solr/job/Solr-main-Windows/3706/consoleText
> https://ci-builds.apache.org/job/Solr/job/Solr-Check-9.x/39/consoleText
> https://ci-builds.apache.org/job/Solr/job/Solr-check-9.4/1208/consoleText
>
>
>
> : Date: Wed, 13 Dec 2023 22:08:04 +0000
> : From: janhoy@apache.org
> : Reply-To: dev@solr.apache.org
> : To: "commits@solr.apache.org" <co...@solr.apache.org>
> : Subject: (solr) branch main updated: SOLR-16949: Restrict certain file
> types
> : from being uploaded to or downloaded from Config Sets
> :
> : This is an automated email from the ASF dual-hosted git repository.
> :
> : janhoy pushed a commit to branch main
> : in repository https://gitbox.apache.org/repos/asf/solr.git
> :
> :
> : The following commit(s) were added to refs/heads/main by this push:
> : new 15534754f49 SOLR-16949: Restrict certain file types from being
> uploaded to or downloaded from Config Sets
> : 15534754f49 is described below
> :
> : commit 15534754f492079e52288dd11abaf1c4261b3ea4
> : Author: Jan Høydahl <ja...@apache.org>
> : AuthorDate: Wed Dec 13 22:49:23 2023 +0100
> :
> : SOLR-16949: Restrict certain file types from being uploaded to or
> downloaded from Config Sets
> : ---
> : solr/CHANGES.txt | 2 +
> : solr/core/build.gradle | 2 +
> : .../org/apache/solr/cli/ConfigSetUploadTool.java | 2 +
> : .../org/apache/solr/cloud/ZkConfigSetService.java | 21 ++-
> : .../solr/core/FileSystemConfigSetService.java | 28 +++-
> : .../org/apache/solr/core/backup/BackupManager.java | 23 ++-
> : .../handler/configsets/UploadConfigSetFileAPI.java | 8 +-
> : .../org/apache/solr/util/FileTypeMagicUtil.java | 166
> +++++++++++++++++++++
> : solr/core/src/resources/magic/executables | 74 +++++++++
> : solr/core/src/test-files/magic/HelloWorld.java.txt | 5 +
> : .../test-files/magic/HelloWorldJavaClass.class.bin | Bin 0 -> 426 bytes
> : solr/core/src/test-files/magic/README.md | 29 ++++
> : solr/core/src/test-files/magic/hello.tar.bin | Bin 0 -> 4096 bytes
> : solr/core/src/test-files/magic/plain.txt | 1 +
> : solr/core/src/test-files/magic/shell.sh.txt | 2 +
> : .../org/apache/solr/cloud/TestConfigSetsAPI.java | 141
> +++++++++++------
> : .../apache/solr/util/FileTypeMagicUtilTest.java | 54 +++++++
> : solr/licenses/simplemagic-1.17.jar.sha1 | 1 +
> : solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt | 15 ++
> : solr/licenses/simplemagic-NOTICE.txt | 0
> : .../solr/common/cloud/ZkMaintenanceUtils.java | 2 +
> : versions.lock | 1 +
> : versions.props | 1 +
> : 23 files changed, 522 insertions(+), 56 deletions(-)
> :
> : diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
> : index 40da8c435ac..2fd94304a24 100644
> : --- a/solr/CHANGES.txt
> : +++ b/solr/CHANGES.txt
> : @@ -169,6 +169,8 @@ Other Changes
> : * SOLR-17091: dev tools script cloud.sh became broken after changes in
> 9.3 added a new -slim.tgz file it was not expecting
> : cloud.sh has been updated to ignore the -slim.tgz version of the
> tarball.
> :
> : +* SOLR-16949: Restrict certain file types from being uploaded to or
> downloaded from Config Sets (janhoy, Houston Putman)
> : +
> : ================== 9.4.0 ==================
> : New Features
> : ---------------------
> : diff --git a/solr/core/build.gradle b/solr/core/build.gradle
> : index 61ecd1713af..ed2c8a370ae 100644
> : --- a/solr/core/build.gradle
> : +++ b/solr/core/build.gradle
> : @@ -159,6 +159,8 @@ dependencies {
> :
> : compileOnly 'com.github.stephenc.jcip:jcip-annotations'
> :
> : + implementation 'com.j256.simplemagic:simplemagic'
> : +
> : // -- Test Dependencies
> :
> : testRuntimeOnly 'org.slf4j:jcl-over-slf4j'
> : diff --git
> a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
> b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
> : index 5fd4a538bd7..6576742a195 100644
> : --- a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
> : +++ b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
> : @@ -27,6 +27,7 @@ import
> org.apache.solr.client.solrj.impl.SolrZkClientTimeout;
> : import org.apache.solr.common.cloud.SolrZkClient;
> : import org.apache.solr.common.cloud.ZkMaintenanceUtils;
> : import org.apache.solr.core.ConfigSetService;
> : +import org.apache.solr.util.FileTypeMagicUtil;
> : import org.slf4j.Logger;
> : import org.slf4j.LoggerFactory;
> :
> : @@ -101,6 +102,7 @@ public class ConfigSetUploadTool extends ToolBase {
> : + cli.getOptionValue("confname")
> : + " to ZooKeeper at "
> : + zkHost);
> : + FileTypeMagicUtil.assertConfigSetFolderLegal(confPath);
> : ZkMaintenanceUtils.uploadToZK(
> : zkClient,
> : confPath,
> : diff --git
> a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
> b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
> : index f02404d636d..9abde098e1c 100644
> : --- a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
> : +++ b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
> : @@ -22,6 +22,7 @@ import java.nio.file.Path;
> : import java.util.ArrayList;
> : import java.util.Collections;
> : import java.util.List;
> : +import java.util.Locale;
> : import java.util.Map;
> : import java.util.Objects;
> : import org.apache.solr.client.solrj.cloud.SolrCloudManager;
> : @@ -39,6 +40,7 @@ import org.apache.solr.core.CoreContainer;
> : import org.apache.solr.core.CoreDescriptor;
> : import org.apache.solr.core.SolrConfig;
> : import org.apache.solr.core.SolrResourceLoader;
> : +import org.apache.solr.util.FileTypeMagicUtil;
> : import org.apache.zookeeper.CreateMode;
> : import org.apache.zookeeper.KeeperException;
> : import org.apache.zookeeper.data.Stat;
> : @@ -199,6 +201,15 @@ public class ZkConfigSetService extends
> ConfigSetService {
> : try {
> : if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {
> : log.warn("Not including uploading file to config, as it is a
> forbidden type: {}", fileName);
> : + } else if (FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
> : + String mimeType =
> FileTypeMagicUtil.INSTANCE.guessMimeType(data);
> : + throw new SolrException(
> : + SolrException.ErrorCode.BAD_REQUEST,
> : + String.format(
> : + Locale.ROOT,
> : + "Not uploading file %s to config, as it matched the
> MAGIC signature of a forbidden mime type %s",
> : + fileName,
> : + mimeType));
> : } else {
> : // if overwriteOnExists is true then zkClient#makePath
> failOnExists is set to false
> : zkClient.makePath(filePath, data, CreateMode.PERSISTENT, null,
> !overwriteOnExists, true);
> : @@ -340,7 +351,15 @@ public class ZkConfigSetService extends
> ConfigSetService {
> : } else {
> : log.debug("Copying zk node {} to {}", fromZkFilePath,
> toZkFilePath);
> : byte[] data = zkClient.getData(fromZkFilePath, null, null, true);
> : - zkClient.makePath(toZkFilePath, data, true);
> : + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
> : + zkClient.makePath(toZkFilePath, data, true);
> : + } else {
> : + String mimeType =
> FileTypeMagicUtil.INSTANCE.guessMimeType(data);
> : + log.warn(
> : + "Skipping copy of file {} in ZK, as it matched the MAGIC
> signature of a forbidden mime type {}",
> : + fromZkFilePath,
> : + mimeType);
> : + }
> : }
> : }
> :
> : diff --git
> a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
> b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
> : index 1f1a42b15e1..4b041252211 100644
> : ---
> a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
> : +++
> b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
> : @@ -37,6 +37,7 @@ import java.util.stream.Stream;
> : import org.apache.solr.common.SolrException;
> : import org.apache.solr.common.cloud.ZkMaintenanceUtils;
> : import org.apache.solr.common.util.Utils;
> : +import org.apache.solr.util.FileTypeMagicUtil;
> : import org.slf4j.Logger;
> : import org.slf4j.LoggerFactory;
> :
> : @@ -150,9 +151,17 @@ public class FileSystemConfigSetService extends
> ConfigSetService {
> : if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {
> : log.warn("Not including uploading file to config, as it is a
> forbidden type: {}", fileName);
> : } else {
> : - Path filePath =
> getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));
> : - if (!Files.exists(filePath) || overwriteOnExists) {
> : - Files.write(filePath, data);
> : + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
> : + Path filePath =
> getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));
> : + if (!Files.exists(filePath) || overwriteOnExists) {
> : + Files.write(filePath, data);
> : + }
> : + } else {
> : + String mimeType =
> FileTypeMagicUtil.INSTANCE.guessMimeType(data);
> : + log.warn(
> : + "Not including uploading file {}, as it matched the MAGIC
> signature of a forbidden mime type {}",
> : + fileName,
> : + mimeType);
> : }
> : }
> : }
> : @@ -205,8 +214,17 @@ public class FileSystemConfigSetService extends
> ConfigSetService {
> : "Not including uploading file to config, as it is a
> forbidden type: {}",
> : file.getFileName());
> : } else {
> : - Files.copy(
> : - file,
> target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);
> : + if
> (!FileTypeMagicUtil.isFileForbiddenInConfigset(Files.newInputStream(file)))
> {
> : + Files.copy(
> : + file,
> target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);
> : + } else {
> : + String mimeType =
> : +
> FileTypeMagicUtil.INSTANCE.guessMimeType(Files.newInputStream(file));
> : + log.warn(
> : + "Not copying file {}, as it matched the MAGIC
> signature of a forbidden mime type {}",
> : + file.getFileName(),
> : + mimeType);
> : + }
> : }
> : return FileVisitResult.CONTINUE;
> : }
> : diff --git
> a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
> b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
> : index be6a1a83c2f..8ff78b27e08 100644
> : --- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
> : +++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
> : @@ -40,6 +40,7 @@ import org.apache.solr.common.cloud.ZkStateReader;
> : import org.apache.solr.common.util.Utils;
> : import org.apache.solr.core.ConfigSetService;
> : import org.apache.solr.core.backup.repository.BackupRepository;
> : +import org.apache.solr.util.FileTypeMagicUtil;
> : import org.apache.zookeeper.CreateMode;
> : import org.apache.zookeeper.KeeperException;
> : import org.slf4j.Logger;
> : @@ -349,8 +350,16 @@ public class BackupManager {
> : if (data == null) {
> : data = new byte[0];
> : }
> : - try (OutputStream os = repository.createOutput(uri)) {
> : - os.write(data);
> : + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
> : + try (OutputStream os = repository.createOutput(uri)) {
> : + os.write(data);
> : + }
> : + } else {
> : + String mimeType =
> FileTypeMagicUtil.INSTANCE.guessMimeType(data);
> : + log.warn(
> : + "Not including zookeeper file {} in backup, as it
> matched the MAGIC signature of a forbidden mime type {}",
> : + filePath,
> : + mimeType);
> : }
> : }
> : } else {
> : @@ -379,7 +388,15 @@ public class BackupManager {
> : // probably ok since the config file should be small.
> : byte[] arr = new byte[(int) is.length()];
> : is.readBytes(arr, 0, (int) is.length());
> : - configSetService.uploadFileToConfig(configName,
> filePath, arr, false);
> : + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(arr))
> {
> : + configSetService.uploadFileToConfig(configName,
> filePath, arr, false);
> : + } else {
> : + String mimeType =
> FileTypeMagicUtil.INSTANCE.guessMimeType(arr);
> : + log.warn(
> : + "Not including zookeeper file {} in restore, as
> it matched the MAGIC signature of a forbidden mime type {}",
> : + filePath,
> : + mimeType);
> : + }
> : }
> : }
> : break;
> : diff --git
> a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
> b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
> : index 98889aff563..2380a79a92b 100644
> : ---
> a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
> : +++
> b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
> : @@ -27,6 +27,7 @@ import org.apache.solr.common.params.ConfigSetParams;
> : import org.apache.solr.core.CoreContainer;
> : import org.apache.solr.request.SolrQueryRequest;
> : import org.apache.solr.response.SolrQueryResponse;
> : +import org.apache.solr.util.FileTypeMagicUtil;
> :
> : /**
> : * V2 API for adding or updating a single file within a configset.
> : @@ -67,11 +68,13 @@ public class UploadConfigSetFileAPI extends
> ConfigSetAPIBase {
> : if (fixedSingleFilePath.charAt(0) == '/') {
> : fixedSingleFilePath = fixedSingleFilePath.substring(1);
> : }
> : + byte[] data = inputStream.readAllBytes();
> : if (fixedSingleFilePath.isEmpty()) {
> : throw new SolrException(
> : SolrException.ErrorCode.BAD_REQUEST,
> : "The file path provided for upload, '" + singleFilePath + "',
> is not valid.");
> : - } else if
> (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fixedSingleFilePath)) {
> : + } else if
> (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fixedSingleFilePath)
> : + || FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
> : throw new SolrException(
> : SolrException.ErrorCode.BAD_REQUEST,
> : "The file type provided for upload, '"
> : @@ -87,8 +90,7 @@ public class UploadConfigSetFileAPI extends
> ConfigSetAPIBase {
> : // For creating the baseNode, the cleanup parameter is only
> allowed to be true when
> : // singleFilePath is not passed.
> : createBaseNode(configSetService, overwritesExisting,
> requestIsTrusted, configSetName);
> : - configSetService.uploadFileToConfig(
> : - configSetName, fixedSingleFilePath,
> inputStream.readAllBytes(), allowOverwrite);
> : + configSetService.uploadFileToConfig(configSetName,
> fixedSingleFilePath, data, allowOverwrite);
> : }
> : }
> : }
> : diff --git
> a/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java
> b/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java
> : new file mode 100644
> : index 00000000000..cfb6c9fa0af
> : --- /dev/null
> : +++ b/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java
> : @@ -0,0 +1,166 @@
> : +/*
> : + * Licensed to the Apache Software Foundation (ASF) under one or more
> : + * contributor license agreements. See the NOTICE file distributed with
> : + * this work for additional information regarding copyright ownership.
> : + * The ASF licenses this file to You under the Apache License, Version
> 2.0
> : + * (the "License"); you may not use this file except in compliance with
> : + * the License. You may obtain a copy of the License at
> : + *
> : + * http://www.apache.org/licenses/LICENSE-2.0
> : + *
> : + * Unless required by applicable law or agreed to in writing, software
> : + * distributed under the License is distributed on an "AS IS" BASIS,
> : + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> : + * See the License for the specific language governing permissions and
> : + * limitations under the License.
> : + */
> : +
> : +package org.apache.solr.util;
> : +
> : +import com.j256.simplemagic.ContentInfo;
> : +import com.j256.simplemagic.ContentInfoUtil;
> : +import com.j256.simplemagic.ContentType;
> : +import java.io.ByteArrayInputStream;
> : +import java.io.IOException;
> : +import java.io.InputStream;
> : +import java.nio.file.FileVisitResult;
> : +import java.nio.file.Files;
> : +import java.nio.file.Path;
> : +import java.nio.file.SimpleFileVisitor;
> : +import java.nio.file.attribute.BasicFileAttributes;
> : +import java.util.Arrays;
> : +import java.util.HashSet;
> : +import java.util.Locale;
> : +import java.util.Set;
> : +import org.apache.solr.common.SolrException;
> : +
> : +/** Utility class to guess the mime type of file based on its magic
> number. */
> : +public class FileTypeMagicUtil implements ContentInfoUtil.ErrorCallBack
> {
> : + private final ContentInfoUtil util;
> : + private static final Set<String> SKIP_FOLDERS = new
> HashSet<>(Arrays.asList(".", ".."));
> : +
> : + public static FileTypeMagicUtil INSTANCE = new FileTypeMagicUtil();
> : +
> : + FileTypeMagicUtil() {
> : + try {
> : + util = new ContentInfoUtil("/magic/executables", this);
> : + } catch (IOException e) {
> : + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
> "Error parsing magic file", e);
> : + }
> : + }
> : +
> : + /**
> : + * Asserts that an entire configset folder is legal to upload.
> : + *
> : + * @param confPath the path to the folder
> : + * @throws SolrException if an illegal file is found in the folder
> structure
> : + */
> : + public static void assertConfigSetFolderLegal(Path confPath) throws
> IOException {
> : + Files.walkFileTree(
> : + confPath,
> : + new SimpleFileVisitor<Path>() {
> : + @Override
> : + public FileVisitResult visitFile(Path file,
> BasicFileAttributes attrs)
> : + throws IOException {
> : + // Read first 100 bytes of the file to determine the mime
> type
> : + try (InputStream fileStream = Files.newInputStream(file)) {
> : + byte[] bytes = new byte[100];
> : + fileStream.read(bytes);
> : + if (FileTypeMagicUtil.isFileForbiddenInConfigset(bytes)) {
> : + throw new SolrException(
> : + SolrException.ErrorCode.BAD_REQUEST,
> : + String.format(
> : + Locale.ROOT,
> : + "Not uploading file %s to configset, as it
> matched the MAGIC signature of a forbidden mime type %s",
> : + file,
> : +
> FileTypeMagicUtil.INSTANCE.guessMimeType(bytes)));
> : + }
> : + return FileVisitResult.CONTINUE;
> : + }
> : + }
> : +
> : + @Override
> : + public FileVisitResult preVisitDirectory(Path dir,
> BasicFileAttributes attrs)
> : + throws IOException {
> : + if (SKIP_FOLDERS.contains(dir.getFileName().toString()))
> : + return FileVisitResult.SKIP_SUBTREE;
> : +
> : + return FileVisitResult.CONTINUE;
> : + }
> : + });
> : + }
> : +
> : + /**
> : + * Guess the mime type of file based on its magic number.
> : + *
> : + * @param stream input stream of the file
> : + * @return string with content-type or "application/octet-stream" if
> unknown
> : + */
> : + public String guessMimeType(InputStream stream) {
> : + try {
> : + ContentInfo info = util.findMatch(stream);
> : + if (info == null) {
> : + return ContentType.OTHER.getMimeType();
> : + }
> : + return info.getContentType().getMimeType();
> : + } catch (IOException e) {
> : + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
> : + }
> : + }
> : +
> : + /**
> : + * Guess the mime type of file bytes based on its magic number.
> : + *
> : + * @param bytes the first bytes at start of the file
> : + * @return string with content-type or "application/octet-stream" if
> unknown
> : + */
> : + public String guessMimeType(byte[] bytes) {
> : + return guessMimeType(new ByteArrayInputStream(bytes));
> : + }
> : +
> : + @Override
> : + public void error(String line, String details, Exception e) {
> : + throw new SolrException(
> : + SolrException.ErrorCode.SERVER_ERROR,
> : + String.format(Locale.ROOT, "%s: %s", line, details),
> : + e);
> : + }
> : +
> : + /**
> : + * Determine forbidden file type based on magic bytes matching of the
> file itself. Forbidden types
> : + * are:
> : + *
> : + * <ul>
> : + * <li><code>application/x-java-applet</code>: java class file
> : + * <li><code>application/zip</code>: jar or zip archives
> : + * <li><code>application/x-tar</code>: tar archives
> : + * <li><code>text/x-shellscript</code>: shell or bash script
> : + * </ul>
> : + *
> : + * @param fileStream stream from the file content
> : + * @return true if file is among the forbidden mime-types
> : + */
> : + public static boolean isFileForbiddenInConfigset(InputStream
> fileStream) {
> : + return
> forbiddenTypes.contains(FileTypeMagicUtil.INSTANCE.guessMimeType(fileStream));
> : + }
> : +
> : + /**
> : + * Determine forbidden file type based on magic bytes matching of the
> first bytes of the file.
> : + *
> : + * @param bytes byte array of the file content
> : + * @return true if file is among the forbidden mime-types
> : + */
> : + public static boolean isFileForbiddenInConfigset(byte[] bytes) {
> : + if (bytes == null || bytes.length == 0)
> : + return false; // A ZK znode may be a folder with no content
> : + return isFileForbiddenInConfigset(new ByteArrayInputStream(bytes));
> : + }
> : +
> : + private static final Set<String> forbiddenTypes =
> : + new HashSet<>(
> : + Arrays.asList(
> : + System.getProperty(
> : + "solr.configset.upload.mimetypes.forbidden",
> : +
> "application/x-java-applet,application/zip,application/x-tar,text/x-shellscript")
> : + .split(",")));
> : +}
> : diff --git a/solr/core/src/resources/magic/executables
> b/solr/core/src/resources/magic/executables
> : new file mode 100644
> : index 00000000000..04094eaf797
> : --- /dev/null
> : +++ b/solr/core/src/resources/magic/executables
> : @@ -0,0 +1,74 @@
> : +# Licensed to the Apache Software Foundation (ASF) under one or more
> : +# contributor license agreements. See the NOTICE file distributed with
> : +# this work for additional information regarding copyright ownership.
> : +# The ASF licenses this file to You under the Apache License, Version
> 2.0
> : +# (the "License"); you may not use this file except in compliance with
> : +# the License. You may obtain a copy of the License at
> : +#
> : +# http://www.apache.org/licenses/LICENSE-2.0
> : +#
> : +# Unless required by applicable law or agreed to in writing, software
> : +# distributed under the License is distributed on an "AS IS" BASIS,
> : +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> : +# See the License for the specific language governing permissions and
> : +# limitations under the License.
> : +
> : +# POSIX tar archives
> : +# URL: https://en.wikipedia.org/wiki/Tar_(computing)
> : +# Reference:
> https://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5&manpath=FreeBSD+8-current
> : +# header mainly padded with nul bytes
> : +500 quad 0
> : +!:strength /2
> : +# filename or extended attribute printable strings in range space null
> til umlaut ue
> : +>0 ubeshort >0x1F00
> : +>>0 ubeshort <0xFCFD
> : +# last 4 header bytes often null but tar\0 in gtarfail2.tar
> gtarfail.tar-bad
> : +# at https://sourceforge.net/projects/s-tar/files/testscripts/
> : +>>>508 ubelong&0x8B9E8DFF 0
> : +# nul, space or ascii digit 0-7 at start of mode
> : +>>>>100 ubyte&0xC8 =0
> : +>>>>>101 ubyte&0xC8 =0
> : +# nul, space at end of check sum
> : +>>>>>>155 ubyte&0xDF =0
> : +# space or ascii digit 0 at start of check sum
> : +>>>>>>>148 ubyte&0xEF =0x20
> : +# check for specific 1st member name that indicates other mime type and
> file name suffix
> : +>>>>>>>>0 string TpmEmuTpms/permall
> : +!:mime application/x-tar
> : +!:ext tar
> : +# other stuff in padding
> : +# some implementations add new fields to the blank area at the end of
> the header record
> : +# created for example by DOS TAR 3.20g 1994 Tim V.Shapore with -j option
> : +>>257 ulong !0 tar archive (old)
> : +!:mime application/x-tar
> : +!:ext tar
> : +# magic in newer, GNU, posix variants
> : +>257 string =ustar
> : +# 2 last char of magic and UStar version because string expression does
> not work
> : +# 2 space characters followed by a null for GNU variant
> : +>>261 ubelong =0x72202000 POSIX tar archive (GNU)
> : +!:mime application/x-gtar
> : +!:ext tar/gtar
> : +
> : +
> : +# Zip archives (Greg Roelofs, c/o zip-bugs@wkuvx1.wku.edu)
> : +0 string PK\005\006 Zip archive data (empty)
> : +0 string PK\003\004 Zip archive data
> : +!:strength +1
> : +!:mime application/zip
> : +!:ext zip/cbz
> : +
> : +
> : +# JAVA
> : +0 belong 0xcafebabe
> : +>4 ubelong >30 compiled Java class data,
> : +!:mime application/x-java-applet
> : +#!:mime application/java-byte-code
> : +!:ext class
> : +
> : +
> : +# SHELL scripts
> : +#0 string/w : shell archive or script
> for antique kernel text
> : +0 regex \^#!\\s?(/bin/|/usr/) POSIX shell script text
> executable
> : +!:mime text/x-shellscript
> : +!:ext sh/bash
> : \ No newline at end of file
> : diff --git a/solr/core/src/test-files/magic/HelloWorld.java.txt
> b/solr/core/src/test-files/magic/HelloWorld.java.txt
> : new file mode 100644
> : index 00000000000..ca9518d2afa
> : --- /dev/null
> : +++ b/solr/core/src/test-files/magic/HelloWorld.java.txt
> : @@ -0,0 +1,5 @@
> : +class HelloWorld {
> : + public static void main(String[] args) {
> : + System.out.println("Hellow world");
> : + }
> : +}
> : \ No newline at end of file
> : diff --git
> a/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin
> b/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin
> : new file mode 100644
> : index 00000000000..e15d0a6c5b9
> : Binary files /dev/null and
> b/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin differ
> : diff --git a/solr/core/src/test-files/magic/README.md
> b/solr/core/src/test-files/magic/README.md
> : new file mode 100644
> : index 00000000000..6e499a2f711
> : --- /dev/null
> : +++ b/solr/core/src/test-files/magic/README.md
> : @@ -0,0 +1,29 @@
> : +<!--
> : + Licensed to the Apache Software Foundation (ASF) under one or more
> : + contributor license agreements. See the NOTICE file distributed
> with
> : + this work for additional information regarding copyright ownership.
> : + The ASF licenses this file to You under the Apache License, Version
> 2.0
> : + the "License"); you may not use this file except in compliance with
> : + the License. You may obtain a copy of the License at
> : +
> : + http://www.apache.org/licenses/LICENSE-2.0
> : +
> : + Unless required by applicable law or agreed to in writing, software
> : + distributed under the License is distributed on an "AS IS" BASIS,
> : + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> : + See the License for the specific language governing permissions and
> : + limitations under the License.
> : + -->
> : +
> : +The two binary files were created by the following commands:
> : +
> : +```bash
> : +echo "Hello" > hello.txt && \
> : + tar -cvf hello.tar.bin hello.txt && \
> : + rm hello.txt
> : +
> : +cp HelloWorld.java.txt HelloWorld.java && \
> : + javac HelloWorld.java && \
> : + mv HelloWorld.class HelloWorldJavaClass.class.bin && \
> : + rm HelloWorld.java
> : +```
> : \ No newline at end of file
> : diff --git a/solr/core/src/test-files/magic/hello.tar.bin
> b/solr/core/src/test-files/magic/hello.tar.bin
> : new file mode 100644
> : index 00000000000..68ca23c362a
> : Binary files /dev/null and
> b/solr/core/src/test-files/magic/hello.tar.bin differ
> : diff --git a/solr/core/src/test-files/magic/plain.txt
> b/solr/core/src/test-files/magic/plain.txt
> : new file mode 100644
> : index 00000000000..70c379b63ff
> : --- /dev/null
> : +++ b/solr/core/src/test-files/magic/plain.txt
> : @@ -0,0 +1 @@
> : +Hello world
> : \ No newline at end of file
> : diff --git a/solr/core/src/test-files/magic/shell.sh.txt
> b/solr/core/src/test-files/magic/shell.sh.txt
> : new file mode 100644
> : index 00000000000..9ea411e111d
> : --- /dev/null
> : +++ b/solr/core/src/test-files/magic/shell.sh.txt
> : @@ -0,0 +1,2 @@
> : +#! /usr/bin/env bash
> : +echo Hello
> : \ No newline at end of file
> : diff --git
> a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
> b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
> : index 1da60d85ff9..f7c9431c296 100644
> : --- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
> : +++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
> : @@ -45,6 +45,7 @@ import java.util.LinkedHashMap;
> : import java.util.List;
> : import java.util.Locale;
> : import java.util.Map;
> : +import java.util.Objects;
> : import java.util.Properties;
> : import java.util.Set;
> : import java.util.concurrent.TimeUnit;
> : @@ -592,14 +593,14 @@ public class TestConfigSetsAPI extends
> SolrCloudTestCase {
> : assertEquals(
> : "Can't overwrite an existing configset unless the overwrite
> parameter is set",
> : 400,
> : - uploadConfigSet(configsetName, configsetSuffix, null, false,
> false, v2, false));
> : + uploadConfigSet(configsetName, configsetSuffix, null, false,
> false, v2, false, false));
> : unIgnoreException("The configuration regulartestOverwrite-1
> already exists in zookeeper");
> : assertEquals(
> : "Expecting version to remain equal",
> : solrconfigZkVersion,
> : getConfigZNodeVersion(zkClient, configsetName,
> configsetSuffix, "solrconfig.xml"));
> : assertEquals(
> : - 0, uploadConfigSet(configsetName, configsetSuffix, null,
> true, false, v2, false));
> : + 0, uploadConfigSet(configsetName, configsetSuffix, null,
> true, false, v2, false, false));
> : assertTrue(
> : "Expecting version bump",
> : solrconfigZkVersion
> : @@ -638,13 +639,14 @@ public class TestConfigSetsAPI extends
> SolrCloudTestCase {
> : zkClient.makePath(f, true);
> : }
> : assertEquals(
> : - 0, uploadConfigSet(configsetName, configsetSuffix, null,
> true, false, v2, false));
> : + 0, uploadConfigSet(configsetName, configsetSuffix, null,
> true, false, v2, false, false));
> : for (String f : extraFiles) {
> : assertTrue(
> : "Expecting file " + f + " to exist in ConfigSet but it's
> gone",
> : zkClient.exists(f, true));
> : }
> : - assertEquals(0, uploadConfigSet(configsetName, configsetSuffix,
> null, true, true, v2, false));
> : + assertEquals(
> : + 0, uploadConfigSet(configsetName, configsetSuffix, null,
> true, true, v2, false, false));
> : for (String f : extraFiles) {
> : assertFalse(
> : "Expecting file " + f + " to be deleted from ConfigSet but
> it wasn't",
> : @@ -675,7 +677,8 @@ public class TestConfigSetsAPI extends
> SolrCloudTestCase {
> : .withConnTimeOut(45000, TimeUnit.MILLISECONDS)
> : .build()) {
> : String configPath = "/configs/" + configsetName + configsetSuffix;
> : - assertEquals(0, uploadConfigSet(configsetName, configsetSuffix,
> null, true, false, v2, true));
> : + assertEquals(
> : + 0, uploadConfigSet(configsetName, configsetSuffix, null,
> true, false, v2, true, false));
> : for (String fileEnding :
> ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
> : String f = configPath + "/test." + fileEnding;
> : assertFalse(
> : @@ -710,7 +713,7 @@ public class TestConfigSetsAPI extends
> SolrCloudTestCase {
> : getConfigZNodeVersion(zkClient, configsetName,
> configsetSuffix, "solrconfig.xml");
> : // Was untrusted, overwrite with untrusted
> : assertEquals(
> : - 0, uploadConfigSet(configsetName, configsetSuffix, null,
> true, false, v2, false));
> : + 0, uploadConfigSet(configsetName, configsetSuffix, null,
> true, false, v2, false, false));
> : assertTrue(
> : "Expecting version bump",
> : solrconfigZkVersion
> : @@ -721,7 +724,8 @@ public class TestConfigSetsAPI extends
> SolrCloudTestCase {
> :
> : // Was untrusted, overwrite with trusted but no cleanup
> : assertEquals(
> : - 0, uploadConfigSet(configsetName, configsetSuffix, "solr",
> true, false, v2, false));
> : + 0,
> : + uploadConfigSet(configsetName, configsetSuffix, "solr", true,
> false, v2, false, false));
> : assertTrue(
> : "Expecting version bump",
> : solrconfigZkVersion
> : @@ -747,7 +751,7 @@ public class TestConfigSetsAPI extends
> SolrCloudTestCase {
> :
> : // Was untrusted, overwrite with trusted with cleanup
> : assertEquals(
> : - 0, uploadConfigSet(configsetName, configsetSuffix, "solr",
> true, true, v2, false));
> : + 0, uploadConfigSet(configsetName, configsetSuffix, "solr",
> true, true, v2, false, false));
> : assertTrue(
> : "Expecting version bump",
> : solrconfigZkVersion
> : @@ -761,7 +765,7 @@ public class TestConfigSetsAPI extends
> SolrCloudTestCase {
> : assertEquals(
> : "Can't upload a trusted configset with an untrusted request",
> : 400,
> : - uploadConfigSet(configsetName, configsetSuffix, null, true,
> false, v2, false));
> : + uploadConfigSet(configsetName, configsetSuffix, null, true,
> false, v2, false, false));
> : assertEquals(
> : "Expecting version to remain equal",
> : solrconfigZkVersion,
> : @@ -773,7 +777,7 @@ public class TestConfigSetsAPI extends
> SolrCloudTestCase {
> : assertEquals(
> : "Can't upload a trusted configset with an untrusted request",
> : 400,
> : - uploadConfigSet(configsetName, configsetSuffix, null, true,
> true, v2, false));
> : + uploadConfigSet(configsetName, configsetSuffix, null, true,
> true, v2, false, false));
> : assertEquals(
> : "Expecting version to remain equal",
> : solrconfigZkVersion,
> : @@ -783,7 +787,8 @@ public class TestConfigSetsAPI extends
> SolrCloudTestCase {
> :
> : // Was trusted, overwrite with trusted no cleanup
> : assertEquals(
> : - 0, uploadConfigSet(configsetName, configsetSuffix, "solr",
> true, false, v2, false));
> : + 0,
> : + uploadConfigSet(configsetName, configsetSuffix, "solr", true,
> false, v2, false, false));
> : assertTrue(
> : "Expecting version bump",
> : solrconfigZkVersion
> : @@ -794,7 +799,7 @@ public class TestConfigSetsAPI extends
> SolrCloudTestCase {
> :
> : // Was trusted, overwrite with trusted with cleanup
> : assertEquals(
> : - 0, uploadConfigSet(configsetName, configsetSuffix, "solr",
> true, true, v2, false));
> : + 0, uploadConfigSet(configsetName, configsetSuffix, "solr",
> true, true, v2, false, false));
> : assertTrue(
> : "Expecting version bump",
> : solrconfigZkVersion
> : @@ -1457,6 +1462,13 @@ public class TestConfigSetsAPI extends
> SolrCloudTestCase {
> : .get("id"));
> : }
> :
> : + @Test
> : + public void testUploadWithForbiddenContent() throws Exception {
> : + // Uploads a config set containing a script, a class file and jar
> file, will return 400 error
> : + long res = uploadConfigSet("forbidden", "suffix", "foo", true,
> false, true, false, true);
> : + assertEquals(400, res);
> : + }
> : +
> : private static String getSecurityJson() {
> : return "{\n"
> : + " 'authentication':{\n"
> : @@ -1511,7 +1523,7 @@ public class TestConfigSetsAPI extends
> SolrCloudTestCase {
> : String configSetName, String suffix, String username,
> SolrZkClient zkClient, boolean v2)
> : throws IOException {
> : assertFalse(getConfigSetService().checkConfigExists(configSetName +
> suffix));
> : - return uploadConfigSet(configSetName, suffix, username, false,
> false, v2, false);
> : + return uploadConfigSet(configSetName, suffix, username, false,
> false, v2, false, false);
> : }
> :
> : private long uploadConfigSet(
> : @@ -1521,21 +1533,25 @@ public class TestConfigSetsAPI extends
> SolrCloudTestCase {
> : boolean overwrite,
> : boolean cleanup,
> : boolean v2,
> : - boolean forbiddenTypes)
> : + boolean forbiddenTypes,
> : + boolean forbiddenContent)
> : throws IOException {
> :
> : + File zipFile;
> : + if (forbiddenTypes) {
> : + log.info("Uploading configset with forbidden file endings");
> : + zipFile =
> : + createTempZipFileWithForbiddenTypes(
> : + "solr/configsets/upload/" + configSetName +
> "/solrconfig.xml");
> : + } else if (forbiddenContent) {
> : + log.info("Uploading configset with forbidden file content");
> : + zipFile = createTempZipFileWithForbiddenContent("magic");
> : + } else {
> : + zipFile = createTempZipFile("solr/configsets/upload/" +
> configSetName);
> : + }
> : +
> : // Read zipped sample config
> : - return uploadGivenConfigSet(
> : - forbiddenTypes
> : - ? createTempZipFileWithForbiddenTypes(
> : - "solr/configsets/upload/" + configSetName +
> "/solrconfig.xml")
> : - : createTempZipFile("solr/configsets/upload/" +
> configSetName),
> : - configSetName,
> : - suffix,
> : - username,
> : - overwrite,
> : - cleanup,
> : - v2);
> : + return uploadGivenConfigSet(zipFile, configSetName, suffix,
> username, overwrite, cleanup, v2);
> : }
> :
> : private long uploadBadConfigSet(String configSetName, String suffix,
> String username, boolean v2)
> : @@ -1702,31 +1718,68 @@ public class TestConfigSetsAPI extends
> SolrCloudTestCase {
> : }
> : }
> :
> : - private static void zipWithForbiddenEndings(File file, File zipfile)
> throws IOException {
> : - OutputStream out = new FileOutputStream(zipfile);
> : - ZipOutputStream zout = new ZipOutputStream(out);
> : + /** Create a zip file (in the temp directory) containing files with
> forbidden content */
> : + private File createTempZipFileWithForbiddenContent(String
> resourcePath) {
> : try {
> : - for (String fileType :
> ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
> : - zout.putNextEntry(new ZipEntry("test." + fileType));
> : + final File zipFile = createTempFile("configset", "zip").toFile();
> : + final File directory = SolrTestCaseJ4.getFile(resourcePath);
> : + if (log.isInfoEnabled()) {
> : + log.info("Directory: {}", directory.getAbsolutePath());
> : + }
> : + zipWithForbiddenContent(directory, zipFile);
> : + if (log.isInfoEnabled()) {
> : + log.info("Zipfile: {}", zipFile.getAbsolutePath());
> : + }
> : + return zipFile;
> : + } catch (IOException e) {
> : + throw new RuntimeException(e);
> : + }
> : + }
> :
> : - InputStream in = new FileInputStream(file);
> : - try {
> : - byte[] buffer = new byte[1024];
> : - while (true) {
> : - int readCount = in.read(buffer);
> : - if (readCount < 0) {
> : - break;
> : + private static void zipWithForbiddenContent(File directory, File
> zipfile) throws IOException {
> : + OutputStream out = Files.newOutputStream(zipfile.toPath());
> : + assertTrue(directory.isDirectory());
> : + try (ZipOutputStream zout = new ZipOutputStream(out)) {
> : + // Copy in all files from the directory
> : + for (File file : Objects.requireNonNull(directory.listFiles())) {
> : + zout.putNextEntry(new ZipEntry(file.getName()));
> : + zout.write(Files.readAllBytes(file.toPath()));
> : + zout.closeEntry();
> : + }
> : + }
> : + }
> : +
> : + private static void zipWithForbiddenEndings(File fileOrDirectory,
> File zipfile)
> : + throws IOException {
> : + OutputStream out = new FileOutputStream(zipfile);
> : + try (ZipOutputStream zout = new ZipOutputStream(out)) {
> : + if (fileOrDirectory.isFile()) {
> : + // Create entries with given file, one for each forbidden
> endding
> : + for (String fileType :
> ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
> : + zout.putNextEntry(new ZipEntry("test." + fileType));
> : +
> : + try (InputStream in = new FileInputStream(fileOrDirectory)) {
> : + byte[] buffer = new byte[1024];
> : + while (true) {
> : + int readCount = in.read(buffer);
> : + if (readCount < 0) {
> : + break;
> : + }
> : + zout.write(buffer, 0, readCount);
> : }
> : - zout.write(buffer, 0, readCount);
> : }
> : - } finally {
> : - in.close();
> : - }
> :
> : - zout.closeEntry();
> : + zout.closeEntry();
> : + }
> : + }
> : + if (fileOrDirectory.isDirectory()) {
> : + // Copy in all files from the directory
> : + for (File file :
> Objects.requireNonNull(fileOrDirectory.listFiles())) {
> : + zout.putNextEntry(new ZipEntry(file.getName()));
> : + zout.write(Files.readAllBytes(file.toPath()));
> : + zout.closeEntry();
> : + }
> : }
> : - } finally {
> : - zout.close();
> : }
> : }
> :
> : diff --git
> a/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java
> b/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java
> : new file mode 100644
> : index 00000000000..b8e9a35a3d9
> : --- /dev/null
> : +++ b/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java
> : @@ -0,0 +1,54 @@
> : +/*
> : + * Licensed to the Apache Software Foundation (ASF) under one or more
> : + * contributor license agreements. See the NOTICE file distributed with
> : + * this work for additional information regarding copyright ownership.
> : + * The ASF licenses this file to You under the Apache License, Version
> 2.0
> : + * (the "License"); you may not use this file except in compliance with
> : + * the License. You may obtain a copy of the License at
> : + *
> : + * http://www.apache.org/licenses/LICENSE-2.0
> : + *
> : + * Unless required by applicable law or agreed to in writing, software
> : + * distributed under the License is distributed on an "AS IS" BASIS,
> : + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> : + * See the License for the specific language governing permissions and
> : + * limitations under the License.
> : + */
> : +
> : +package org.apache.solr.util;
> : +
> : +import org.apache.solr.SolrTestCaseJ4;
> : +
> : +public class FileTypeMagicUtilTest extends SolrTestCaseJ4 {
> : + public void testGuessMimeType() {
> : + assertEquals(
> : + "application/x-java-applet",
> : + FileTypeMagicUtil.INSTANCE.guessMimeType(
> : +
> FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin")));
> : + assertEquals(
> : + "application/zip",
> : + FileTypeMagicUtil.INSTANCE.guessMimeType(
> : + FileTypeMagicUtil.class.getResourceAsStream(
> : + "/runtimecode/containerplugin.v.1.jar.bin")));
> : + assertEquals(
> : + "application/x-tar",
> : + FileTypeMagicUtil.INSTANCE.guessMimeType(
> : +
> FileTypeMagicUtil.class.getResourceAsStream("/magic/hello.tar.bin")));
> : + assertEquals(
> : + "text/x-shellscript",
> : + FileTypeMagicUtil.INSTANCE.guessMimeType(
> : +
> FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt")));
> : + }
> : +
> : + public void testIsFileForbiddenInConfigset() {
> : + assertTrue(
> : + FileTypeMagicUtil.isFileForbiddenInConfigset(
> : +
> FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin")));
> : + assertTrue(
> : + FileTypeMagicUtil.isFileForbiddenInConfigset(
> : +
> FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt")));
> : + assertFalse(
> : + FileTypeMagicUtil.isFileForbiddenInConfigset(
> : +
> FileTypeMagicUtil.class.getResourceAsStream("/magic/plain.txt")));
> : + }
> : +}
> : diff --git a/solr/licenses/simplemagic-1.17.jar.sha1
> b/solr/licenses/simplemagic-1.17.jar.sha1
> : new file mode 100644
> : index 00000000000..cf101094cc8
> : --- /dev/null
> : +++ b/solr/licenses/simplemagic-1.17.jar.sha1
> : @@ -0,0 +1 @@
> : +b6e2d1e47d7172e57fa858a2e3940c09a590e61e
> : diff --git a/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt
> b/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt
> : new file mode 100644
> : index 00000000000..9228230f933
> : --- /dev/null
> : +++ b/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt
> : @@ -0,0 +1,15 @@
> : +ISC License (https://opensource.org/licenses/ISC)
> : +
> : +Copyright 2021, Gray Watson
> : +
> : +Permission to use, copy, modify, and/or distribute this software for any
> : +purpose with or without fee is hereby granted, provided that the above
> : +copyright notice and this permission notice appear in all copies.
> : +
> : +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
> : +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
> : +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
> : +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
> : +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
> : +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
> : +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
> : \ No newline at end of file
> : diff --git a/solr/licenses/simplemagic-NOTICE.txt
> b/solr/licenses/simplemagic-NOTICE.txt
> : new file mode 100644
> : index 00000000000..e69de29bb2d
> : diff --git
> a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
> b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
> : index d571339880c..e40294a6683 100644
> : ---
> a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
> : +++
> b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
> : @@ -348,6 +348,7 @@ public class ZkMaintenanceUtils {
> : USE_FORBIDDEN_FILE_TYPES);
> : return FileVisitResult.CONTINUE;
> : }
> : + // TODO: Cannot check MAGIC header for file since
> FileTypeGuesser is in core
> : String zkNode = createZkNodeName(zkPath, rootPath, file);
> : try {
> : // if the path exists (and presumably we're uploading
> data to it) just set its data
> : @@ -437,6 +438,7 @@ public class ZkMaintenanceUtils {
> : if (isFileForbiddenInConfigSets(zkPath)) {
> : log.warn("Skipping download of file from ZK, as it is a
> forbidden type: {}", zkPath);
> : } else {
> : + // TODO: Cannot check MAGIC header for file since
> FileTypeGuesser is in core
> : if (copyDataDown(zkClient, zkPath, file) == 0) {
> : Files.createFile(file);
> : }
> : diff --git a/versions.lock b/versions.lock
> : index 9c0f9ef53d4..6b6ea467e0d 100644
> : --- a/versions.lock
> : +++ b/versions.lock
> : @@ -62,6 +62,7 @@ com.googlecode.plist:dd-plist:1.24 (1 constraints:
> 300c84f5)
> : com.healthmarketscience.jackcess:jackcess:4.0.2 (1 constraints:
> 5d0cf201)
> : com.healthmarketscience.jackcess:jackcess-encrypt:4.0.1 (1 constraints:
> 5c0cf101)
> : com.ibm.icu:icu4j:70.1 (1 constraints: a90f1784)
> : +com.j256.simplemagic:simplemagic:1.17 (1 constraints: dd04f830)
> : com.jayway.jsonpath:json-path:2.8.0 (2 constraints: 6c12952c)
> : com.lmax:disruptor:3.4.4 (1 constraints: 0d050a36)
> : com.mchange:c3p0:0.9.5.5 (1 constraints: c80c571b)
> : diff --git a/versions.props b/versions.props
> : index 10b583c43e2..cfa127e1094 100644
> : --- a/versions.props
> : +++ b/versions.props
> : @@ -13,6 +13,7 @@ com.google.cloud:google-cloud-bom=0.204.0
> : com.google.errorprone:*=2.23.0
> : com.google.guava:guava=32.1.3-jre
> : com.google.re2j:re2j=1.7
> : +com.j256.simplemagic:simplemagic=1.17
> : com.jayway.jsonpath:json-path=2.8.0
> : com.lmax:disruptor=3.4.4
> : com.tdunning:t-digest=3.1
> :
> :
>
> -Hoss
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@solr.apache.org
> For additional commands, e-mail: dev-help@solr.apache.org
Re: (solr) branch main updated: SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets
Posted by Chris Hostetter <ho...@fucit.org>.
This change, and it's associated backports, seem to have broken
TestConfigSetService (regardless of seed?) due to leaked files.
https://ci-builds.apache.org/job/Solr/job/Solr-Check-main/8576/consoleText
https://jenkins.thetaphi.de/view/Solr/job/Solr-main-Windows/3706/consoleText
https://ci-builds.apache.org/job/Solr/job/Solr-Check-9.x/39/consoleText
https://ci-builds.apache.org/job/Solr/job/Solr-check-9.4/1208/consoleText
: Date: Wed, 13 Dec 2023 22:08:04 +0000
: From: janhoy@apache.org
: Reply-To: dev@solr.apache.org
: To: "commits@solr.apache.org" <co...@solr.apache.org>
: Subject: (solr) branch main updated: SOLR-16949: Restrict certain file types
: from being uploaded to or downloaded from Config Sets
:
: This is an automated email from the ASF dual-hosted git repository.
:
: janhoy pushed a commit to branch main
: in repository https://gitbox.apache.org/repos/asf/solr.git
:
:
: The following commit(s) were added to refs/heads/main by this push:
: new 15534754f49 SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets
: 15534754f49 is described below
:
: commit 15534754f492079e52288dd11abaf1c4261b3ea4
: Author: Jan Høydahl <ja...@apache.org>
: AuthorDate: Wed Dec 13 22:49:23 2023 +0100
:
: SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets
: ---
: solr/CHANGES.txt | 2 +
: solr/core/build.gradle | 2 +
: .../org/apache/solr/cli/ConfigSetUploadTool.java | 2 +
: .../org/apache/solr/cloud/ZkConfigSetService.java | 21 ++-
: .../solr/core/FileSystemConfigSetService.java | 28 +++-
: .../org/apache/solr/core/backup/BackupManager.java | 23 ++-
: .../handler/configsets/UploadConfigSetFileAPI.java | 8 +-
: .../org/apache/solr/util/FileTypeMagicUtil.java | 166 +++++++++++++++++++++
: solr/core/src/resources/magic/executables | 74 +++++++++
: solr/core/src/test-files/magic/HelloWorld.java.txt | 5 +
: .../test-files/magic/HelloWorldJavaClass.class.bin | Bin 0 -> 426 bytes
: solr/core/src/test-files/magic/README.md | 29 ++++
: solr/core/src/test-files/magic/hello.tar.bin | Bin 0 -> 4096 bytes
: solr/core/src/test-files/magic/plain.txt | 1 +
: solr/core/src/test-files/magic/shell.sh.txt | 2 +
: .../org/apache/solr/cloud/TestConfigSetsAPI.java | 141 +++++++++++------
: .../apache/solr/util/FileTypeMagicUtilTest.java | 54 +++++++
: solr/licenses/simplemagic-1.17.jar.sha1 | 1 +
: solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt | 15 ++
: solr/licenses/simplemagic-NOTICE.txt | 0
: .../solr/common/cloud/ZkMaintenanceUtils.java | 2 +
: versions.lock | 1 +
: versions.props | 1 +
: 23 files changed, 522 insertions(+), 56 deletions(-)
:
: diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
: index 40da8c435ac..2fd94304a24 100644
: --- a/solr/CHANGES.txt
: +++ b/solr/CHANGES.txt
: @@ -169,6 +169,8 @@ Other Changes
: * SOLR-17091: dev tools script cloud.sh became broken after changes in 9.3 added a new -slim.tgz file it was not expecting
: cloud.sh has been updated to ignore the -slim.tgz version of the tarball.
:
: +* SOLR-16949: Restrict certain file types from being uploaded to or downloaded from Config Sets (janhoy, Houston Putman)
: +
: ================== 9.4.0 ==================
: New Features
: ---------------------
: diff --git a/solr/core/build.gradle b/solr/core/build.gradle
: index 61ecd1713af..ed2c8a370ae 100644
: --- a/solr/core/build.gradle
: +++ b/solr/core/build.gradle
: @@ -159,6 +159,8 @@ dependencies {
:
: compileOnly 'com.github.stephenc.jcip:jcip-annotations'
:
: + implementation 'com.j256.simplemagic:simplemagic'
: +
: // -- Test Dependencies
:
: testRuntimeOnly 'org.slf4j:jcl-over-slf4j'
: diff --git a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
: index 5fd4a538bd7..6576742a195 100644
: --- a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
: +++ b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java
: @@ -27,6 +27,7 @@ import org.apache.solr.client.solrj.impl.SolrZkClientTimeout;
: import org.apache.solr.common.cloud.SolrZkClient;
: import org.apache.solr.common.cloud.ZkMaintenanceUtils;
: import org.apache.solr.core.ConfigSetService;
: +import org.apache.solr.util.FileTypeMagicUtil;
: import org.slf4j.Logger;
: import org.slf4j.LoggerFactory;
:
: @@ -101,6 +102,7 @@ public class ConfigSetUploadTool extends ToolBase {
: + cli.getOptionValue("confname")
: + " to ZooKeeper at "
: + zkHost);
: + FileTypeMagicUtil.assertConfigSetFolderLegal(confPath);
: ZkMaintenanceUtils.uploadToZK(
: zkClient,
: confPath,
: diff --git a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
: index f02404d636d..9abde098e1c 100644
: --- a/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
: +++ b/solr/core/src/java/org/apache/solr/cloud/ZkConfigSetService.java
: @@ -22,6 +22,7 @@ import java.nio.file.Path;
: import java.util.ArrayList;
: import java.util.Collections;
: import java.util.List;
: +import java.util.Locale;
: import java.util.Map;
: import java.util.Objects;
: import org.apache.solr.client.solrj.cloud.SolrCloudManager;
: @@ -39,6 +40,7 @@ import org.apache.solr.core.CoreContainer;
: import org.apache.solr.core.CoreDescriptor;
: import org.apache.solr.core.SolrConfig;
: import org.apache.solr.core.SolrResourceLoader;
: +import org.apache.solr.util.FileTypeMagicUtil;
: import org.apache.zookeeper.CreateMode;
: import org.apache.zookeeper.KeeperException;
: import org.apache.zookeeper.data.Stat;
: @@ -199,6 +201,15 @@ public class ZkConfigSetService extends ConfigSetService {
: try {
: if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {
: log.warn("Not including uploading file to config, as it is a forbidden type: {}", fileName);
: + } else if (FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
: + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
: + throw new SolrException(
: + SolrException.ErrorCode.BAD_REQUEST,
: + String.format(
: + Locale.ROOT,
: + "Not uploading file %s to config, as it matched the MAGIC signature of a forbidden mime type %s",
: + fileName,
: + mimeType));
: } else {
: // if overwriteOnExists is true then zkClient#makePath failOnExists is set to false
: zkClient.makePath(filePath, data, CreateMode.PERSISTENT, null, !overwriteOnExists, true);
: @@ -340,7 +351,15 @@ public class ZkConfigSetService extends ConfigSetService {
: } else {
: log.debug("Copying zk node {} to {}", fromZkFilePath, toZkFilePath);
: byte[] data = zkClient.getData(fromZkFilePath, null, null, true);
: - zkClient.makePath(toZkFilePath, data, true);
: + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
: + zkClient.makePath(toZkFilePath, data, true);
: + } else {
: + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
: + log.warn(
: + "Skipping copy of file {} in ZK, as it matched the MAGIC signature of a forbidden mime type {}",
: + fromZkFilePath,
: + mimeType);
: + }
: }
: }
:
: diff --git a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
: index 1f1a42b15e1..4b041252211 100644
: --- a/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
: +++ b/solr/core/src/java/org/apache/solr/core/FileSystemConfigSetService.java
: @@ -37,6 +37,7 @@ import java.util.stream.Stream;
: import org.apache.solr.common.SolrException;
: import org.apache.solr.common.cloud.ZkMaintenanceUtils;
: import org.apache.solr.common.util.Utils;
: +import org.apache.solr.util.FileTypeMagicUtil;
: import org.slf4j.Logger;
: import org.slf4j.LoggerFactory;
:
: @@ -150,9 +151,17 @@ public class FileSystemConfigSetService extends ConfigSetService {
: if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fileName)) {
: log.warn("Not including uploading file to config, as it is a forbidden type: {}", fileName);
: } else {
: - Path filePath = getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));
: - if (!Files.exists(filePath) || overwriteOnExists) {
: - Files.write(filePath, data);
: + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
: + Path filePath = getConfigDir(configName).resolve(normalizePathToOsSeparator(fileName));
: + if (!Files.exists(filePath) || overwriteOnExists) {
: + Files.write(filePath, data);
: + }
: + } else {
: + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
: + log.warn(
: + "Not including uploading file {}, as it matched the MAGIC signature of a forbidden mime type {}",
: + fileName,
: + mimeType);
: }
: }
: }
: @@ -205,8 +214,17 @@ public class FileSystemConfigSetService extends ConfigSetService {
: "Not including uploading file to config, as it is a forbidden type: {}",
: file.getFileName());
: } else {
: - Files.copy(
: - file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);
: + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(Files.newInputStream(file))) {
: + Files.copy(
: + file, target.resolve(source.relativize(file).toString()), REPLACE_EXISTING);
: + } else {
: + String mimeType =
: + FileTypeMagicUtil.INSTANCE.guessMimeType(Files.newInputStream(file));
: + log.warn(
: + "Not copying file {}, as it matched the MAGIC signature of a forbidden mime type {}",
: + file.getFileName(),
: + mimeType);
: + }
: }
: return FileVisitResult.CONTINUE;
: }
: diff --git a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
: index be6a1a83c2f..8ff78b27e08 100644
: --- a/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
: +++ b/solr/core/src/java/org/apache/solr/core/backup/BackupManager.java
: @@ -40,6 +40,7 @@ import org.apache.solr.common.cloud.ZkStateReader;
: import org.apache.solr.common.util.Utils;
: import org.apache.solr.core.ConfigSetService;
: import org.apache.solr.core.backup.repository.BackupRepository;
: +import org.apache.solr.util.FileTypeMagicUtil;
: import org.apache.zookeeper.CreateMode;
: import org.apache.zookeeper.KeeperException;
: import org.slf4j.Logger;
: @@ -349,8 +350,16 @@ public class BackupManager {
: if (data == null) {
: data = new byte[0];
: }
: - try (OutputStream os = repository.createOutput(uri)) {
: - os.write(data);
: + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
: + try (OutputStream os = repository.createOutput(uri)) {
: + os.write(data);
: + }
: + } else {
: + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(data);
: + log.warn(
: + "Not including zookeeper file {} in backup, as it matched the MAGIC signature of a forbidden mime type {}",
: + filePath,
: + mimeType);
: }
: }
: } else {
: @@ -379,7 +388,15 @@ public class BackupManager {
: // probably ok since the config file should be small.
: byte[] arr = new byte[(int) is.length()];
: is.readBytes(arr, 0, (int) is.length());
: - configSetService.uploadFileToConfig(configName, filePath, arr, false);
: + if (!FileTypeMagicUtil.isFileForbiddenInConfigset(arr)) {
: + configSetService.uploadFileToConfig(configName, filePath, arr, false);
: + } else {
: + String mimeType = FileTypeMagicUtil.INSTANCE.guessMimeType(arr);
: + log.warn(
: + "Not including zookeeper file {} in restore, as it matched the MAGIC signature of a forbidden mime type {}",
: + filePath,
: + mimeType);
: + }
: }
: }
: break;
: diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
: index 98889aff563..2380a79a92b 100644
: --- a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
: +++ b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSetFileAPI.java
: @@ -27,6 +27,7 @@ import org.apache.solr.common.params.ConfigSetParams;
: import org.apache.solr.core.CoreContainer;
: import org.apache.solr.request.SolrQueryRequest;
: import org.apache.solr.response.SolrQueryResponse;
: +import org.apache.solr.util.FileTypeMagicUtil;
:
: /**
: * V2 API for adding or updating a single file within a configset.
: @@ -67,11 +68,13 @@ public class UploadConfigSetFileAPI extends ConfigSetAPIBase {
: if (fixedSingleFilePath.charAt(0) == '/') {
: fixedSingleFilePath = fixedSingleFilePath.substring(1);
: }
: + byte[] data = inputStream.readAllBytes();
: if (fixedSingleFilePath.isEmpty()) {
: throw new SolrException(
: SolrException.ErrorCode.BAD_REQUEST,
: "The file path provided for upload, '" + singleFilePath + "', is not valid.");
: - } else if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fixedSingleFilePath)) {
: + } else if (ZkMaintenanceUtils.isFileForbiddenInConfigSets(fixedSingleFilePath)
: + || FileTypeMagicUtil.isFileForbiddenInConfigset(data)) {
: throw new SolrException(
: SolrException.ErrorCode.BAD_REQUEST,
: "The file type provided for upload, '"
: @@ -87,8 +90,7 @@ public class UploadConfigSetFileAPI extends ConfigSetAPIBase {
: // For creating the baseNode, the cleanup parameter is only allowed to be true when
: // singleFilePath is not passed.
: createBaseNode(configSetService, overwritesExisting, requestIsTrusted, configSetName);
: - configSetService.uploadFileToConfig(
: - configSetName, fixedSingleFilePath, inputStream.readAllBytes(), allowOverwrite);
: + configSetService.uploadFileToConfig(configSetName, fixedSingleFilePath, data, allowOverwrite);
: }
: }
: }
: diff --git a/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java b/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java
: new file mode 100644
: index 00000000000..cfb6c9fa0af
: --- /dev/null
: +++ b/solr/core/src/java/org/apache/solr/util/FileTypeMagicUtil.java
: @@ -0,0 +1,166 @@
: +/*
: + * Licensed to the Apache Software Foundation (ASF) under one or more
: + * contributor license agreements. See the NOTICE file distributed with
: + * this work for additional information regarding copyright ownership.
: + * The ASF licenses this file to You under the Apache License, Version 2.0
: + * (the "License"); you may not use this file except in compliance with
: + * the License. You may obtain a copy of the License at
: + *
: + * http://www.apache.org/licenses/LICENSE-2.0
: + *
: + * Unless required by applicable law or agreed to in writing, software
: + * distributed under the License is distributed on an "AS IS" BASIS,
: + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
: + * See the License for the specific language governing permissions and
: + * limitations under the License.
: + */
: +
: +package org.apache.solr.util;
: +
: +import com.j256.simplemagic.ContentInfo;
: +import com.j256.simplemagic.ContentInfoUtil;
: +import com.j256.simplemagic.ContentType;
: +import java.io.ByteArrayInputStream;
: +import java.io.IOException;
: +import java.io.InputStream;
: +import java.nio.file.FileVisitResult;
: +import java.nio.file.Files;
: +import java.nio.file.Path;
: +import java.nio.file.SimpleFileVisitor;
: +import java.nio.file.attribute.BasicFileAttributes;
: +import java.util.Arrays;
: +import java.util.HashSet;
: +import java.util.Locale;
: +import java.util.Set;
: +import org.apache.solr.common.SolrException;
: +
: +/** Utility class to guess the mime type of file based on its magic number. */
: +public class FileTypeMagicUtil implements ContentInfoUtil.ErrorCallBack {
: + private final ContentInfoUtil util;
: + private static final Set<String> SKIP_FOLDERS = new HashSet<>(Arrays.asList(".", ".."));
: +
: + public static FileTypeMagicUtil INSTANCE = new FileTypeMagicUtil();
: +
: + FileTypeMagicUtil() {
: + try {
: + util = new ContentInfoUtil("/magic/executables", this);
: + } catch (IOException e) {
: + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error parsing magic file", e);
: + }
: + }
: +
: + /**
: + * Asserts that an entire configset folder is legal to upload.
: + *
: + * @param confPath the path to the folder
: + * @throws SolrException if an illegal file is found in the folder structure
: + */
: + public static void assertConfigSetFolderLegal(Path confPath) throws IOException {
: + Files.walkFileTree(
: + confPath,
: + new SimpleFileVisitor<Path>() {
: + @Override
: + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs)
: + throws IOException {
: + // Read first 100 bytes of the file to determine the mime type
: + try (InputStream fileStream = Files.newInputStream(file)) {
: + byte[] bytes = new byte[100];
: + fileStream.read(bytes);
: + if (FileTypeMagicUtil.isFileForbiddenInConfigset(bytes)) {
: + throw new SolrException(
: + SolrException.ErrorCode.BAD_REQUEST,
: + String.format(
: + Locale.ROOT,
: + "Not uploading file %s to configset, as it matched the MAGIC signature of a forbidden mime type %s",
: + file,
: + FileTypeMagicUtil.INSTANCE.guessMimeType(bytes)));
: + }
: + return FileVisitResult.CONTINUE;
: + }
: + }
: +
: + @Override
: + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs)
: + throws IOException {
: + if (SKIP_FOLDERS.contains(dir.getFileName().toString()))
: + return FileVisitResult.SKIP_SUBTREE;
: +
: + return FileVisitResult.CONTINUE;
: + }
: + });
: + }
: +
: + /**
: + * Guess the mime type of file based on its magic number.
: + *
: + * @param stream input stream of the file
: + * @return string with content-type or "application/octet-stream" if unknown
: + */
: + public String guessMimeType(InputStream stream) {
: + try {
: + ContentInfo info = util.findMatch(stream);
: + if (info == null) {
: + return ContentType.OTHER.getMimeType();
: + }
: + return info.getContentType().getMimeType();
: + } catch (IOException e) {
: + throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
: + }
: + }
: +
: + /**
: + * Guess the mime type of file bytes based on its magic number.
: + *
: + * @param bytes the first bytes at start of the file
: + * @return string with content-type or "application/octet-stream" if unknown
: + */
: + public String guessMimeType(byte[] bytes) {
: + return guessMimeType(new ByteArrayInputStream(bytes));
: + }
: +
: + @Override
: + public void error(String line, String details, Exception e) {
: + throw new SolrException(
: + SolrException.ErrorCode.SERVER_ERROR,
: + String.format(Locale.ROOT, "%s: %s", line, details),
: + e);
: + }
: +
: + /**
: + * Determine forbidden file type based on magic bytes matching of the file itself. Forbidden types
: + * are:
: + *
: + * <ul>
: + * <li><code>application/x-java-applet</code>: java class file
: + * <li><code>application/zip</code>: jar or zip archives
: + * <li><code>application/x-tar</code>: tar archives
: + * <li><code>text/x-shellscript</code>: shell or bash script
: + * </ul>
: + *
: + * @param fileStream stream from the file content
: + * @return true if file is among the forbidden mime-types
: + */
: + public static boolean isFileForbiddenInConfigset(InputStream fileStream) {
: + return forbiddenTypes.contains(FileTypeMagicUtil.INSTANCE.guessMimeType(fileStream));
: + }
: +
: + /**
: + * Determine forbidden file type based on magic bytes matching of the first bytes of the file.
: + *
: + * @param bytes byte array of the file content
: + * @return true if file is among the forbidden mime-types
: + */
: + public static boolean isFileForbiddenInConfigset(byte[] bytes) {
: + if (bytes == null || bytes.length == 0)
: + return false; // A ZK znode may be a folder with no content
: + return isFileForbiddenInConfigset(new ByteArrayInputStream(bytes));
: + }
: +
: + private static final Set<String> forbiddenTypes =
: + new HashSet<>(
: + Arrays.asList(
: + System.getProperty(
: + "solr.configset.upload.mimetypes.forbidden",
: + "application/x-java-applet,application/zip,application/x-tar,text/x-shellscript")
: + .split(",")));
: +}
: diff --git a/solr/core/src/resources/magic/executables b/solr/core/src/resources/magic/executables
: new file mode 100644
: index 00000000000..04094eaf797
: --- /dev/null
: +++ b/solr/core/src/resources/magic/executables
: @@ -0,0 +1,74 @@
: +# Licensed to the Apache Software Foundation (ASF) under one or more
: +# contributor license agreements. See the NOTICE file distributed with
: +# this work for additional information regarding copyright ownership.
: +# The ASF licenses this file to You under the Apache License, Version 2.0
: +# (the "License"); you may not use this file except in compliance with
: +# the License. You may obtain a copy of the License at
: +#
: +# http://www.apache.org/licenses/LICENSE-2.0
: +#
: +# Unless required by applicable law or agreed to in writing, software
: +# distributed under the License is distributed on an "AS IS" BASIS,
: +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
: +# See the License for the specific language governing permissions and
: +# limitations under the License.
: +
: +# POSIX tar archives
: +# URL: https://en.wikipedia.org/wiki/Tar_(computing)
: +# Reference: https://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5&manpath=FreeBSD+8-current
: +# header mainly padded with nul bytes
: +500 quad 0
: +!:strength /2
: +# filename or extended attribute printable strings in range space null til umlaut ue
: +>0 ubeshort >0x1F00
: +>>0 ubeshort <0xFCFD
: +# last 4 header bytes often null but tar\0 in gtarfail2.tar gtarfail.tar-bad
: +# at https://sourceforge.net/projects/s-tar/files/testscripts/
: +>>>508 ubelong&0x8B9E8DFF 0
: +# nul, space or ascii digit 0-7 at start of mode
: +>>>>100 ubyte&0xC8 =0
: +>>>>>101 ubyte&0xC8 =0
: +# nul, space at end of check sum
: +>>>>>>155 ubyte&0xDF =0
: +# space or ascii digit 0 at start of check sum
: +>>>>>>>148 ubyte&0xEF =0x20
: +# check for specific 1st member name that indicates other mime type and file name suffix
: +>>>>>>>>0 string TpmEmuTpms/permall
: +!:mime application/x-tar
: +!:ext tar
: +# other stuff in padding
: +# some implementations add new fields to the blank area at the end of the header record
: +# created for example by DOS TAR 3.20g 1994 Tim V.Shapore with -j option
: +>>257 ulong !0 tar archive (old)
: +!:mime application/x-tar
: +!:ext tar
: +# magic in newer, GNU, posix variants
: +>257 string =ustar
: +# 2 last char of magic and UStar version because string expression does not work
: +# 2 space characters followed by a null for GNU variant
: +>>261 ubelong =0x72202000 POSIX tar archive (GNU)
: +!:mime application/x-gtar
: +!:ext tar/gtar
: +
: +
: +# Zip archives (Greg Roelofs, c/o zip-bugs@wkuvx1.wku.edu)
: +0 string PK\005\006 Zip archive data (empty)
: +0 string PK\003\004 Zip archive data
: +!:strength +1
: +!:mime application/zip
: +!:ext zip/cbz
: +
: +
: +# JAVA
: +0 belong 0xcafebabe
: +>4 ubelong >30 compiled Java class data,
: +!:mime application/x-java-applet
: +#!:mime application/java-byte-code
: +!:ext class
: +
: +
: +# SHELL scripts
: +#0 string/w : shell archive or script for antique kernel text
: +0 regex \^#!\\s?(/bin/|/usr/) POSIX shell script text executable
: +!:mime text/x-shellscript
: +!:ext sh/bash
: \ No newline at end of file
: diff --git a/solr/core/src/test-files/magic/HelloWorld.java.txt b/solr/core/src/test-files/magic/HelloWorld.java.txt
: new file mode 100644
: index 00000000000..ca9518d2afa
: --- /dev/null
: +++ b/solr/core/src/test-files/magic/HelloWorld.java.txt
: @@ -0,0 +1,5 @@
: +class HelloWorld {
: + public static void main(String[] args) {
: + System.out.println("Hellow world");
: + }
: +}
: \ No newline at end of file
: diff --git a/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin b/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin
: new file mode 100644
: index 00000000000..e15d0a6c5b9
: Binary files /dev/null and b/solr/core/src/test-files/magic/HelloWorldJavaClass.class.bin differ
: diff --git a/solr/core/src/test-files/magic/README.md b/solr/core/src/test-files/magic/README.md
: new file mode 100644
: index 00000000000..6e499a2f711
: --- /dev/null
: +++ b/solr/core/src/test-files/magic/README.md
: @@ -0,0 +1,29 @@
: +<!--
: + Licensed to the Apache Software Foundation (ASF) under one or more
: + contributor license agreements. See the NOTICE file distributed with
: + this work for additional information regarding copyright ownership.
: + The ASF licenses this file to You under the Apache License, Version 2.0
: + the "License"); you may not use this file except in compliance with
: + the License. You may obtain a copy of the License at
: +
: + http://www.apache.org/licenses/LICENSE-2.0
: +
: + Unless required by applicable law or agreed to in writing, software
: + distributed under the License is distributed on an "AS IS" BASIS,
: + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
: + See the License for the specific language governing permissions and
: + limitations under the License.
: + -->
: +
: +The two binary files were created by the following commands:
: +
: +```bash
: +echo "Hello" > hello.txt && \
: + tar -cvf hello.tar.bin hello.txt && \
: + rm hello.txt
: +
: +cp HelloWorld.java.txt HelloWorld.java && \
: + javac HelloWorld.java && \
: + mv HelloWorld.class HelloWorldJavaClass.class.bin && \
: + rm HelloWorld.java
: +```
: \ No newline at end of file
: diff --git a/solr/core/src/test-files/magic/hello.tar.bin b/solr/core/src/test-files/magic/hello.tar.bin
: new file mode 100644
: index 00000000000..68ca23c362a
: Binary files /dev/null and b/solr/core/src/test-files/magic/hello.tar.bin differ
: diff --git a/solr/core/src/test-files/magic/plain.txt b/solr/core/src/test-files/magic/plain.txt
: new file mode 100644
: index 00000000000..70c379b63ff
: --- /dev/null
: +++ b/solr/core/src/test-files/magic/plain.txt
: @@ -0,0 +1 @@
: +Hello world
: \ No newline at end of file
: diff --git a/solr/core/src/test-files/magic/shell.sh.txt b/solr/core/src/test-files/magic/shell.sh.txt
: new file mode 100644
: index 00000000000..9ea411e111d
: --- /dev/null
: +++ b/solr/core/src/test-files/magic/shell.sh.txt
: @@ -0,0 +1,2 @@
: +#! /usr/bin/env bash
: +echo Hello
: \ No newline at end of file
: diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
: index 1da60d85ff9..f7c9431c296 100644
: --- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
: +++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java
: @@ -45,6 +45,7 @@ import java.util.LinkedHashMap;
: import java.util.List;
: import java.util.Locale;
: import java.util.Map;
: +import java.util.Objects;
: import java.util.Properties;
: import java.util.Set;
: import java.util.concurrent.TimeUnit;
: @@ -592,14 +593,14 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
: assertEquals(
: "Can't overwrite an existing configset unless the overwrite parameter is set",
: 400,
: - uploadConfigSet(configsetName, configsetSuffix, null, false, false, v2, false));
: + uploadConfigSet(configsetName, configsetSuffix, null, false, false, v2, false, false));
: unIgnoreException("The configuration regulartestOverwrite-1 already exists in zookeeper");
: assertEquals(
: "Expecting version to remain equal",
: solrconfigZkVersion,
: getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml"));
: assertEquals(
: - 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));
: + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false));
: assertTrue(
: "Expecting version bump",
: solrconfigZkVersion
: @@ -638,13 +639,14 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
: zkClient.makePath(f, true);
: }
: assertEquals(
: - 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));
: + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false));
: for (String f : extraFiles) {
: assertTrue(
: "Expecting file " + f + " to exist in ConfigSet but it's gone",
: zkClient.exists(f, true));
: }
: - assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false));
: + assertEquals(
: + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false, false));
: for (String f : extraFiles) {
: assertFalse(
: "Expecting file " + f + " to be deleted from ConfigSet but it wasn't",
: @@ -675,7 +677,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
: .withConnTimeOut(45000, TimeUnit.MILLISECONDS)
: .build()) {
: String configPath = "/configs/" + configsetName + configsetSuffix;
: - assertEquals(0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, true));
: + assertEquals(
: + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, true, false));
: for (String fileEnding : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
: String f = configPath + "/test." + fileEnding;
: assertFalse(
: @@ -710,7 +713,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
: getConfigZNodeVersion(zkClient, configsetName, configsetSuffix, "solrconfig.xml");
: // Was untrusted, overwrite with untrusted
: assertEquals(
: - 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));
: + 0, uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false));
: assertTrue(
: "Expecting version bump",
: solrconfigZkVersion
: @@ -721,7 +724,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
:
: // Was untrusted, overwrite with trusted but no cleanup
: assertEquals(
: - 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false));
: + 0,
: + uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false, false));
: assertTrue(
: "Expecting version bump",
: solrconfigZkVersion
: @@ -747,7 +751,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
:
: // Was untrusted, overwrite with trusted with cleanup
: assertEquals(
: - 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false));
: + 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false, false));
: assertTrue(
: "Expecting version bump",
: solrconfigZkVersion
: @@ -761,7 +765,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
: assertEquals(
: "Can't upload a trusted configset with an untrusted request",
: 400,
: - uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false));
: + uploadConfigSet(configsetName, configsetSuffix, null, true, false, v2, false, false));
: assertEquals(
: "Expecting version to remain equal",
: solrconfigZkVersion,
: @@ -773,7 +777,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
: assertEquals(
: "Can't upload a trusted configset with an untrusted request",
: 400,
: - uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false));
: + uploadConfigSet(configsetName, configsetSuffix, null, true, true, v2, false, false));
: assertEquals(
: "Expecting version to remain equal",
: solrconfigZkVersion,
: @@ -783,7 +787,8 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
:
: // Was trusted, overwrite with trusted no cleanup
: assertEquals(
: - 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false));
: + 0,
: + uploadConfigSet(configsetName, configsetSuffix, "solr", true, false, v2, false, false));
: assertTrue(
: "Expecting version bump",
: solrconfigZkVersion
: @@ -794,7 +799,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
:
: // Was trusted, overwrite with trusted with cleanup
: assertEquals(
: - 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false));
: + 0, uploadConfigSet(configsetName, configsetSuffix, "solr", true, true, v2, false, false));
: assertTrue(
: "Expecting version bump",
: solrconfigZkVersion
: @@ -1457,6 +1462,13 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
: .get("id"));
: }
:
: + @Test
: + public void testUploadWithForbiddenContent() throws Exception {
: + // Uploads a config set containing a script, a class file and jar file, will return 400 error
: + long res = uploadConfigSet("forbidden", "suffix", "foo", true, false, true, false, true);
: + assertEquals(400, res);
: + }
: +
: private static String getSecurityJson() {
: return "{\n"
: + " 'authentication':{\n"
: @@ -1511,7 +1523,7 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
: String configSetName, String suffix, String username, SolrZkClient zkClient, boolean v2)
: throws IOException {
: assertFalse(getConfigSetService().checkConfigExists(configSetName + suffix));
: - return uploadConfigSet(configSetName, suffix, username, false, false, v2, false);
: + return uploadConfigSet(configSetName, suffix, username, false, false, v2, false, false);
: }
:
: private long uploadConfigSet(
: @@ -1521,21 +1533,25 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
: boolean overwrite,
: boolean cleanup,
: boolean v2,
: - boolean forbiddenTypes)
: + boolean forbiddenTypes,
: + boolean forbiddenContent)
: throws IOException {
:
: + File zipFile;
: + if (forbiddenTypes) {
: + log.info("Uploading configset with forbidden file endings");
: + zipFile =
: + createTempZipFileWithForbiddenTypes(
: + "solr/configsets/upload/" + configSetName + "/solrconfig.xml");
: + } else if (forbiddenContent) {
: + log.info("Uploading configset with forbidden file content");
: + zipFile = createTempZipFileWithForbiddenContent("magic");
: + } else {
: + zipFile = createTempZipFile("solr/configsets/upload/" + configSetName);
: + }
: +
: // Read zipped sample config
: - return uploadGivenConfigSet(
: - forbiddenTypes
: - ? createTempZipFileWithForbiddenTypes(
: - "solr/configsets/upload/" + configSetName + "/solrconfig.xml")
: - : createTempZipFile("solr/configsets/upload/" + configSetName),
: - configSetName,
: - suffix,
: - username,
: - overwrite,
: - cleanup,
: - v2);
: + return uploadGivenConfigSet(zipFile, configSetName, suffix, username, overwrite, cleanup, v2);
: }
:
: private long uploadBadConfigSet(String configSetName, String suffix, String username, boolean v2)
: @@ -1702,31 +1718,68 @@ public class TestConfigSetsAPI extends SolrCloudTestCase {
: }
: }
:
: - private static void zipWithForbiddenEndings(File file, File zipfile) throws IOException {
: - OutputStream out = new FileOutputStream(zipfile);
: - ZipOutputStream zout = new ZipOutputStream(out);
: + /** Create a zip file (in the temp directory) containing files with forbidden content */
: + private File createTempZipFileWithForbiddenContent(String resourcePath) {
: try {
: - for (String fileType : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
: - zout.putNextEntry(new ZipEntry("test." + fileType));
: + final File zipFile = createTempFile("configset", "zip").toFile();
: + final File directory = SolrTestCaseJ4.getFile(resourcePath);
: + if (log.isInfoEnabled()) {
: + log.info("Directory: {}", directory.getAbsolutePath());
: + }
: + zipWithForbiddenContent(directory, zipFile);
: + if (log.isInfoEnabled()) {
: + log.info("Zipfile: {}", zipFile.getAbsolutePath());
: + }
: + return zipFile;
: + } catch (IOException e) {
: + throw new RuntimeException(e);
: + }
: + }
:
: - InputStream in = new FileInputStream(file);
: - try {
: - byte[] buffer = new byte[1024];
: - while (true) {
: - int readCount = in.read(buffer);
: - if (readCount < 0) {
: - break;
: + private static void zipWithForbiddenContent(File directory, File zipfile) throws IOException {
: + OutputStream out = Files.newOutputStream(zipfile.toPath());
: + assertTrue(directory.isDirectory());
: + try (ZipOutputStream zout = new ZipOutputStream(out)) {
: + // Copy in all files from the directory
: + for (File file : Objects.requireNonNull(directory.listFiles())) {
: + zout.putNextEntry(new ZipEntry(file.getName()));
: + zout.write(Files.readAllBytes(file.toPath()));
: + zout.closeEntry();
: + }
: + }
: + }
: +
: + private static void zipWithForbiddenEndings(File fileOrDirectory, File zipfile)
: + throws IOException {
: + OutputStream out = new FileOutputStream(zipfile);
: + try (ZipOutputStream zout = new ZipOutputStream(out)) {
: + if (fileOrDirectory.isFile()) {
: + // Create entries with given file, one for each forbidden endding
: + for (String fileType : ZkMaintenanceUtils.DEFAULT_FORBIDDEN_FILE_TYPES) {
: + zout.putNextEntry(new ZipEntry("test." + fileType));
: +
: + try (InputStream in = new FileInputStream(fileOrDirectory)) {
: + byte[] buffer = new byte[1024];
: + while (true) {
: + int readCount = in.read(buffer);
: + if (readCount < 0) {
: + break;
: + }
: + zout.write(buffer, 0, readCount);
: }
: - zout.write(buffer, 0, readCount);
: }
: - } finally {
: - in.close();
: - }
:
: - zout.closeEntry();
: + zout.closeEntry();
: + }
: + }
: + if (fileOrDirectory.isDirectory()) {
: + // Copy in all files from the directory
: + for (File file : Objects.requireNonNull(fileOrDirectory.listFiles())) {
: + zout.putNextEntry(new ZipEntry(file.getName()));
: + zout.write(Files.readAllBytes(file.toPath()));
: + zout.closeEntry();
: + }
: }
: - } finally {
: - zout.close();
: }
: }
:
: diff --git a/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java b/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java
: new file mode 100644
: index 00000000000..b8e9a35a3d9
: --- /dev/null
: +++ b/solr/core/src/test/org/apache/solr/util/FileTypeMagicUtilTest.java
: @@ -0,0 +1,54 @@
: +/*
: + * Licensed to the Apache Software Foundation (ASF) under one or more
: + * contributor license agreements. See the NOTICE file distributed with
: + * this work for additional information regarding copyright ownership.
: + * The ASF licenses this file to You under the Apache License, Version 2.0
: + * (the "License"); you may not use this file except in compliance with
: + * the License. You may obtain a copy of the License at
: + *
: + * http://www.apache.org/licenses/LICENSE-2.0
: + *
: + * Unless required by applicable law or agreed to in writing, software
: + * distributed under the License is distributed on an "AS IS" BASIS,
: + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
: + * See the License for the specific language governing permissions and
: + * limitations under the License.
: + */
: +
: +package org.apache.solr.util;
: +
: +import org.apache.solr.SolrTestCaseJ4;
: +
: +public class FileTypeMagicUtilTest extends SolrTestCaseJ4 {
: + public void testGuessMimeType() {
: + assertEquals(
: + "application/x-java-applet",
: + FileTypeMagicUtil.INSTANCE.guessMimeType(
: + FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin")));
: + assertEquals(
: + "application/zip",
: + FileTypeMagicUtil.INSTANCE.guessMimeType(
: + FileTypeMagicUtil.class.getResourceAsStream(
: + "/runtimecode/containerplugin.v.1.jar.bin")));
: + assertEquals(
: + "application/x-tar",
: + FileTypeMagicUtil.INSTANCE.guessMimeType(
: + FileTypeMagicUtil.class.getResourceAsStream("/magic/hello.tar.bin")));
: + assertEquals(
: + "text/x-shellscript",
: + FileTypeMagicUtil.INSTANCE.guessMimeType(
: + FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt")));
: + }
: +
: + public void testIsFileForbiddenInConfigset() {
: + assertTrue(
: + FileTypeMagicUtil.isFileForbiddenInConfigset(
: + FileTypeMagicUtil.class.getResourceAsStream("/magic/HelloWorldJavaClass.class.bin")));
: + assertTrue(
: + FileTypeMagicUtil.isFileForbiddenInConfigset(
: + FileTypeMagicUtil.class.getResourceAsStream("/magic/shell.sh.txt")));
: + assertFalse(
: + FileTypeMagicUtil.isFileForbiddenInConfigset(
: + FileTypeMagicUtil.class.getResourceAsStream("/magic/plain.txt")));
: + }
: +}
: diff --git a/solr/licenses/simplemagic-1.17.jar.sha1 b/solr/licenses/simplemagic-1.17.jar.sha1
: new file mode 100644
: index 00000000000..cf101094cc8
: --- /dev/null
: +++ b/solr/licenses/simplemagic-1.17.jar.sha1
: @@ -0,0 +1 @@
: +b6e2d1e47d7172e57fa858a2e3940c09a590e61e
: diff --git a/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt b/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt
: new file mode 100644
: index 00000000000..9228230f933
: --- /dev/null
: +++ b/solr/licenses/simplemagic-LICENSE-BSD_LIKE.txt
: @@ -0,0 +1,15 @@
: +ISC License (https://opensource.org/licenses/ISC)
: +
: +Copyright 2021, Gray Watson
: +
: +Permission to use, copy, modify, and/or distribute this software for any
: +purpose with or without fee is hereby granted, provided that the above
: +copyright notice and this permission notice appear in all copies.
: +
: +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
: +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
: +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
: +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
: +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
: +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
: +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
: \ No newline at end of file
: diff --git a/solr/licenses/simplemagic-NOTICE.txt b/solr/licenses/simplemagic-NOTICE.txt
: new file mode 100644
: index 00000000000..e69de29bb2d
: diff --git a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
: index d571339880c..e40294a6683 100644
: --- a/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
: +++ b/solr/solrj-zookeeper/src/java/org/apache/solr/common/cloud/ZkMaintenanceUtils.java
: @@ -348,6 +348,7 @@ public class ZkMaintenanceUtils {
: USE_FORBIDDEN_FILE_TYPES);
: return FileVisitResult.CONTINUE;
: }
: + // TODO: Cannot check MAGIC header for file since FileTypeGuesser is in core
: String zkNode = createZkNodeName(zkPath, rootPath, file);
: try {
: // if the path exists (and presumably we're uploading data to it) just set its data
: @@ -437,6 +438,7 @@ public class ZkMaintenanceUtils {
: if (isFileForbiddenInConfigSets(zkPath)) {
: log.warn("Skipping download of file from ZK, as it is a forbidden type: {}", zkPath);
: } else {
: + // TODO: Cannot check MAGIC header for file since FileTypeGuesser is in core
: if (copyDataDown(zkClient, zkPath, file) == 0) {
: Files.createFile(file);
: }
: diff --git a/versions.lock b/versions.lock
: index 9c0f9ef53d4..6b6ea467e0d 100644
: --- a/versions.lock
: +++ b/versions.lock
: @@ -62,6 +62,7 @@ com.googlecode.plist:dd-plist:1.24 (1 constraints: 300c84f5)
: com.healthmarketscience.jackcess:jackcess:4.0.2 (1 constraints: 5d0cf201)
: com.healthmarketscience.jackcess:jackcess-encrypt:4.0.1 (1 constraints: 5c0cf101)
: com.ibm.icu:icu4j:70.1 (1 constraints: a90f1784)
: +com.j256.simplemagic:simplemagic:1.17 (1 constraints: dd04f830)
: com.jayway.jsonpath:json-path:2.8.0 (2 constraints: 6c12952c)
: com.lmax:disruptor:3.4.4 (1 constraints: 0d050a36)
: com.mchange:c3p0:0.9.5.5 (1 constraints: c80c571b)
: diff --git a/versions.props b/versions.props
: index 10b583c43e2..cfa127e1094 100644
: --- a/versions.props
: +++ b/versions.props
: @@ -13,6 +13,7 @@ com.google.cloud:google-cloud-bom=0.204.0
: com.google.errorprone:*=2.23.0
: com.google.guava:guava=32.1.3-jre
: com.google.re2j:re2j=1.7
: +com.j256.simplemagic:simplemagic=1.17
: com.jayway.jsonpath:json-path=2.8.0
: com.lmax:disruptor=3.4.4
: com.tdunning:t-digest=3.1
:
:
-Hoss