You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zh...@apache.org on 2022/05/23 07:01:02 UTC
[dolphinscheduler] 13/33: [fix] Enhance name pre checker in resource center (#10094)
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.0.0-beta-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit cbfeb7f7777e846346e4b46897cd32071ea1a9ef
Author: Jiajie Zhong <zh...@hotmail.com>
AuthorDate: Wed May 18 18:40:17 2022 +0800
[fix] Enhance name pre checker in resource center (#10094)
* [fix] Enhance name pre checker in resource center
Add file name and directory checker to avoid directory traversal
* add some missing change and change docs
* change var name in directoryTraversal
* Fix ci
(cherry picked from commit 63f835715f8ca8bff79c0e7177ebfa5917ebb3bd)
---
docs/docs/en/guide/resource/file-manage.md | 31 +++++++++++++---------
docs/docs/zh/guide/resource/file-manage.md | 31 +++++++++++++---------
.../apache/dolphinscheduler/api/enums/Status.java | 2 +-
.../api/service/impl/ResourcesServiceImpl.java | 21 +++++++++++++--
.../dolphinscheduler/common/utils/FileUtils.java | 21 +++++++++++++++
.../common/utils/FileUtilsTest.java | 30 +++++++++++++++++++++
6 files changed, 108 insertions(+), 28 deletions(-)
diff --git a/docs/docs/en/guide/resource/file-manage.md b/docs/docs/en/guide/resource/file-manage.md
index ae3e8d2ec4..87b965801b 100644
--- a/docs/docs/en/guide/resource/file-manage.md
+++ b/docs/docs/en/guide/resource/file-manage.md
@@ -4,40 +4,47 @@ When third party jars are used in the scheduling process or user defined scripts
-- Create a file
- > The file format supports the following types: txt, log, sh, conf, cfg, py, java, sql, xml, hql, properties.
+## Basic Operator
+
+### Create a File
+
+The file format supports the following types: txt, log, sh, conf, cfg, py, java, sql, xml, hql, properties.
-- upload files
+### Upload Files
-> Upload file: Click the "Upload File" button to upload, drag the file to the upload area, the file name will be automatically completed with the uploaded file name.
+Click the "Upload File" button to upload, drag the file to the upload area, the file name will be automatically completed with the uploaded file name.
-- File View
+### View File Content
-> For the files that can be viewed, click the file name to view the file details.
+ For the files that can be viewed, click the file name to view the file details.
-- Download file
+### Download file
> Click the "Download" button in the file list to download the file or click the "Download" button in the upper right corner of the file details to download the file.
-- File rename
+### Rename File
-- delete
- > File list -> Click the "Delete" button to delete the specified file.
+### Delete File
-- Re-upload file
+File list -> Click the "Delete" button to delete the specified file.
- > Re-upload file: Click the "Re-upload File" button to upload a new file to replace the old file, drag the file to the re-upload area, the file name will be automatically completed with the new uploaded file name.
+### Re-upload file
+
+Click the "Re-upload File" button to upload a new file to replace the old file, drag the file to the re-upload area, the file name will be automatically completed with the new uploaded file name.
+> Note: File name or source name of your local file can not contain specific characters like `.` or `/` when you trying to
+> upload, create or rename file in resource center.
+
## Example
The example uses a simple shell script to demonstrate the use of resource center files in workflow definitions. The same is true for tasks such as MR and Spark, which require jar packages.
diff --git a/docs/docs/zh/guide/resource/file-manage.md b/docs/docs/zh/guide/resource/file-manage.md
index 77d9509548..a0d2f0ab56 100644
--- a/docs/docs/zh/guide/resource/file-manage.md
+++ b/docs/docs/zh/guide/resource/file-manage.md
@@ -2,43 +2,48 @@
当在调度过程中需要使用到第三方的 jar 或者用户需要自定义脚本的情况，可以通过在该页面完成相关操作。可创建的文件类型包括：`txt/log/sh/conf/py/java` 等。并且可以对文件进行编辑、重命名、下载和删除等操作。
+## 基础操作
+
-* 创建文件
+### 创建文件
- > 文件格式支持以下几种类型：txt、log、sh、conf、cfg、py、java、sql、xml、hql、properties
+文件格式支持以下几种类型：txt、log、sh、conf、cfg、py、java、sql、xml、hql、properties
-* 上传文件
- > 上传文件：点击"上传文件"按钮进行上传，将文件拖拽到上传区域，文件名会自动以上传的文件名称补全
+### 上传文件
+
+上传文件：点击"上传文件"按钮进行上传，将文件拖拽到上传区域，文件名会自动以上传的文件名称补全
-* 文件查看
+### 文件查看
- > 对可查看的文件类型，点击文件名称，可查看文件详情
+对可查看的文件类型，点击文件名称，可查看文件详情
-* 下载文件
+### 下载文件
- > 点击文件列表的"下载"按钮下载文件或者在文件详情中点击右上角"下载"按钮下载文件
+点击文件列表的"下载"按钮下载文件或者在文件详情中点击右上角"下载"按钮下载文件
-* 文件重命名
+### 文件重命名
-* 删除
+### 删除文件
-> 文件列表->点击"删除"按钮，删除指定文件
+文件列表->点击"删除"按钮，删除指定文件
-* 重新上传文件
+### 重新上传文件
- > 点击文件列表中的”重新上传文件“按钮进行重新上传文件，将文件拖拽到上传区域，文件名会自动以上传的文件名称补全
+点击文件列表中的”重新上传文件“按钮进行重新上传文件，将文件拖拽到上传区域，文件名会自动以上传的文件名称补全
+> 注意：上传、创建、重命名文件时，文件名和源文件名（上传时）均不能带有 `.` 以及 `/` 特殊符号。
+
## 任务样例
该样例主要通过一个简单的 shell 脚本，来演示如何在工作流定义中使用资源中心的文件。像 MR、Spark 等任务需要用到 jar 包，也是同理。
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
index b315c313cb..0847d6920d 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
@@ -395,7 +395,7 @@ public enum Status {
K8S_CLIENT_OPS_ERROR(1300006, "k8s error with exception {0}", "k8s操作报错[{0}]"),
VERIFY_K8S_NAMESPACE_ERROR(1300007, "verify k8s and namespace error", "验证k8s命名空间信息错误"),
DELETE_K8S_NAMESPACE_BY_ID_ERROR(1300008, "delete k8s namespace by id error", "删除命名空间错误"),
- VERIFY_PARAMETER_NAME_FAILED(1300009, "The file name verify failed", "文件命名校验失败"),
+ VERIFY_PARAMETER_NAME_FAILED(1300009, "The file name verify failed", "文件命名校验失败"),
STORE_OPERATE_CREATE_ERROR(1300010, "create the resource failed", "存储操作失败"),
GRANT_K8S_NAMESPACE_ERROR(1300011, "grant namespace error", "授权资源错误"),
QUERY_UNAUTHORIZED_NAMESPACE_ERROR(1300012, "query unauthorized namespace error", "查询未授权命名空间错误"),
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
index 200aeb1c65..1c5804cd81 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
@@ -129,8 +129,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
return result;
}
- if (name.endsWith(FOLDER_SEPARATOR)) {
- result.setCode(Status.VERIFY_PARAMETER_NAME_FAILED.getCode());
+ if (FileUtils.directoryTraversal(name)) {
+ putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
return result;
}
String fullName = getFullName(currentDir, name);
@@ -517,6 +517,19 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
private Result<Object> verifyFile(String name, ResourceType type, MultipartFile file) {
Result<Object> result = new Result<>();
putMsg(result, Status.SUCCESS);
+
+ if (FileUtils.directoryTraversal(name)) {
+ logger.error("file alias name {} verify failed", name);
+ putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
+ return result;
+ }
+
+ if (file != null && FileUtils.directoryTraversal(Objects.requireNonNull(file.getOriginalFilename()))) {
+ logger.error("file original name {} verify failed", file.getOriginalFilename());
+ putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
+ return result;
+ }
+
if (file != null) {
// file is empty
if (file.isEmpty()) {
@@ -973,6 +986,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
return result;
}
+ if (FileUtils.directoryTraversal(fileName)) {
+ putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
+ return result;
+ }
//check file suffix
String nameSuffix = fileSuffix.trim();
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
index e7817ffe80..bdcf62f76a 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/FileUtils.java
@@ -204,4 +204,25 @@ public class FileUtils {
}
}
+ /**
+ * Check whether the given string type of path can be traversal or not, return true if path could
+ * traversal, and return false if it is not.
+ *
+ * @param filename String type of filename
+ * @return whether file path could be traversal or not
+ */
+ public static boolean directoryTraversal(String filename){
+ if (filename.contains(FOLDER_SEPARATOR)) {
+ return true;
+ }
+ File file = new File(filename);
+ try {
+ File canonical = file.getCanonicalFile();
+ File absolute = file.getAbsoluteFile();
+ return !canonical.equals(absolute);
+ } catch (IOException e) {
+ return true;
+ }
+ }
+
}
diff --git a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
index d023b8651f..fdcaccd697 100644
--- a/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
+++ b/dolphinscheduler-common/src/test/java/org/apache/dolphinscheduler/common/utils/FileUtilsTest.java
@@ -88,4 +88,34 @@ public class FileUtilsTest {
Assert.assertEquals(content, fileContent);
}
+ @Test
+ public void testDirectoryTraversal() {
+ // test case which do not directory traversal
+ String path;
+ path = "abc.txt";
+ Assert.assertFalse(FileUtils.directoryTraversal(path));
+
+ path = "abc...txt";
+ Assert.assertFalse(FileUtils.directoryTraversal(path));
+
+ path = "..abc.txt";
+ Assert.assertFalse(FileUtils.directoryTraversal(path));
+
+ // test case which will directory traversal
+ path = "../abc.txt";
+ Assert.assertTrue(FileUtils.directoryTraversal(path));
+
+ path = "../../abc.txt";
+ Assert.assertTrue(FileUtils.directoryTraversal(path));
+
+ path = "abc../def.txt";
+ Assert.assertTrue(FileUtils.directoryTraversal(path));
+
+ path = "abc./def.txt";
+ Assert.assertTrue(FileUtils.directoryTraversal(path));
+
+ path = "abc/def...txt";
+ Assert.assertTrue(FileUtils.directoryTraversal(path));
+ }
+
}