You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zi...@apache.org on 2022/10/26 05:53:53 UTC
[dolphinscheduler] branch dev updated: [Feature-10495][Resource Center] Resource Center Refactor (#12076)
This is an automated email from the ASF dual-hosted git repository.
zihaoxiang pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 489e7fe4e2 [Feature-10495][Resource Center] Resource Center Refactor (#12076)
489e7fe4e2 is described below
commit 489e7fe4e2d980031c91eac2a4823b9f1713659a
Author: HanayoZz <er...@outlook.com>
AuthorDate: Wed Oct 26 01:53:44 2022 -0400
[Feature-10495][Resource Center] Resource Center Refactor (#12076)
* resource center refactor - S3 services connection
Co-authored-by: caishunfeng <ca...@gmail.com>
---
.../api/controller/ResourcesController.java | 151 +-
.../api/dto/resources/ResourceComponent.java | 4 +-
.../dto/resources/visitor/ResourceTreeVisitor.java | 43 +-
.../api/dto/resources/visitor/Visitor.java | 2 +-
.../permission/ResourcePermissionCheckService.java | 1 -
.../api/service/ResourcesService.java | 44 +-
.../api/service/UdfFuncService.java | 8 +-
.../api/service/impl/ResourcesServiceImpl.java | 1507 ++++++++++++--------
.../api/service/impl/UdfFuncServiceImpl.java | 55 +-
.../api/service/impl/UsersServiceImpl.java | 6 +-
.../api/controller/ResourcesControllerTest.java | 63 +-
.../api/controller/TenantControllerTest.java | 12 +-
.../resources/visitor/ResourceTreeVisitorTest.java | 58 +-
.../api/service/ResourcesServiceTest.java | 582 ++++----
.../api/service/TenantServiceTest.java | 4 +
.../api/service/UdfFuncServiceTest.java | 34 +-
.../dolphinscheduler/dao/entity/ResourcesTask.java | 56 +
.../dao/mapper/ResourceMapper.java | 1 -
.../dao/mapper/ResourceTaskMapper.java | 45 +
.../dao/mapper/TaskDefinitionMapper.java | 2 +-
.../dolphinscheduler/dao/mapper/UdfFuncMapper.java | 11 +-
.../dao/utils/ResourceProcessDefinitionUtils.java | 30 +
.../dao/mapper/ProcessDefinitionMapper.xml | 4 +-
.../dao/mapper/ResourceTaskMapper.xml | 58 +
.../dao/mapper/TaskDefinitionMapper.xml | 4 +-
.../dolphinscheduler/dao/mapper/UdfFuncMapper.xml | 14 +
.../src/main/resources/sql/dolphinscheduler_h2.sql | 18 +
.../main/resources/sql/dolphinscheduler_mysql.sql | 17 +
.../resources/sql/dolphinscheduler_postgresql.sql | 12 +
.../master/runner/task/BaseTaskProcessor.java | 27 +-
.../service/process/ProcessServiceImpl.java | 101 +-
.../service/storage/StorageEntity.java | 66 +
.../service/storage/StorageOperate.java | 34 +-
.../service/storage/impl/HadoopUtils.java | 245 +++-
.../service/storage/impl/OssOperator.java | 33 +-
.../service/storage/impl/S3Utils.java | 252 +++-
.../service/process/ProcessServiceTest.java | 21 +-
.../service/storage/impl/OssOperatorTest.java | 4 +-
.../dolphinscheduler/plugin/task/sql/SqlTask.java | 4 +-
dolphinscheduler-ui/src/locales/en_US/resource.ts | 5 +-
.../src/router/modules/resources.ts | 10 +-
.../src/service/modules/resources/index.ts | 64 +-
.../src/service/modules/resources/types.ts | 12 +-
dolphinscheduler-ui/src/utils/tree-format.ts | 2 +-
.../task/components/node/fields/use-resources.ts | 4 +-
.../projects/task/components/node/format-data.ts | 11 +-
.../views/projects/task/components/node/types.ts | 12 +-
.../workflow/definition/components/use-modal.ts | 1 -
.../src/views/resource/file/create/index.tsx | 4 +-
.../src/views/resource/file/create/use-create.ts | 7 +-
.../src/views/resource/file/edit/index.tsx | 8 +-
.../src/views/resource/file/edit/use-edit.ts | 15 +-
.../src/views/resource/file/index.tsx | 104 +-
.../src/views/resource/file/rename/index.tsx | 20 +-
.../src/views/resource/file/rename/use-form.ts | 11 +-
.../src/views/resource/file/rename/use-rename.ts | 8 +-
.../src/views/resource/file/table/table-action.tsx | 34 +-
.../src/views/resource/file/table/use-table.ts | 19 +-
.../src/views/resource/file/types.ts | 7 +-
.../src/views/resource/file/upload/use-upload.ts | 6 +-
.../src/views/resource/file/use-file.ts | 14 +-
.../udf/function/components/function-modal.tsx | 13 +-
.../resource/udf/function/components/use-form.ts | 14 +-
.../resource/udf/function/components/use-modal.ts | 9 +-
.../src/views/resource/udf/function/index.tsx | 1 +
.../src/views/resource/udf/function/types.ts | 1 +
.../src/views/resource/udf/function/use-table.ts | 9 +-
.../udf/resource/components/folder-modal.tsx | 8 +-
.../resource/udf/resource/components/use-modal.ts | 17 +-
.../src/views/resource/udf/resource/index.tsx | 6 +-
.../src/views/resource/udf/resource/types.ts | 2 +
.../src/views/resource/udf/resource/use-table.ts | 74 +-
.../worker/utils/TaskExecutionCheckerUtils.java | 12 +-
73 files changed, 2682 insertions(+), 1495 deletions(-)
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
index e2f3c6f33e..ff65bfd94b 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
@@ -55,6 +55,7 @@ import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.lang3.StringUtils;
+import java.io.IOException;
import java.util.Map;
import org.slf4j.Logger;
@@ -139,7 +140,6 @@ public class ResourcesController extends BaseController {
@Parameter(name = "name", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "description", description = "RESOURCE_DESC", schema = @Schema(implementation = String.class)),
@Parameter(name = "file", description = "RESOURCE_FILE", required = true, schema = @Schema(implementation = MultipartFile.class)),
- @Parameter(name = "pid", description = "RESOURCE_PID", required = true, schema = @Schema(implementation = int.class, example = "10")),
@Parameter(name = "currentDir", description = "RESOURCE_CURRENT_DIR", required = true, schema = @Schema(implementation = String.class))
})
@PostMapping()
@@ -150,10 +150,9 @@ public class ResourcesController extends BaseController {
@RequestParam(value = "name") String alias,
@RequestParam(value = "description", required = false) String description,
@RequestParam("file") MultipartFile file,
- @RequestParam(value = "pid") int pid,
@RequestParam(value = "currentDir") String currentDir) {
// todo verify the file name
- return resourceService.createResource(loginUser, alias, description, type, file, pid, currentDir);
+ return resourceService.createResource(loginUser, alias, description, type, file, currentDir);
}
/**
@@ -161,7 +160,6 @@ public class ResourcesController extends BaseController {
*
* @param loginUser login user
* @param alias alias
- * @param resourceId resource id
* @param type resource type
* @param description description
* @param file resource file
@@ -169,23 +167,24 @@ public class ResourcesController extends BaseController {
*/
@Operation(summary = "updateResource", description = "UPDATE_RESOURCE_NOTES")
@Parameters({
- @Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")),
+ @Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class)),
+ @Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
@Parameter(name = "name", description = "RESOURCE_NAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "description", description = "RESOURCE_DESC", schema = @Schema(implementation = String.class)),
@Parameter(name = "file", description = "RESOURCE_FILE", required = true, schema = @Schema(implementation = MultipartFile.class))
})
- @PutMapping(value = "/{id}")
+ @PutMapping()
@ApiException(UPDATE_RESOURCE_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result<Object> updateResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
- @PathVariable(value = "id") int resourceId,
+ @RequestParam(value = "fullName") String fullName,
+ @RequestParam(value = "tenantCode", required = false) String tenantCode,
@RequestParam(value = "type") ResourceType type,
@RequestParam(value = "name") String alias,
@RequestParam(value = "description", required = false) String description,
@RequestParam(value = "file", required = false) MultipartFile file) {
- // todo verify the resource name
- return resourceService.updateResource(loginUser, resourceId, alias, description, type, file);
+ return resourceService.updateResource(loginUser, fullName, tenantCode, alias, description, type, file);
}
/**
@@ -197,15 +196,17 @@ public class ResourcesController extends BaseController {
*/
@Operation(summary = "queryResourceList", description = "QUERY_RESOURCE_LIST_NOTES")
@Parameters({
- @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class))
+ @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
+ @Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class))
})
@GetMapping(value = "/list")
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result<Object> queryResourceList(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
- @RequestParam(value = "type") ResourceType type) {
- Map<String, Object> result = resourceService.queryResourceList(loginUser, type);
+ @RequestParam(value = "type") ResourceType type,
+ @RequestParam(value = "fullName") String fullName) {
+ Map<String, Object> result = resourceService.queryResourceList(loginUser, type, fullName);
return returnDataList(result);
}
@@ -222,7 +223,7 @@ public class ResourcesController extends BaseController {
@Operation(summary = "queryResourceListPaging", description = "QUERY_RESOURCE_LIST_PAGING_NOTES")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
- @Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "10")),
+ @Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "bucket_name/tenant_name/type/ds")),
@Parameter(name = "searchVal", description = "SEARCH_VAL", schema = @Schema(implementation = String.class)),
@Parameter(name = "pageNo", description = "PAGE_NO", required = true, schema = @Schema(implementation = int.class, example = "1")),
@Parameter(name = "pageSize", description = "PAGE_SIZE", required = true, schema = @Schema(implementation = int.class, example = "20"))
@@ -232,8 +233,9 @@ public class ResourcesController extends BaseController {
@ApiException(QUERY_RESOURCES_LIST_PAGING)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result<Object> queryResourceListPaging(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
+ @RequestParam(value = "fullName") String fullName,
+ @RequestParam(value = "tenantCode") String tenantCode,
@RequestParam(value = "type") ResourceType type,
- @RequestParam(value = "id") int id,
@RequestParam("pageNo") Integer pageNo,
@RequestParam(value = "searchVal", required = false) String searchVal,
@RequestParam("pageSize") Integer pageSize) {
@@ -243,7 +245,8 @@ public class ResourcesController extends BaseController {
}
searchVal = ParameterUtils.handleEscapes(searchVal);
- result = resourceService.queryResourceListPaging(loginUser, id, type, searchVal, pageNo, pageSize);
+ result = resourceService.queryResourceListPaging(loginUser, fullName, tenantCode, type, searchVal, pageNo,
+ pageSize);
return result;
}
@@ -251,20 +254,20 @@ public class ResourcesController extends BaseController {
* delete resource
*
* @param loginUser login user
- * @param resourceId resource id
* @return delete result code
*/
@Operation(summary = "deleteResource", description = "DELETE_RESOURCE_BY_ID_NOTES")
@Parameters({
- @Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100"))
+ @Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "test/"))
})
- @DeleteMapping(value = "/{id}")
+ @DeleteMapping()
@ResponseStatus(HttpStatus.OK)
@ApiException(DELETE_RESOURCE_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result<Object> deleteResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
- @PathVariable(value = "id") int resourceId) throws Exception {
- return resourceService.delete(loginUser, resourceId);
+ @RequestParam(value = "fullName") String fullName,
+ @RequestParam(value = "tenantCode", required = false) String tenantCode) throws Exception {
+ return resourceService.delete(loginUser, fullName, tenantCode);
}
/**
@@ -312,55 +315,56 @@ public class ResourcesController extends BaseController {
}
/**
- * query resource by full name and type
+ * query resource by file name and type
*
* @param loginUser login user
- * @param fullName resource full name
+ * @param fileName resource full name
+ * @param tenantCode tenantcode of the owner of the resource
* @param type resource type
- * @param id resource id
* @return true if the resource name not exists, otherwise return false
*/
- @Operation(summary = "queryResource", description = "QUERY_BY_RESOURCE_NAME")
+ @Operation(summary = "queryResourceByFileName", description = "QUERY_BY_RESOURCE_FILE_NAME")
@Parameters({
@Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
- @Parameter(name = "fullName", description = "RESOURCE_FULL_NAME", required = true, schema = @Schema(implementation = String.class)),
- @Parameter(name = "id", description = "RESOURCE_ID", required = false, schema = @Schema(implementation = int.class, example = "10"))
+ @Parameter(name = "fileName", description = "RESOURCE_FILE_NAME", required = true, schema = @Schema(implementation = String.class)),
+ @Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)),
})
- @GetMapping(value = "/{id}")
+ @GetMapping(value = "/query-file-name")
@ResponseStatus(HttpStatus.OK)
@ApiException(RESOURCE_NOT_EXIST)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
- public Result<Object> queryResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
- @RequestParam(value = "fullName", required = false) String fullName,
- @PathVariable(value = "id", required = false) Integer id,
- @RequestParam(value = "type") ResourceType type) {
+ public Result<Object> queryResourceByFileName(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
+ @RequestParam(value = "fileName", required = false) String fileName,
+ @RequestParam(value = "tenantCode", required = false) String tenantCode,
+ @RequestParam(value = "type") ResourceType type) {
- return resourceService.queryResource(loginUser, fullName, id, type);
+ return resourceService.queryResourceByFileName(loginUser, fileName, type, tenantCode);
}
/**
* view resource file online
*
* @param loginUser login user
- * @param resourceId resource id
* @param skipLineNum skip line number
* @param limit limit
* @return resource content
*/
@Operation(summary = "viewResource", description = "VIEW_RESOURCE_BY_ID_NOTES")
@Parameters({
- @Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")),
+ @Parameter(name = "fullName", description = "RESOURCE_FULL_NAME", required = true, schema = @Schema(implementation = String.class, example = "tenant/1.png")),
+ @Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "skipLineNum", description = "SKIP_LINE_NUM", required = true, schema = @Schema(implementation = int.class, example = "100")),
@Parameter(name = "limit", description = "LIMIT", required = true, schema = @Schema(implementation = int.class, example = "100"))
})
- @GetMapping(value = "/{id}/view")
+ @GetMapping(value = "/view")
@ApiException(VIEW_RESOURCE_FILE_ON_LINE_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result viewResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
- @PathVariable(value = "id") int resourceId,
@RequestParam(value = "skipLineNum") int skipLineNum,
- @RequestParam(value = "limit") int limit) {
- return resourceService.readResource(loginUser, resourceId, skipLineNum, limit);
+ @RequestParam(value = "limit") int limit,
+ @RequestParam(value = "fullName") String fullName,
+ @RequestParam(value = "tenantCode") String tenantCode) {
+ return resourceService.readResource(loginUser, fullName, tenantCode, skipLineNum, limit);
}
/**
@@ -375,7 +379,6 @@ public class ResourcesController extends BaseController {
@Parameter(name = "suffix", description = "SUFFIX", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "description", description = "RESOURCE_DESC", schema = @Schema(implementation = String.class)),
@Parameter(name = "content", description = "CONTENT", required = true, schema = @Schema(implementation = String.class)),
- @Parameter(name = "pid", description = "RESOURCE_PID", required = true, schema = @Schema(implementation = int.class, example = "10")),
@Parameter(name = "currentDir", description = "RESOURCE_CURRENTDIR", required = true, schema = @Schema(implementation = String.class))
})
@PostMapping(value = "/online-create")
@@ -387,13 +390,12 @@ public class ResourcesController extends BaseController {
@RequestParam(value = "suffix") String fileSuffix,
@RequestParam(value = "description", required = false) String description,
@RequestParam(value = "content") String content,
- @RequestParam(value = "pid") int pid,
@RequestParam(value = "currentDir") String currentDir) {
if (StringUtils.isEmpty(content)) {
logger.error("resource file contents are not allowed to be empty");
return error(RESOURCE_FILE_IS_EMPTY.getCode(), RESOURCE_FILE_IS_EMPTY.getMsg());
}
- return resourceService.onlineCreateResource(loginUser, type, fileName, fileSuffix, description, content, pid,
+ return resourceService.onlineCreateResource(loginUser, type, fileName, fileSuffix, description, content,
currentDir);
}
@@ -401,46 +403,46 @@ public class ResourcesController extends BaseController {
* edit resource file online
*
* @param loginUser login user
- * @param resourceId resource id
* @param content content
* @return update result code
*/
@Operation(summary = "updateResourceContent", description = "UPDATE_RESOURCE_NOTES")
@Parameters({
- @Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100")),
- @Parameter(name = "content", description = "CONTENT", required = true, schema = @Schema(implementation = String.class))
+ @Parameter(name = "content", description = "CONTENT", required = true, schema = @Schema(implementation = String.class)),
+ @Parameter(name = "fullName", description = "FULL_NAME", required = true, schema = @Schema(implementation = String.class)),
+ @Parameter(name = "tenantCode", description = "TENANT_CODE", required = true, schema = @Schema(implementation = String.class))
})
- @PutMapping(value = "/{id}/update-content")
+ @PutMapping(value = "/update-content")
@ApiException(EDIT_RESOURCE_FILE_ON_LINE_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result updateResourceContent(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
- @PathVariable(value = "id") int resourceId,
+ @RequestParam(value = "fullName") String fullName,
+ @RequestParam(value = "tenantCode") String tenantCode,
@RequestParam(value = "content") String content) {
if (StringUtils.isEmpty(content)) {
logger.error("The resource file contents are not allowed to be empty");
return error(RESOURCE_FILE_IS_EMPTY.getCode(), RESOURCE_FILE_IS_EMPTY.getMsg());
}
- return resourceService.updateResourceContent(loginUser, resourceId, content);
+ return resourceService.updateResourceContent(loginUser, fullName, tenantCode, content);
}
/**
* download resource file
*
* @param loginUser login user
- * @param resourceId resource id
* @return resource content
*/
@Operation(summary = "downloadResource", description = "DOWNLOAD_RESOURCE_NOTES")
@Parameters({
- @Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100"))
+ @Parameter(name = "fullName", description = "RESOURCE_FULLNAME", required = true, schema = @Schema(implementation = String.class, example = "test/"))
})
- @GetMapping(value = "/{id}/download")
+ @GetMapping(value = "/download")
@ResponseBody
@ApiException(DOWNLOAD_RESOURCE_FILE_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public ResponseEntity downloadResource(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
- @PathVariable(value = "id") int resourceId) throws Exception {
- Resource file = resourceService.downloadResource(loginUser, resourceId);
+ @RequestParam(value = "fullName") String fullName) throws Exception {
+ Resource file = resourceService.downloadResource(loginUser, fullName);
if (file == null) {
return ResponseEntity.status(HttpStatus.BAD_REQUEST).body(RESOURCE_NOT_EXIST.getMsg());
}
@@ -460,7 +462,6 @@ public class ResourcesController extends BaseController {
* @param database database
* @param description description
* @param className class name
- * @param resourceId resource id
* @return create result code
*/
@Operation(summary = "createUdfFunc", description = "CREATE_UDF_FUNCTION_NOTES")
@@ -474,7 +475,7 @@ public class ResourcesController extends BaseController {
@Parameter(name = "resourceId", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100"))
})
- @PostMapping(value = "/{resourceId}/udf-func")
+ @PostMapping(value = "/udf-func")
@ResponseStatus(HttpStatus.CREATED)
@ApiException(CREATE_UDF_FUNCTION_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
@@ -482,20 +483,20 @@ public class ResourcesController extends BaseController {
@RequestParam(value = "type") UdfType type,
@RequestParam(value = "funcName") String funcName,
@RequestParam(value = "className") String className,
+ @RequestParam(value = "fullName") String fullName,
@RequestParam(value = "argTypes", required = false) String argTypes,
@RequestParam(value = "database", required = false) String database,
- @RequestParam(value = "description", required = false) String description,
- @PathVariable(value = "resourceId") int resourceId) {
+ @RequestParam(value = "description", required = false) String description) {
// todo verify the sourceName
- return udfFuncService.createUdfFunction(loginUser, funcName, className, argTypes, database, description, type,
- resourceId);
+ return udfFuncService.createUdfFunction(loginUser, funcName, className, fullName,
+ argTypes, database, description, type);
}
/**
* view udf function
*
* @param loginUser login user
- * @param id resource id
+ * @param id udf function id
* @return udf function detail
*/
@Operation(summary = "viewUIUdfFunction", description = "VIEW_UDF_FUNCTION_NOTES")
@@ -521,7 +522,6 @@ public class ResourcesController extends BaseController {
* @param argTypes argument types
* @param database data base
* @param description description
- * @param resourceId resource id
* @param className class name
* @param udfFuncId udf function id
* @return update result code
@@ -534,11 +534,9 @@ public class ResourcesController extends BaseController {
@Parameter(name = "className", description = "CLASS_NAME", required = true, schema = @Schema(implementation = String.class)),
@Parameter(name = "argTypes", description = "ARG_TYPES", schema = @Schema(implementation = String.class)),
@Parameter(name = "database", description = "DATABASE_NAME", schema = @Schema(implementation = String.class)),
- @Parameter(name = "description", description = "UDF_DESC", schema = @Schema(implementation = String.class)),
- @Parameter(name = "resourceId", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "100"))
-
+ @Parameter(name = "description", description = "UDF_DESC", schema = @Schema(implementation = String.class))
})
- @PutMapping(value = "/{resourceId}/udf-func/{id}")
+ @PutMapping(value = "/udf-func/{id}")
@ApiException(UPDATE_UDF_FUNCTION_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
public Result updateUdfFunc(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
@@ -549,9 +547,9 @@ public class ResourcesController extends BaseController {
@RequestParam(value = "argTypes", required = false) String argTypes,
@RequestParam(value = "database", required = false) String database,
@RequestParam(value = "description", required = false) String description,
- @PathVariable(value = "resourceId") int resourceId) {
- return udfFuncService.updateUdfFunc(loginUser, udfFuncId, funcName, className, argTypes, database, description,
- type, resourceId);
+ @RequestParam(value = "fullName") String fullName) {
+ return udfFuncService.updateUdfFunc(loginUser, udfFuncId, funcName, className,
+ argTypes, database, description, type, fullName);
}
/**
@@ -731,23 +729,26 @@ public class ResourcesController extends BaseController {
}
/**
- * query resource by resource id
+ * query a resource by resource full name
*
* @param loginUser login user
- * @param id resource id
+ * @param fullName resource full name
* @return resource
*/
- @Operation(summary = "queryResourceById", description = "QUERY_BY_RESOURCE_NAME")
+ @Operation(summary = "queryResourceByFullName", description = "QUERY_BY_RESOURCE_FULL_NAME")
@Parameters({
- @Parameter(name = "id", description = "RESOURCE_ID", required = true, schema = @Schema(implementation = int.class, example = "10"))
+ @Parameter(name = "type", description = "RESOURCE_TYPE", required = true, schema = @Schema(implementation = ResourceType.class)),
+ @Parameter(name = "fullName", description = "RESOURCE_FULL_NAME", required = true, schema = @Schema(implementation = String.class)),
})
- @GetMapping(value = "/{id}/query")
+ @GetMapping(value = "/query-full-name")
@ResponseStatus(HttpStatus.OK)
@ApiException(RESOURCE_NOT_EXIST)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
- public Result queryResourceById(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
- @PathVariable(value = "id", required = true) Integer id) {
+ public Result queryResourceByFullName(@Parameter(hidden = true) @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
+ @RequestParam(value = "type") ResourceType type,
+ @RequestParam(value = "fullName") String fullName,
+ @RequestParam(value = "tenantCode") String tenantCode) throws IOException {
- return resourceService.queryResourceById(loginUser, id);
+ return resourceService.queryResourceByFullName(loginUser, fullName, tenantCode, type);
}
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java
index e520212adf..708f9efc0a 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/ResourceComponent.java
@@ -35,7 +35,7 @@ import com.fasterxml.jackson.annotation.JsonPropertyOrder;
@JsonPropertyOrder({"id", "pid", "name", "fullName", "description", "isDirctory", "children", "type"})
public abstract class ResourceComponent {
- public ResourceComponent(int id, int pid, String name, String fullName, String description, boolean isDirctory) {
+ public ResourceComponent(int id, String pid, String name, String fullName, String description, boolean isDirctory) {
this.id = id;
this.pid = pid;
this.name = name;
@@ -53,7 +53,7 @@ public abstract class ResourceComponent {
/**
* parent id
*/
- protected int pid;
+ protected String pid;
/**
* name
*/
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java
index 5b84fde1a5..1f404c31e1 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java
@@ -19,10 +19,11 @@ package org.apache.dolphinscheduler.api.dto.resources.visitor;
import org.apache.dolphinscheduler.api.dto.resources.Directory;
import org.apache.dolphinscheduler.api.dto.resources.FileLeaf;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
-import org.apache.dolphinscheduler.dao.entity.Resource;
+import org.apache.dolphinscheduler.service.storage.StorageEntity;
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
/**
* resource tree visitor
@@ -32,7 +33,7 @@ public class ResourceTreeVisitor implements Visitor {
/**
* resource list
*/
- private List<Resource> resourceList;
+ private List<StorageEntity> resourceList;
public ResourceTreeVisitor() {
}
@@ -41,7 +42,7 @@ public class ResourceTreeVisitor implements Visitor {
* constructor
* @param resourceList resource list
*/
- public ResourceTreeVisitor(List<Resource> resourceList) {
+ public ResourceTreeVisitor(List<StorageEntity> resourceList) {
this.resourceList = resourceList;
}
@@ -50,14 +51,15 @@ public class ResourceTreeVisitor implements Visitor {
* @return resoruce component
*/
@Override
- public ResourceComponent visit() {
+ public ResourceComponent visit(String rootPath) {
ResourceComponent rootDirectory = new Directory();
- for (Resource resource : resourceList) {
+ for (StorageEntity resource : resourceList) {
// judge whether is root node
- if (rootNode(resource)) {
+ if (rootNode(resource, rootPath)) {
+ // if it is a root node.
ResourceComponent tempResourceComponent = getResourceComponent(resource);
rootDirectory.add(tempResourceComponent);
- tempResourceComponent.setChildren(setChildren(tempResourceComponent.getId(), resourceList));
+ tempResourceComponent.setChildren(setChildren(tempResourceComponent.getFullName(), resourceList));
}
}
return rootDirectory;
@@ -65,20 +67,21 @@ public class ResourceTreeVisitor implements Visitor {
/**
* set children
- * @param id id
+ * @param fullName unique path
* @param list resource list
* @return resource component list
*/
- public static List<ResourceComponent> setChildren(int id, List<Resource> list) {
+ public static List<ResourceComponent> setChildren(String fullName, List<StorageEntity> list) {
+ // id is the unique value,
List<ResourceComponent> childList = new ArrayList<>();
- for (Resource resource : list) {
- if (id == resource.getPid()) {
+ for (StorageEntity resource : list) {
+ if (Objects.equals(fullName, resource.getPfullName())) {
ResourceComponent tempResourceComponent = getResourceComponent(resource);
childList.add(tempResourceComponent);
}
}
for (ResourceComponent resourceComponent : childList) {
- resourceComponent.setChildren(setChildren(resourceComponent.getId(), list));
+ resourceComponent.setChildren(setChildren(resourceComponent.getFullName(), list));
}
if (childList.size() == 0) {
return new ArrayList<>();
@@ -91,17 +94,18 @@ public class ResourceTreeVisitor implements Visitor {
* @param resource resource
* @return true if it is the root node
*/
- public boolean rootNode(Resource resource) {
+ public boolean rootNode(StorageEntity resource, String rootPath) {
boolean isRootNode = true;
- if (resource.getPid() != -1) {
- for (Resource parent : resourceList) {
- if (resource.getPid() == parent.getId()) {
+ if (!Objects.equals(resource.getPfullName(), rootPath)) {
+ for (StorageEntity parent : resourceList) {
+ if (Objects.equals(resource.getPfullName(), parent.getFullName())) {
isRootNode = false;
break;
}
}
}
+
return isRootNode;
}
@@ -110,7 +114,7 @@ public class ResourceTreeVisitor implements Visitor {
* @param resource resource
* @return resource component
*/
- private static ResourceComponent getResourceComponent(Resource resource) {
+ private static ResourceComponent getResourceComponent(StorageEntity resource) {
ResourceComponent tempResourceComponent;
if (resource.isDirectory()) {
tempResourceComponent = new Directory();
@@ -119,9 +123,10 @@ public class ResourceTreeVisitor implements Visitor {
}
tempResourceComponent.setName(resource.getAlias());
- tempResourceComponent.setFullName(resource.getFullName().replaceFirst("/", ""));
+ // tempResourceComponent.setFullName(resource.getFullName().replaceFirst("/",""));
+ tempResourceComponent.setFullName(resource.getFullName());
tempResourceComponent.setId(resource.getId());
- tempResourceComponent.setPid(resource.getPid());
+ tempResourceComponent.setPid(resource.getPfullName());
tempResourceComponent.setIdValue(resource.getId(), resource.isDirectory());
tempResourceComponent.setDescription(resource.getDescription());
tempResourceComponent.setType(resource.getType());
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java
index 382bcc5e16..1792b2769c 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/Visitor.java
@@ -28,5 +28,5 @@ public interface Visitor {
* visit
* @return resource component
*/
- ResourceComponent visit();
+ ResourceComponent visit(String rootPath);
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java
index b8016c5926..9f5fde0501 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/permission/ResourcePermissionCheckService.java
@@ -38,7 +38,6 @@ public interface ResourcePermissionCheckService<T> {
* @param authorizationType
* @param userId
* @param logger
- * @param <T>
* @return
*/
Set<T> userOwnedResourceIdsAcquisition(Object authorizationType, Integer userId, Logger logger);
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
index 4de830c074..04f477e36b 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
@@ -57,9 +57,8 @@ public interface ResourcesService {
* @param loginUser login user
* @param name alias
* @param desc description
- * @param file file
* @param type type
- * @param pid parent id
+ * @param file file
* @param currentDir current directory
* @return create result code
*/
@@ -68,13 +67,11 @@ public interface ResourcesService {
String desc,
ResourceType type,
MultipartFile file,
- int pid,
String currentDir);
/**
* update resource
* @param loginUser login user
- * @param resourceId resource id
* @param name name
* @param desc description
* @param type resource type
@@ -82,7 +79,8 @@ public interface ResourcesService {
* @return update result code
*/
Result<Object> updateResource(User loginUser,
- int resourceId,
+ String fullName,
+ String tenantCode,
String name,
String desc,
ResourceType type,
@@ -98,8 +96,8 @@ public interface ResourcesService {
* @param pageSize page size
* @return resource list page
*/
- Result queryResourceListPaging(User loginUser, int directoryId, ResourceType type, String searchVal, Integer pageNo,
- Integer pageSize);
+ Result queryResourceListPaging(User loginUser, String fullName, String resTenantCode,
+ ResourceType type, String searchVal, Integer pageNo, Integer pageSize);
/**
* query resource list
@@ -108,7 +106,7 @@ public interface ResourcesService {
* @param type resource type
* @return resource list
*/
- Map<String, Object> queryResourceList(User loginUser, ResourceType type);
+ Map<String, Object> queryResourceList(User loginUser, ResourceType type, String fullName);
/**
* query resource list by program type
@@ -123,11 +121,10 @@ public interface ResourcesService {
* delete resource
*
* @param loginUser login user
- * @param resourceId resource id
* @return delete result code
* @throws IOException exception
*/
- Result<Object> delete(User loginUser, int resourceId) throws IOException;
+ Result<Object> delete(User loginUser, String fullName, String tenantCode) throws IOException;
/**
* verify resource by name and type
@@ -139,23 +136,22 @@ public interface ResourcesService {
Result<Object> verifyResourceName(String fullName, ResourceType type, User loginUser);
/**
- * verify resource by full name or pid and type
- * @param fullName resource full name
- * @param id resource id
+ * verify resource by file name
+ * @param fileName resource file name
* @param type resource type
- * @return true if the resource full name or pid not exists, otherwise return false
+ * @return true if the resource file name, otherwise return false
*/
- Result<Object> queryResource(User loginUser, String fullName, Integer id, ResourceType type);
+ Result<Object> queryResourceByFileName(User loginUser, String fileName, ResourceType type, String resTenantCode);
/**
* view resource file online
*
- * @param resourceId resource id
* @param skipLineNum skip line number
* @param limit limit
+ * @param fullName fullName
* @return resource content
*/
- Result<Object> readResource(User loginUser, int resourceId, int skipLineNum, int limit);
+ Result<Object> readResource(User loginUser, String fullName, String tenantCode, int skipLineNum, int limit);
/**
* create resource file online
@@ -169,7 +165,7 @@ public interface ResourcesService {
* @return create result code
*/
Result<Object> onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix,
- String desc, String content, int pid, String currentDirectory);
+ String desc, String content, String currentDirectory);
/**
* create or update resource.
@@ -203,16 +199,16 @@ public interface ResourcesService {
* @param content content
* @return update result cod
*/
- Result<Object> updateResourceContent(User loginUser, int resourceId, String content);
+ Result<Object> updateResourceContent(User loginUser, String fullName, String tenantCode,
+ String content);
/**
* download file
*
- * @param resourceId resource id
* @return resource content
* @throws IOException exception
*/
- org.springframework.core.io.Resource downloadResource(User loginUser, int resourceId) throws IOException;
+ org.springframework.core.io.Resource downloadResource(User loginUser, String fullName) throws IOException;
/**
* list all file
@@ -270,9 +266,11 @@ public interface ResourcesService {
/**
* get resource by id
- * @param resourceId resource id
+ * @param fullName resource full name
+ * @param tenantCode owner's tenant code of resource
* @return resource
*/
- Result<Object> queryResourceById(User loginUser, Integer resourceId);
+ Result<Object> queryResourceByFullName(User loginUser, String fullName, String tenantCode,
+ ResourceType type) throws IOException;
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java
index 0cac41663f..35be4066d8 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/UdfFuncService.java
@@ -35,18 +35,17 @@ public interface UdfFuncService {
* @param argTypes argument types
* @param database database
* @param desc description
- * @param resourceId resource id
* @param className class name
* @return create result code
*/
Result<Object> createUdfFunction(User loginUser,
String funcName,
String className,
+ String fullName,
String argTypes,
String database,
String desc,
- UdfType type,
- int resourceId);
+ UdfType type);
/**
* query udf function
@@ -66,6 +65,7 @@ public interface UdfFuncService {
* @param database data base
* @param desc description
* @param resourceId resource id
+ * @param fullName resource full name
* @param className class name
* @return update result code
*/
@@ -77,7 +77,7 @@ public interface UdfFuncService {
String database,
String desc,
UdfType type,
- int resourceId);
+ String fullName);
/**
* query udf function list paging
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
index 85ac4734b8..d82a0e4b7d 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ResourcesServiceImpl.java
@@ -26,13 +26,13 @@ import static org.apache.dolphinscheduler.common.constants.Constants.FORMAT_S_S;
import static org.apache.dolphinscheduler.common.constants.Constants.JAR;
import static org.apache.dolphinscheduler.common.constants.Constants.PERIOD;
-import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
import org.apache.dolphinscheduler.api.dto.resources.filter.ResourceFilter;
import org.apache.dolphinscheduler.api.dto.resources.visitor.ResourceTreeVisitor;
import org.apache.dolphinscheduler.api.dto.resources.visitor.Visitor;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.exceptions.ServiceException;
+import org.apache.dolphinscheduler.api.service.ProcessDefinitionService;
import org.apache.dolphinscheduler.api.service.ResourcesService;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.RegexUtils;
@@ -44,18 +44,26 @@ import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
+import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
+import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelation;
import org.apache.dolphinscheduler.dao.entity.Resource;
-import org.apache.dolphinscheduler.dao.entity.ResourcesUser;
+import org.apache.dolphinscheduler.dao.entity.ResourcesTask;
+import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
-import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
+import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionLogMapper;
+import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
+import org.apache.dolphinscheduler.dao.mapper.ResourceTaskMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
+import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
-import org.apache.dolphinscheduler.dao.utils.ResourceProcessDefinitionUtils;
+import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
+import org.apache.dolphinscheduler.service.process.ProcessService;
+import org.apache.dolphinscheduler.service.storage.StorageEntity;
import org.apache.dolphinscheduler.service.storage.StorageOperate;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
@@ -77,19 +85,16 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
-import java.util.regex.Matcher;
import java.util.stream.Collectors;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.dao.DuplicateKeyException;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
-import com.baomidou.mybatisplus.core.metadata.IPage;
-import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.google.common.base.Joiner;
import com.google.common.io.Files;
@@ -105,9 +110,21 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
@Autowired
private ResourceMapper resourcesMapper;
+ @Autowired
+ private ResourceTaskMapper resourceTaskMapper;
+
+ @Autowired
+ private TaskDefinitionMapper taskDefinitionMapper;
+
@Autowired
private UdfFuncMapper udfFunctionMapper;
+ @Autowired
+ private ProcessService processService;
+
+ @Autowired
+ private ProcessDefinitionService processDefinitionService;
+
@Autowired
private TenantMapper tenantMapper;
@@ -118,7 +135,10 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
private ResourceUserMapper resourceUserMapper;
@Autowired
- private ProcessDefinitionMapper processDefinitionMapper;
+ private ProcessDefinitionLogMapper processDefinitionLogMapper;
+
+ @Autowired
+ private ProcessTaskRelationMapper processTaskRelationMapper;
@Autowired(required = false)
private StorageOperate storageOperate;
@@ -143,14 +163,6 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
int pid,
String currentDir) {
Result<Object> result = new Result<>();
- String funcPermissionKey = type.equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FOLDER_ONLINE_CREATE
- : ApiFuncIdentificationConstant.UDF_FOLDER_ONLINE_CREATE;
- boolean canOperatorPermissions =
- canOperatorPermissions(loginUser, null, AuthorizationType.RESOURCE_FILE_ID, funcPermissionKey);
- if (!canOperatorPermissions) {
- putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
- return result;
- }
result = checkResourceUploadStartupState();
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
@@ -162,48 +174,49 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
return result;
}
- if (checkDescriptionLength(description)) {
- logger.warn("Parameter description is too long.");
- putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
+ User user = userMapper.selectById(loginUser.getId());
+ if (user == null) {
+ logger.error("user {} not exists", loginUser.getId());
+ putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
return result;
}
- String fullName = getFullName(currentDir, name);
- result = verifyResource(loginUser, type, fullName, pid);
- if (!result.getCode().equals(Status.SUCCESS.getCode())) {
+ Tenant tenant = tenantMapper.queryById(user.getTenantId());
+ if (tenant == null) {
+ logger.error("tenant not exists");
+ putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST);
return result;
}
- if (checkResourceExists(fullName, type.ordinal())) {
- logger.warn("Resource directory exists, can not create again, fullName:{}.", fullName);
- putMsg(result, Status.RESOURCE_EXIST);
+ String tenantCode = tenant.getTenantCode();
+
+ if (!isUserTenantValid(isAdmin(loginUser), tenantCode, "")) {
+ logger.error("current user does not have permission");
+ putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
return result;
}
- Date now = new Date();
+ if (checkDescriptionLength(description)) {
+ logger.warn("Parameter description is too long.");
+ putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
+ return result;
+ }
- Resource resource =
- new Resource(pid, name, fullName, true, description, name, loginUser.getId(), type, 0, now, now);
+ String userResRootPath = ResourceType.UDF.equals(type) ? storageOperate.getUdfDir(tenantCode)
+ : storageOperate.getResDir(tenantCode);
+ String fullName = !currentDir.contains(userResRootPath) ? userResRootPath + name : currentDir + name;
try {
- resourcesMapper.insert(resource);
- putMsg(result, Status.SUCCESS);
- permissionPostHandle(resource.getType(), loginUser, resource.getId());
- Map<String, Object> resultMap = new HashMap<>();
- for (Map.Entry<Object, Object> entry : new BeanMap(resource).entrySet()) {
- if (!"class".equalsIgnoreCase(entry.getKey().toString())) {
- resultMap.put(entry.getKey().toString(), entry.getValue());
- }
+ if (checkResourceExists(fullName)) {
+ logger.error("resource directory {} has exist, can't recreate", fullName);
+ putMsg(result, Status.RESOURCE_EXIST);
+ return result;
}
- result.setData(resultMap);
- } catch (DuplicateKeyException e) {
- logger.warn("Resource directory exists, can not create again, fullName:{}.", fullName);
- putMsg(result, Status.RESOURCE_EXIST);
- return result;
} catch (Exception e) {
logger.warn("Resource exists, can not create again, fullName:{}.", fullName, e);
throw new ServiceException("resource already exists, can't recreate");
}
+
// create directory in storage
createDirectory(loginUser, fullName, type, result);
return result;
@@ -220,9 +233,8 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
* @param loginUser login user
* @param name alias
* @param desc description
- * @param file file
* @param type type
- * @param pid parent id
+ * @param file file
* @param currentDir current directory
* @return create result code
*/
@@ -233,36 +245,33 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
String desc,
ResourceType type,
MultipartFile file,
- int pid,
String currentDir) {
Result<Object> result = new Result<>();
- String funcPermissionKey = type.equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_UPLOAD
- : ApiFuncIdentificationConstant.UDF_UPLOAD;
- boolean canOperatorPermissions =
- canOperatorPermissions(loginUser, null, AuthorizationType.RESOURCE_FILE_ID, funcPermissionKey);
- if (!canOperatorPermissions) {
- putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
- return result;
- }
+
result = checkResourceUploadStartupState();
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
return result;
}
- result = verifyPid(loginUser, pid);
- if (!result.getCode().equals(Status.SUCCESS.getCode())) {
+ User user = userMapper.selectById(loginUser.getId());
+ if (user == null) {
+ logger.error("user {} not exists", loginUser.getId());
+ putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
return result;
}
- if (checkDescriptionLength(desc)) {
- logger.warn("Parameter description is too long.");
- putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
+
+ Tenant tenant = tenantMapper.queryById(user.getTenantId());
+ if (tenant == null) {
+ logger.error("tenant not exists");
+ putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST);
return result;
}
- // make sure login user has tenant
- String tenantCode = getTenantCode(loginUser.getId(), result);
- if (StringUtils.isEmpty(tenantCode)) {
- logger.error("Tenant of current login user does not specified, loginUserName:{}.", loginUser.getUserName());
+ String tenantCode = tenant.getTenantCode();
+
+ if (!isUserTenantValid(isAdmin(loginUser), tenantCode, "")) {
+ logger.error("current user does not have permission");
+ putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
return result;
}
@@ -272,45 +281,32 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
}
// check resource name exists
- String fullName = getFullName(currentDir, name);
- if (checkResourceExists(fullName, type.ordinal())) {
- logger.warn("Resource exists, can not create again, fullName:{}.", RegexUtils.escapeNRT(name));
- putMsg(result, Status.RESOURCE_EXIST);
- return result;
- }
- if (fullName.length() > Constants.RESOURCE_FULL_NAME_MAX_LENGTH) {
- logger.warn(
- "Resource file's name is longer than max full name length, fullName:{}, fullNameSize:{}, maxFullNameSize:{}",
- RegexUtils.escapeNRT(name), fullName.length(), Constants.RESOURCE_FULL_NAME_MAX_LENGTH);
- putMsg(result, Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR);
- return result;
- }
-
- Date now = new Date();
- Resource resource = new Resource(pid, name, fullName, false, desc, file.getOriginalFilename(),
- loginUser.getId(), type, file.getSize(), now, now);
+ String userResRootPath = ResourceType.UDF.equals(type) ? storageOperate.getUdfDir(tenantCode)
+ : storageOperate.getResDir(tenantCode);
+ String currDirNFileName = !currentDir.contains(userResRootPath) ? userResRootPath + name : currentDir + name;
try {
- resourcesMapper.insert(resource);
- updateParentResourceSize(resource, resource.getSize());
- putMsg(result, Status.SUCCESS);
- permissionPostHandle(resource.getType(), loginUser, resource.getId());
- Map<String, Object> resultMap = new HashMap<>();
- for (Map.Entry<Object, Object> entry : new BeanMap(resource).entrySet()) {
- if (!"class".equalsIgnoreCase(entry.getKey().toString())) {
- resultMap.put(entry.getKey().toString(), entry.getValue());
- }
+ if (checkResourceExists(currDirNFileName)) {
+ logger.error("resource {} has exist, can't recreate", RegexUtils.escapeNRT(name));
+ putMsg(result, Status.RESOURCE_EXIST);
+ return result;
}
- result.setData(resultMap);
} catch (Exception e) {
- logger.warn("Resource exists, can not create again, fullName:{}.", fullName, e);
throw new ServiceException("resource already exists, can't recreate");
}
+ if (currDirNFileName.length() > Constants.RESOURCE_FULL_NAME_MAX_LENGTH) {
+ logger.error(
+ "Resource file's name is longer than max full name length, fullName:{}, " +
+ "fullNameSize:{}, maxFullNameSize:{}",
+ RegexUtils.escapeNRT(name), currDirNFileName.length(), Constants.RESOURCE_FULL_NAME_MAX_LENGTH);
+ putMsg(result, Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR);
+ return result;
+ }
// fail upload
- if (!upload(loginUser, fullName, file, type)) {
- logger.error("Upload resource file failed, resourceName:{}, fileName:{}.",
- RegexUtils.escapeNRT(name), RegexUtils.escapeNRT(file.getOriginalFilename()));
+ if (!upload(loginUser, currDirNFileName, file, type)) {
+ logger.error("upload resource: {} file: {} failed.", RegexUtils.escapeNRT(name),
+ RegexUtils.escapeNRT(file.getOriginalFilename()));
putMsg(result, Status.STORE_OPERATE_CREATE_ERROR);
throw new ServiceException(
String.format("upload resource: %s file: %s failed.", name, file.getOriginalFilename()));
@@ -354,11 +350,15 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
* check resource is exists
*
* @param fullName fullName
- * @param type type
* @return true if resource exists
*/
- private boolean checkResourceExists(String fullName, int type) {
- Boolean existResource = resourcesMapper.existResource(fullName, type);
+ private boolean checkResourceExists(String fullName) {
+ Boolean existResource = false;
+ try {
+ existResource = storageOperate.exists(fullName);
+ } catch (IOException e) {
+ logger.error("error occurred when checking resource: " + fullName, e);
+ }
return Boolean.TRUE.equals(existResource);
}
@@ -366,7 +366,9 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
* update resource
*
* @param loginUser login user
- * @param resourceId resource id
+ * @param resourceFullName resource full name
+ * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource,
+ * can be different from the login user in the case of logging in as admin users.
* @param name name
* @param desc description
* @param type resource type
@@ -376,37 +378,52 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
@Override
@Transactional
public Result<Object> updateResource(User loginUser,
- int resourceId,
+ String resourceFullName,
+ String resTenantCode,
String name,
String desc,
ResourceType type,
MultipartFile file) {
Result<Object> result = new Result<>();
- String funcPermissionKey = type.equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_UPDATE
- : ApiFuncIdentificationConstant.UDF_UPDATE;
- boolean canOperatorPermissions =
- canOperatorPermissions(loginUser, new Object[]{resourceId}, checkResourceType(type), funcPermissionKey);
- if (!canOperatorPermissions) {
- putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
- return result;
- }
+
result = checkResourceUploadStartupState();
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
return result;
}
- Resource resource = resourcesMapper.selectById(resourceId);
- if (resource == null) {
- logger.error("Resource does not exist, resourceId:{}.", resourceId);
- putMsg(result, Status.RESOURCE_NOT_EXIST);
+ User user = userMapper.selectById(loginUser.getId());
+ if (user == null) {
+ logger.error("user {} not exists", loginUser.getId());
+ putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
return result;
}
- if (checkDescriptionLength(desc)) {
- logger.warn("Parameter description is too long.");
- putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
+
+ Tenant tenant = tenantMapper.queryById(user.getTenantId());
+ if (tenant == null) {
+ logger.error("tenant not exists");
+ putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST);
return result;
}
+ String tenantCode = tenant.getTenantCode();
+
+ if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) {
+ logger.error("current user does not have permission");
+ putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
+ return result;
+ }
+
+ String defaultPath = storageOperate.getResDir(tenantCode);
+
+ StorageEntity resource;
+ try {
+ resource = storageOperate.getFileStatus(resourceFullName, defaultPath, resTenantCode, type);
+ } catch (Exception e) {
+ logger.error("Get file status fail, resource path: {}", resourceFullName, e);
+ putMsg(result, Status.RESOURCE_NOT_EXIST);
+ throw new ServiceException((String.format("Get file status fail, resource path: %s", resourceFullName)));
+ }
+
if (!PropertyUtils.getResUploadStartupState()) {
logger.error("Storage does not start up, resource upload startup state: {}.",
PropertyUtils.getResUploadStartupState());
@@ -423,21 +440,33 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
}
if (file == null && name.equals(resource.getAlias()) && desc.equals(resource.getDescription())) {
- logger.info("Resource does not need to be updated due to no change, resourceId:{}.", resourceId);
+ logger.info("Resource does not need to be updated due to no change, resource full name:{}.",
+ resourceFullName);
putMsg(result, Status.SUCCESS);
return result;
}
- // check resource already exists
+ // check if updated name of the resource already exists
String originFullName = resource.getFullName();
String originResourceName = resource.getAlias();
+ // the format of hdfs folders in the implementation has a "/" at the very end, we need to remove it.
+ originFullName = originFullName.endsWith("/") ? StringUtils.chop(originFullName) : originFullName;
+ name = name.endsWith("/") ? StringUtils.chop(name) : name;
+ // updated fullName
String fullName = String.format(FORMAT_SS,
originFullName.substring(0, originFullName.lastIndexOf(FOLDER_SEPARATOR) + 1), name);
- if (!originResourceName.equals(name) && checkResourceExists(fullName, type.ordinal())) {
- logger.warn("Resource exists, can not create again, fullName:{}.", RegexUtils.escapeNRT(name));
- putMsg(result, Status.RESOURCE_EXIST);
- return result;
+ if (!originResourceName.equals(name)) {
+ try {
+ if (checkResourceExists(fullName)) {
+ logger.error("resource {} already exists, can't recreate", fullName);
+ putMsg(result, Status.RESOURCE_EXIST);
+ return result;
+ }
+ } catch (Exception e) {
+ throw new ServiceException(String.format("error occurs while querying resource: %s", fullName));
+ }
+
}
result = verifyFile(name, type, file);
@@ -445,60 +474,6 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
return result;
}
- // query tenant by user id
- String tenantCode = getTenantCode(resource.getUserId(), result);
- if (StringUtils.isEmpty(tenantCode)) {
- logger.error("Tenant of current login user does not specified, loginUserName:{}.", loginUser.getUserName());
- return result;
- }
- // verify whether the resource exists in storage
- // get the path of origin file in storage
- String originFileName = storageOperate.getFileName(resource.getType(), tenantCode, originFullName);
- try {
- if (!storageOperate.exists(tenantCode, originFileName)) {
- logger.error(
- "Resource file does not exist in {} storage, tenantCode:{}, resourceId:{}, originFileName:{}.",
- resource.getType(), tenantCode, resourceId, originFileName);
- putMsg(result, Status.RESOURCE_NOT_EXIST);
- return result;
- }
- } catch (IOException e) {
- logger.error("Storage operation error, storageType:{}, tenantCode:{}, resourceId:{}, originFileName:{}.",
- resource.getType(), tenantCode, resourceId, originFileName, e);
- throw new ServiceException(Status.HDFS_OPERATION_ERROR);
- }
-
- if (!resource.isDirectory()) {
- // get the origin file suffix
- String originSuffix = Files.getFileExtension(originFullName);
- String suffix = Files.getFileExtension(fullName);
- boolean suffixIsChanged = false;
- if (StringUtils.isBlank(suffix) && StringUtils.isNotBlank(originSuffix)) {
- suffixIsChanged = true;
- }
- if (StringUtils.isNotBlank(suffix) && !suffix.equals(originSuffix)) {
- suffixIsChanged = true;
- }
- // verify whether suffix is changed
- if (suffixIsChanged) {
- // need verify whether this resource is authorized to other users
- Map<String, Object> columnMap = new HashMap<>();
- columnMap.put("resources_id", resourceId);
-
- List<ResourcesUser> resourcesUsers = resourceUserMapper.selectByMap(columnMap);
- if (CollectionUtils.isNotEmpty(resourcesUsers)) {
- List<Integer> userIds =
- resourcesUsers.stream().map(ResourcesUser::getUserId).collect(Collectors.toList());
- List<User> users = userMapper.selectBatchIds(userIds);
- String userNames = users.stream().map(User::getUserName).collect(Collectors.toList()).toString();
- logger.error("resource is authorized to user {},suffix not allowed to be modified", userNames);
- putMsg(result, Status.RESOURCE_IS_AUTHORIZED, userNames);
- return result;
- }
- }
- }
-
- // updateResource data
Date now = new Date();
long originFileSize = resource.getSize();
@@ -511,97 +486,123 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
resource.setSize(file.getSize());
}
- try {
- resourcesMapper.updateById(resource);
- if (resource.isDirectory()) {
- List<Integer> childrenResource = listAllChildren(resource, false);
- if (CollectionUtils.isNotEmpty(childrenResource)) {
- String matcherFullName = Matcher.quoteReplacement(fullName);
- List<Resource> childResourceList;
- Integer[] childResIdArray = childrenResource.toArray(new Integer[childrenResource.size()]);
- List<Resource> resourceList = resourcesMapper.listResourceByIds(childResIdArray);
- childResourceList = resourceList.stream().map(t -> {
- t.setFullName(t.getFullName().replaceFirst(originFullName, matcherFullName));
- t.setUpdateTime(now);
- return t;
- }).collect(Collectors.toList());
- resourcesMapper.batchUpdateResource(childResourceList);
-
- if (ResourceType.UDF.equals(resource.getType())) {
- List<UdfFunc> udfFuncs = udfFunctionMapper.listUdfByResourceId(childResIdArray);
- if (CollectionUtils.isNotEmpty(udfFuncs)) {
- udfFuncs = udfFuncs.stream().map(t -> {
- t.setResourceName(t.getResourceName().replaceFirst(originFullName, matcherFullName));
- t.setUpdateTime(now);
- return t;
- }).collect(Collectors.toList());
- udfFunctionMapper.batchUpdateUdfFunc(udfFuncs);
- }
- }
- }
- } else if (ResourceType.UDF.equals(resource.getType())) {
- List<UdfFunc> udfFuncs = udfFunctionMapper.listUdfByResourceId(new Integer[]{resourceId});
- if (CollectionUtils.isNotEmpty(udfFuncs)) {
- udfFuncs = udfFuncs.stream().map(t -> {
- t.setResourceName(fullName);
- t.setUpdateTime(now);
- return t;
- }).collect(Collectors.toList());
- udfFunctionMapper.batchUpdateUdfFunc(udfFuncs);
- }
+ // if name unchanged, return directly without moving on HDFS
+ if (originResourceName.equals(name) && file == null) {
+ return result;
+ }
+ List<ResourcesTask> existResourcesList;
+ if (resource.isDirectory()) {
+ existResourcesList = resourceTaskMapper.selectSubfoldersFullNames(originFullName + FOLDER_SEPARATOR);
+ } else {
+ existResourcesList = resourceTaskMapper.selectByMap(
+ Collections.singletonMap("full_name", originFullName));
+ }
+
+ if (existResourcesList.size() > 0 && !fullName.equals(originFullName)) {
+ // check if any related task is online. If it is, it can not be updated.
+ for (ResourcesTask existResource : existResourcesList) {
+ int taskId = existResource.getTaskId();
+ if (processService.isTaskOnline(taskDefinitionMapper.selectById(taskId).getCode())) {
+ logger.error("can't be updated,because it is used of process definition that's online");
+ logger.error("resource task relation id:{} is used of task code {}", existResource.getId(),
+ taskDefinitionMapper.selectById(taskId).getCode());
+ putMsg(result, Status.RESOURCE_IS_USED);
+ return result;
+ }
}
- putMsg(result, Status.SUCCESS);
- Map<String, Object> resultMap = new HashMap<>();
- for (Map.Entry<Object, Object> entry : new BeanMap(resource).entrySet()) {
- if (!Constants.CLASS.equalsIgnoreCase(entry.getKey().toString())) {
- resultMap.put(entry.getKey().toString(), entry.getValue());
+ for (ResourcesTask existResource : existResourcesList) {
+ int taskId = existResource.getTaskId();
+ long taskCode = taskDefinitionMapper.selectById(taskId).getCode();
+
+ List<ProcessTaskRelation> processTaskRelation = processTaskRelationMapper.selectByMap(
+ Collections.singletonMap("post_task_code", taskCode));
+ if (processTaskRelation.size() > 0) {
+ long processDefinitionCode = processTaskRelation.get(0).getProcessDefinitionCode();
+ int processDefinitionVersion = processTaskRelation.get(0).getProcessDefinitionVersion();
+ List<ProcessTaskRelation> taskRelationList = processTaskRelationMapper.queryByProcessCode(
+ processTaskRelation.get(0).getProjectCode(),
+ processDefinitionCode);
+
+ List<TaskDefinition> taskDefinitionLogList = new ArrayList<>();
+
+ if (taskRelationList.size() > 0) {
+ ProcessDefinitionLog processDefinition =
+ processDefinitionLogMapper.queryByDefinitionCodeAndVersion(
+ processDefinitionCode, processDefinitionVersion);
+ for (ProcessTaskRelation taskRelation : taskRelationList) {
+ long taskCodeInProcess = taskRelation.getPostTaskCode();
+ TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(taskCodeInProcess);
+ if (taskCodeInProcess == taskCode) {
+ // originFullName is a prefix if isDirectory is true
+ taskDefinition.setTaskParams(RemoveResourceFromResourceList(originFullName,
+ taskDefinition.getTaskParams(),
+ resource.isDirectory()));
+ // if isDirectory is true, fullName is the new prefix. we replace old prefix
+ // of resource fullname with the new prefix.
+ // if isDirectory is false, fullName is the new path.
+ taskDefinition.setTaskParams(AddResourceToResourceList(originFullName,
+ fullName,
+ existResource.getFullName(),
+ taskDefinition.getTaskParams(),
+ resource.isDirectory()));
+ }
+ taskDefinitionLogList.add(taskDefinition);
+ }
+
+ // update workflow & task definition associated to the resource
+ if (processDefinition != null) {
+ processDefinitionService.updateProcessDefinition(loginUser,
+ processDefinition.getProjectCode(),
+ processDefinition.getName(),
+ processDefinition.getCode(),
+ processDefinition.getDescription(),
+ processDefinition.getGlobalParams(),
+ processDefinition.getLocations(),
+ processDefinition.getTimeout(),
+ tenantCode,
+ JSONUtils.toJsonString(taskRelationList.toArray()),
+ JSONUtils.toJsonString(taskDefinitionLogList.toArray()),
+ "",
+ processDefinition.getExecutionType());
+ }
+ }
}
}
- result.setData(resultMap);
- } catch (Exception e) {
- logger.error(Status.UPDATE_RESOURCE_ERROR.getMsg(), e);
- throw new ServiceException(Status.UPDATE_RESOURCE_ERROR);
- }
-
- // if name unchanged, return directly without moving on HDFS
- if (originResourceName.equals(name) && file == null) {
- return result;
}
if (file != null) {
// fail upload
if (!upload(loginUser, fullName, file, type)) {
- logger.error("Storage operation error, resourceId:{}, resourceName:{}, originFileName:{}.",
- resourceId, name, RegexUtils.escapeNRT(file.getOriginalFilename()));
+ logger.error("Storage operation error, resourceName:{}, originFileName:{}.",
+ name, RegexUtils.escapeNRT(file.getOriginalFilename()));
putMsg(result, Status.HDFS_OPERATION_ERROR);
throw new ServiceException(
String.format("upload resource: %s file: %s failed.", name, file.getOriginalFilename()));
}
if (!fullName.equals(originFullName)) {
try {
- storageOperate.delete(tenantCode, originFileName, false);
+ storageOperate.delete(originFullName, false);
} catch (IOException e) {
logger.error("Resource delete error, resourceFullName:{}.", originFullName, e);
throw new ServiceException(String.format("delete resource: %s failed.", originFullName));
}
}
- updateParentResourceSize(resource, resource.getSize() - originFileSize);
return result;
}
// get the path of dest file in hdfs
- String destHdfsFileName = storageOperate.getFileName(resource.getType(), tenantCode, fullName);
-
+ String destHdfsFileName = fullName;
try {
- logger.info("Start copy file {} -> {}.", originFileName, destHdfsFileName);
- storageOperate.copy(originFileName, destHdfsFileName, true, true);
+ logger.info("start copy {} -> {}", originFullName, destHdfsFileName);
+ storageOperate.copy(originFullName, destHdfsFileName, true, true);
} catch (Exception e) {
- logger.error(MessageFormat.format("Copy file {0} -> {1} fail.", originFileName, destHdfsFileName), e);
+ logger.error(MessageFormat.format(" copy {0} -> {1} fail", originFullName, destHdfsFileName), e);
putMsg(result, Status.HDFS_COPY_FAIL);
- throw new ServiceException(Status.HDFS_COPY_FAIL);
+ throw new ServiceException(MessageFormat.format(
+ Status.HDFS_COPY_FAIL.getMsg(), originFullName, destHdfsFileName));
}
return result;
@@ -667,6 +668,9 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
* query resources list paging
*
* @param loginUser login user
+ * @param fullName resource full name
+ * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource,
+ * can be different from the login user in the case of logging in as admin users.
* @param type resource type
* @param searchVal search value
* @param pageNo page number
@@ -674,30 +678,91 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
* @return resource list page
*/
@Override
- public Result queryResourceListPaging(User loginUser, int directoryId, ResourceType type, String searchVal,
- Integer pageNo, Integer pageSize) {
+ public Result queryResourceListPaging(User loginUser, String fullName, String resTenantCode,
+ ResourceType type, String searchVal, Integer pageNo, Integer pageSize) {
Result<Object> result = new Result<>();
- Page<Resource> page = new Page<>(pageNo, pageSize);
- if (directoryId != -1) {
- Resource directory = resourcesMapper.selectById(directoryId);
- if (directory == null) {
- logger.error("Resource does not exist, resourceId:{}.", directoryId);
- putMsg(result, Status.RESOURCE_NOT_EXIST);
- return result;
- }
+ PageInfo<StorageEntity> pageInfo = new PageInfo<>(pageNo, pageSize);
+
+ User user = userMapper.selectById(loginUser.getId());
+ if (user == null) {
+ logger.error("user {} not exists", loginUser.getId());
+ putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
+ return result;
}
- PageInfo<Resource> pageInfo = new PageInfo<>(pageNo, pageSize);
- Set<Integer> resourcesIds = resourcePermissionCheckService
- .userOwnedResourceIdsAcquisition(checkResourceType(type), loginUser.getId(), logger);
- if (resourcesIds.isEmpty()) {
- result.setData(pageInfo);
- putMsg(result, Status.SUCCESS);
+
+ Tenant tenant = tenantMapper.queryById(user.getTenantId());
+ if (tenant == null) {
+ logger.error("tenant not exists");
+ putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST);
+ return result;
+ }
+
+ String tenantCode = tenant.getTenantCode();
+
+ if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) {
+ logger.error("current user does not have permission");
+ putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
return result;
}
- IPage<Resource> resourceIPage = resourcesMapper.queryResourcePaging(page, directoryId, type.ordinal(),
- searchVal, new ArrayList<>(resourcesIds));
- pageInfo.setTotal((int) resourceIPage.getTotal());
- pageInfo.setTotalList(resourceIPage.getRecords());
+
+ String defaultPath = "";
+ List<StorageEntity> resourcesList = new ArrayList<>();
+
+ if (isAdmin(loginUser) && StringUtils.isBlank(fullName)) {
+ // list all tenants' resources to admin users in the root directory
+ List<User> userList = userMapper.selectList(null);
+ Set<String> visitedTenantEntityCode = new HashSet<>();
+ for (User userEntity : userList) {
+ String tenantEntityCode = tenantMapper.queryById(userEntity.getTenantId()).getTenantCode();
+ if (!visitedTenantEntityCode.contains(tenantEntityCode)) {
+ defaultPath = storageOperate.getResDir(tenantEntityCode);
+ if (type.equals(ResourceType.UDF)) {
+ defaultPath = storageOperate.getUdfDir(tenantEntityCode);
+ }
+ try {
+ resourcesList.addAll(storageOperate.listFilesStatus(defaultPath, defaultPath,
+ tenantEntityCode, type));
+
+ visitedTenantEntityCode.add(tenantEntityCode);
+ } catch (Exception e) {
+ logger.error(e.getMessage() + " Resource path: {}", defaultPath, e);
+ putMsg(result, Status.RESOURCE_NOT_EXIST);
+ throw new ServiceException(String.format(e.getMessage() +
+ " make sure resource path: %s exists in hdfs", defaultPath));
+ }
+ }
+ }
+ } else {
+ defaultPath = storageOperate.getResDir(tenantCode);
+ if (type.equals(ResourceType.UDF)) {
+ defaultPath = storageOperate.getUdfDir(tenantCode);
+ }
+
+ try {
+ if (StringUtils.isBlank(fullName)) {
+ resourcesList = storageOperate.listFilesStatus(defaultPath, defaultPath, tenantCode, type);
+ } else {
+ resourcesList = storageOperate.listFilesStatus(fullName, defaultPath, tenantCode, type);
+ }
+ } catch (Exception e) {
+ logger.error(e.getMessage() + " Resource path: {}", fullName, e);
+ putMsg(result, Status.RESOURCE_NOT_EXIST);
+ throw new ServiceException(String.format(e.getMessage() +
+ " make sure resource path: %s exists in hdfs", defaultPath));
+ }
+ }
+
+ // remove leading and trailing spaces in searchVal
+ String trimmedSearchVal = searchVal != null ? searchVal.trim() : "";
+ // filter based on trimmed searchVal
+ List<StorageEntity> filteredResourceList = resourcesList.stream()
+ .filter(x -> x.getFileName().matches("(.*)" + trimmedSearchVal + "(.*)")).collect(Collectors.toList());
+ // inefficient pagination
+ List<StorageEntity> slicedResourcesList = filteredResourceList.stream().skip((long) (pageNo - 1) * pageSize)
+ .limit(pageSize).collect(Collectors.toList());
+
+ pageInfo.setTotal(resourcesList.size());
+ pageInfo.setTotalList(slicedResourcesList);
result.setData(pageInfo);
putMsg(result, Status.SUCCESS);
return result;
@@ -714,22 +779,22 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
*/
private void createDirectory(User loginUser, String fullName, ResourceType type, Result<Object> result) {
String tenantCode = tenantMapper.queryById(loginUser.getTenantId()).getTenantCode();
- String directoryName = storageOperate.getFileName(type, tenantCode, fullName);
+ // String directoryName = storageOperate.getFileName(type, tenantCode, fullName);
String resourceRootPath = storageOperate.getDir(type, tenantCode);
try {
- if (!storageOperate.exists(tenantCode, resourceRootPath)) {
+ if (!storageOperate.exists(resourceRootPath)) {
storageOperate.createTenantDirIfNotExists(tenantCode);
}
- if (!storageOperate.mkdir(tenantCode, directoryName)) {
- logger.error("create resource directory {} failed", directoryName);
+ if (!storageOperate.mkdir(tenantCode, fullName)) {
+ logger.error("create resource directory {} failed", fullName);
putMsg(result, Status.STORE_OPERATE_CREATE_ERROR);
- throw new ServiceException(String.format("create resource directory: %s failed.", directoryName));
+ // throw new ServiceException(String.format("create resource directory: %s failed.", fullName));
}
} catch (Exception e) {
- logger.error("create resource directory {} failed", directoryName);
+ logger.error("create resource directory {} failed", fullName);
putMsg(result, Status.STORE_OPERATE_CREATE_ERROR);
- throw new ServiceException(String.format("create resource directory: %s failed.", directoryName));
+ throw new ServiceException(String.format("create resource directory: %s failed.", fullName));
}
}
@@ -755,15 +820,14 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
String localFilename = FileUtils.getUploadFilename(tenantCode, UUID.randomUUID().toString());
// save file to hdfs, and delete original file
- String fileName = storageOperate.getFileName(type, tenantCode, fullName);
String resourcePath = storageOperate.getDir(type, tenantCode);
try {
// if tenant dir not exists
- if (!storageOperate.exists(tenantCode, resourcePath)) {
+ if (!storageOperate.exists(resourcePath)) {
storageOperate.createTenantDirIfNotExists(tenantCode);
}
org.apache.dolphinscheduler.api.utils.FileUtils.copyInputStreamToFile(file, localFilename);
- storageOperate.upload(tenantCode, localFilename, fileName, true, true);
+ storageOperate.upload(tenantCode, localFilename, fullName, true, true);
} catch (Exception e) {
FileUtils.deleteFile(localFilename);
logger.error(e.getMessage(), e);
@@ -777,15 +841,67 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
*
* @param loginUser login user
* @param type resource type
+ * @param fullName resource full name
* @return resource list
*/
@Override
- public Map<String, Object> queryResourceList(User loginUser, ResourceType type) {
+ public Map<String, Object> queryResourceList(User loginUser, ResourceType type, String fullName) {
Map<String, Object> result = new HashMap<>();
- List<Resource> allResourceList = queryAuthoredResourceList(loginUser, type);
- Visitor resourceTreeVisitor = new ResourceTreeVisitor(allResourceList);
- result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren());
+ User user = userMapper.selectById(loginUser.getId());
+ if (user == null) {
+ logger.error("user {} not exists", loginUser.getId());
+ putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
+ return null;
+ }
+
+ Tenant tenant = tenantMapper.queryById(user.getTenantId());
+ if (tenant == null) {
+ logger.error("tenant not exists");
+ putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST);
+ return null;
+ }
+ String tenantCode = tenant.getTenantCode();
+
+ String defaultPath = "";
+ List<StorageEntity> resourcesList = new ArrayList<>();
+
+ if (StringUtils.isBlank(fullName)) {
+ if (isAdmin(loginUser)) {
+ List<User> userList = userMapper.selectList(null);
+ Set<String> visitedTenantEntityCode = new HashSet<>();
+ for (User userEntity : userList) {
+ Tenant tt = tenantMapper.queryById(userEntity.getTenantId());
+ String tenantEntityCode = tenantMapper.queryById(userEntity.getTenantId()).getTenantCode();
+ if (!visitedTenantEntityCode.contains(tenantEntityCode)) {
+ defaultPath = storageOperate.getResDir(tenantEntityCode);
+ if (type.equals(ResourceType.UDF)) {
+ defaultPath = storageOperate.getUdfDir(tenantEntityCode);
+ }
+ resourcesList.addAll(storageOperate.listFilesStatusRecursively(defaultPath, defaultPath,
+ tenantEntityCode, type));
+ visitedTenantEntityCode.add(tenantEntityCode);
+ }
+ }
+ } else {
+ defaultPath = storageOperate.getResDir(tenantCode);
+ if (type.equals(ResourceType.UDF)) {
+ defaultPath = storageOperate.getUdfDir(tenantCode);
+ }
+
+ resourcesList = storageOperate.listFilesStatusRecursively(defaultPath, defaultPath, tenantCode, type);
+ }
+ } else {
+ defaultPath = storageOperate.getResDir(tenantCode);
+ if (type.equals(ResourceType.UDF)) {
+ defaultPath = storageOperate.getUdfDir(tenantCode);
+ }
+
+ resourcesList = storageOperate.listFilesStatusRecursively(fullName, defaultPath, tenantCode, type);
+ }
+
+ Visitor resourceTreeVisitor = new ResourceTreeVisitor(resourcesList);
+ result.put(Constants.DATA_LIST, resourceTreeVisitor.visit(defaultPath).getChildren());
putMsg(result, Status.SUCCESS);
return result;
@@ -824,121 +940,264 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
}
}
List<Resource> resources = new ResourceFilter(suffix, new ArrayList<>(allResourceList)).filter();
- Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources);
- result.setData(resourceTreeVisitor.visit().getChildren());
+ // Transform into StorageEntity for compatibility
+ List<StorageEntity> transformedResourceList = resources.stream()
+ .map(this::createStorageEntityBasedOnResource)
+ .collect(Collectors.toList());
+ Visitor visitor = new ResourceTreeVisitor(transformedResourceList);
+ result.setData(visitor.visit("").getChildren());
putMsg(result, Status.SUCCESS);
return result;
}
+ /**
+ * transform resource object into StorageEntity object
+ *
+ * @param resource a resource object
+ * @return a storageEntity object
+ */
+ private StorageEntity createStorageEntityBasedOnResource(Resource resource) {
+ StorageEntity entity = new StorageEntity();
+ entity.setFullName(resource.getFullName());
+ entity.setPfullName(resource.getPid() == -1 ? ""
+ : resourcesMapper.selectById(resource.getPid()).getFullName());
+ entity.setDirectory(resource.isDirectory());
+ entity.setAlias(resource.getAlias());
+ entity.setId(resource.getId());
+ entity.setType(resource.getType());
+ entity.setDescription(resource.getDescription());
+
+ return entity;
+ }
+
/**
* delete resource
*
* @param loginUser login user
- * @param resourceId resource id
+ * @param fullName resource full name
+ * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource,
+ * can be different from the login user in the case of logging in as admin users.
* @return delete result code
* @throws IOException exception
*/
@Override
- @Transactional
- public Result<Object> delete(User loginUser, int resourceId) throws IOException {
- // get resource by id
- Result<Object> resultCheck = new Result<>();
- Resource resource = resourcesMapper.selectById(resourceId);
- if (resource == null) {
- logger.error("Resource does not exist, resourceId:{}.", resourceId);
- putMsg(resultCheck, Status.RESOURCE_NOT_EXIST);
- return resultCheck;
- }
- String funcPermissionKey =
- resource.getType().equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_DELETE
- : ApiFuncIdentificationConstant.UDF_DELETE;
- boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resourceId},
- checkResourceType(resource.getType()), funcPermissionKey);
- if (!canOperatorPermissions) {
- putMsg(resultCheck, Status.NO_CURRENT_OPERATING_PERMISSION);
- return resultCheck;
- }
+ @Transactional(rollbackFor = Exception.class)
+ public Result<Object> delete(User loginUser, String fullName,
+ String resTenantCode) throws IOException {
+ Result<Object> result = new Result<>();
- Result<Object> result = checkResourceUploadStartupState();
+ result = checkResourceUploadStartupState();
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
return result;
}
- if (!canOperator(loginUser, resource.getUserId())) {
- putMsg(result, Status.USER_NO_OPERATION_PERM);
+
+ User user = userMapper.selectById(loginUser.getId());
+ if (user == null) {
+ logger.error("user {} not exists", loginUser.getId());
+ putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
+ return result;
+ }
+
+ Tenant tenant = tenantMapper.queryById(user.getTenantId());
+ if (tenant == null) {
+ logger.error("tenant not exists");
+ putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST);
return result;
}
- String tenantCode = getTenantCode(resource.getUserId(), result);
- if (StringUtils.isEmpty(tenantCode)) {
- logger.error("Tenant of current login user does not specified, loginUserName:{}.", loginUser.getUserName());
+ String tenantCode = tenant.getTenantCode();
+
+ if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) {
+ logger.error("current user does not have permission");
+ putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
return result;
}
- // get all resource id of process definitions those are released
- List<Map<String, Object>> list = processDefinitionMapper.listResources();
- Map<Integer, Set<Long>> resourceProcessMap =
- ResourceProcessDefinitionUtils.getResourceProcessDefinitionMap(list);
- Set<Integer> resourceIdSet = resourceProcessMap.keySet();
- // get all children of the resource
- List<Integer> allChildren = listAllChildren(resource, true);
+ String defaultPath = storageOperate.getResDir(tenantCode);
+ StorageEntity resource;
+ try {
+ resource = storageOperate.getFileStatus(fullName, defaultPath, resTenantCode, null);
+ } catch (Exception e) {
+ logger.error(e.getMessage() + " Resource path: {}", fullName, e);
+ putMsg(result, Status.RESOURCE_NOT_EXIST);
+ throw new ServiceException(String.format(e.getMessage() + " Resource path: %s", fullName));
+ }
- Integer[] needDeleteResourceIdArray = allChildren.toArray(new Integer[allChildren.size()]);
- if (needDeleteResourceIdArray.length >= 2) {
- logger.warn("Resource can not be deleted because there are files or folders {} in the current directory.",
- resource.getFileName());
- putMsg(result, Status.RESOURCE_HAS_FOLDER, resource.getFileName());
+ if (resource == null) {
+ logger.error("Resource does not exist, resource full name:{}.", fullName);
+ putMsg(result, Status.RESOURCE_NOT_EXIST);
return result;
}
+ // recursively delete a folder
+ List<String> allChildren = storageOperate.listFilesStatusRecursively(fullName, defaultPath,
+ resTenantCode, resource.getType()).stream().map(storageEntity -> storageEntity.getFullName())
+ .collect(Collectors.toList());
+
+ Set<ResourcesTask> resourcesNeedToDeleteSet = new HashSet<>();
+ String[] allChildrenFullNameArray = allChildren.stream().toArray(String[]::new);
+
+ // check before using allChildrenFullNameArray to query full names.
+ if (allChildrenFullNameArray.length != 0) {
+ resourcesNeedToDeleteSet.addAll(
+ resourceTaskMapper.selectBatchFullNames(allChildrenFullNameArray));
+ }
+
// if resource type is UDF,need check whether it is bound by UDF function
if (resource.getType() == (ResourceType.UDF)) {
- List<UdfFunc> udfFuncs = udfFunctionMapper.listUdfByResourceId(needDeleteResourceIdArray);
- List<Integer> udfFuncIds = udfFuncs.stream().map(UdfFunc::getId).collect(Collectors.toList());
+ List<UdfFunc> udfFuncs = udfFunctionMapper.listUdfByResourceFullName(allChildrenFullNameArray);
if (CollectionUtils.isNotEmpty(udfFuncs)) {
logger.warn("Resource can not be deleted because it is bound by UDF functions, udfFuncIds:{}",
- udfFuncIds);
+ udfFuncs);
putMsg(result, Status.UDF_RESOURCE_IS_BOUND, udfFuncs.get(0).getFuncName());
return result;
}
}
- if (resourceIdSet.contains(resource.getPid())) {
- logger.warn(
- "Resource can not be deleted because it is used by process definition, resourceId:{}, processDefinitionCode:{}.",
- resource.getId(), resource.getPid());
- putMsg(result, Status.RESOURCE_IS_USED);
- return result;
- }
- resourceIdSet.retainAll(allChildren);
- if (CollectionUtils.isNotEmpty(resourceIdSet)) {
- for (Integer resId : resourceIdSet) {
- logger.warn(
- "Resource can not be deleted because it is used by process definition, resourceId:{}, processDefinitionCode:{}.",
- resId, resourceProcessMap.get(resId));
+ // delete data in database
+ if (resourcesNeedToDeleteSet.size() > 0) {
+ for (ResourcesTask resourcesTask : resourcesNeedToDeleteSet) {
+ int taskId = resourcesTask.getTaskId();
+ if (processService.isTaskOnline(taskDefinitionMapper.selectById(taskId).getCode())) {
+ logger.error("can't be deleted,because it is used of process definition that's online");
+ logger.error("resource task relation id:{} is used of task code {}", resourcesTask.getId(),
+ taskDefinitionMapper.selectById(taskId).getCode());
+ putMsg(result, Status.RESOURCE_IS_USED);
+ return result;
+ }
}
- putMsg(result, Status.RESOURCE_IS_USED);
- return result;
- }
- // get hdfs file by type
- String storageFilename = storageOperate.getFileName(resource.getType(), tenantCode, resource.getFullName());
- // delete data in database
- resourcesMapper.selectBatchIds(Arrays.asList(needDeleteResourceIdArray)).forEach(item -> {
- updateParentResourceSize(item, item.getSize() * -1);
- });
- resourcesMapper.deleteIds(needDeleteResourceIdArray);
- resourceUserMapper.deleteResourceUserArray(0, needDeleteResourceIdArray);
+ for (ResourcesTask existResource : resourcesNeedToDeleteSet) {
+ int taskId = existResource.getTaskId();
+ long taskCode = taskDefinitionMapper.selectById(taskId).getCode();
+
+ // use taskCode to get processDefinitionCode, then get a list of processDefinitionLog.
+ List<ProcessTaskRelation> processTaskRelation = processTaskRelationMapper.selectByMap(
+ Collections.singletonMap("post_task_code", taskCode));
+ if (processTaskRelation.size() > 0) {
+ long processDefinitionCode = processTaskRelation.get(0).getProcessDefinitionCode();
+ int processDefinitionVersion = processTaskRelation.get(0).getProcessDefinitionVersion();
+ List<ProcessTaskRelation> taskRelationList = processTaskRelationMapper.queryByProcessCode(
+ processTaskRelation.get(0).getProjectCode(),
+ processDefinitionCode);
+
+ List<TaskDefinition> taskDefinitionLogList = new ArrayList<>();
+
+ if (taskRelationList.size() > 0) {
+ ProcessDefinitionLog processDefinition =
+ processDefinitionLogMapper.queryByDefinitionCodeAndVersion(
+ processDefinitionCode, processDefinitionVersion);
+ for (ProcessTaskRelation taskRelation : taskRelationList) {
+ long taskCodeInProcess = taskRelation.getPostTaskCode();
+ TaskDefinition taskDefinition = taskDefinitionMapper.queryByCode(taskCodeInProcess);
+ if (taskCodeInProcess == taskCode) {
+ taskDefinition.setTaskParams(RemoveResourceFromResourceList(existResource.getFullName(),
+ taskDefinition.getTaskParams(), false));
+ }
+ taskDefinitionLogList.add(taskDefinition);
+ }
+
+ // update workflow & task definition associated to the resource
+ if (processDefinition != null) {
+ processDefinitionService.updateProcessDefinition(loginUser,
+ processDefinition.getProjectCode(),
+ processDefinition.getName(),
+ processDefinition.getCode(),
+ processDefinition.getDescription(),
+ processDefinition.getGlobalParams(),
+ processDefinition.getLocations(),
+ processDefinition.getTimeout(),
+ tenantCode,
+ JSONUtils.toJsonString(taskRelationList.toArray()),
+ JSONUtils.toJsonString(taskDefinitionLogList.toArray()),
+ "",
+ processDefinition.getExecutionType());
+ }
+ }
+ }
+ }
+ }
- // delete file on hdfs
+ // delete file on hdfs,S3
+ storageOperate.delete(fullName, allChildren, true);
- // delete file on storage
- storageOperate.delete(tenantCode, storageFilename, true);
- logger.info("Resource delete complete, tenantCode:{}, fileName:{}.", tenantCode, storageFilename);
putMsg(result, Status.SUCCESS);
return result;
}
+ private String RemoveResourceFromResourceList(String stringToDelete, String taskParameter, boolean isDir) {
+ Map<String, Object> taskParameters = JSONUtils.parseObject(
+ taskParameter,
+ new TypeReference<Map<String, Object>>() {
+ });
+ if (taskParameters.containsKey("resourceList")) {
+ String resourceListStr = JSONUtils.toJsonString(taskParameters.get("resourceList"));
+ List<ResourceInfo> resourceInfoList = JSONUtils.toList(resourceListStr, ResourceInfo.class);
+ List<ResourceInfo> updatedResourceInfoList;
+ if (isDir) {
+ String stringToDeleteWSeparator = stringToDelete + FOLDER_SEPARATOR;
+ // use start with to identify any prefix matching folder path
+ updatedResourceInfoList = resourceInfoList.stream()
+ .filter(Objects::nonNull)
+ .filter(resourceInfo -> !resourceInfo.getResourceName().startsWith(stringToDeleteWSeparator))
+ .collect(Collectors.toList());
+ } else {
+ updatedResourceInfoList = resourceInfoList.stream()
+ .filter(Objects::nonNull)
+ .filter(resourceInfo -> !resourceInfo.getResourceName().equals(stringToDelete))
+ .collect(Collectors.toList());
+ }
+ taskParameters.put("resourceList", updatedResourceInfoList);
+ return JSONUtils.toJsonString(taskParameters);
+ }
+ return taskParameter;
+ }
+
+ private String AddResourceToResourceList(String oldPrefix, String newPrefix, String resFullName,
+ String taskParameter, boolean isDir) {
+ Map<String, Object> taskParameters = JSONUtils.parseObject(
+ taskParameter,
+ new TypeReference<Map<String, Object>>() {
+ });
+
+ if (taskParameters.containsKey("resourceList")) {
+ String resourceListStr = JSONUtils.toJsonString(taskParameters.get("resourceList"));
+ List<ResourceInfo> resourceInfos = JSONUtils.toList(resourceListStr, ResourceInfo.class);
+
+ // add updated resource to replace the original resource.
+ ResourceInfo newResource = new ResourceInfo();
+ if (isDir) {
+ // we add spearator here because we dont want rare cases like
+ // oldFullName: .../folderToDelete and a resource path: .../folderToDeleteAnotherFolder
+ // Therefore, we make sure the oldFullName has a format of .../folderToDelete/ when
+ // modifying resourceFullNames in taskDefinition.
+ String oldFullNameWSeparator = oldPrefix + FOLDER_SEPARATOR;
+ String newFullNameWSpearator = newPrefix + FOLDER_SEPARATOR;
+
+ newResource.setResourceName(resFullName.replace(oldFullNameWSeparator, newFullNameWSpearator));
+ } else {
+ newResource.setResourceName(newPrefix);
+ }
+ resourceInfos.add(newResource);
+
+ taskParameters.put("resourceList", resourceInfos);
+
+ return JSONUtils.toJsonString(taskParameters);
+ }
+ return taskParameter;
+ }
+
+ private String RemoveResourceFromIdsNew(int idToDelete, String idNews) {
+
+ String[] resourceIds = idNews.split(",");
+ Set<Integer> resourceIdSet = Arrays.stream(resourceIds)
+ .map(Integer::parseInt)
+ .filter(integerId -> !integerId.equals(idToDelete))
+ .collect(Collectors.toSet());
+ return Joiner.on(",").join(resourceIdSet);
+ }
+
/**
* verify resource by name and type
*
@@ -950,43 +1209,11 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
@Override
public Result<Object> verifyResourceName(String fullName, ResourceType type, User loginUser) {
Result<Object> result = new Result<>();
- String funcPermissionKey = type.equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_RENAME
- : ApiFuncIdentificationConstant.UDF_FILE_VIEW;
- boolean canOperatorPermissions =
- canOperatorPermissions(loginUser, null, AuthorizationType.RESOURCE_FILE_ID, funcPermissionKey);
- if (!canOperatorPermissions) {
- putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
- return result;
- }
putMsg(result, Status.SUCCESS);
- if (checkResourceExists(fullName, type.ordinal())) {
- logger.warn("Resource with same name exists so can not create again, resourceType:{}, resourceName:{}.",
+ if (checkResourceExists(fullName)) {
+ logger.error("Resource with same name exists so can not create again, resourceType:{}, resourceName:{}.",
type, RegexUtils.escapeNRT(fullName));
putMsg(result, Status.RESOURCE_EXIST);
- } else {
- // query tenant
- Tenant tenant = tenantMapper.queryById(loginUser.getTenantId());
- if (tenant != null) {
- String tenantCode = tenant.getTenantCode();
- try {
- String filename = storageOperate.getFileName(type, tenantCode, fullName);
- if (storageOperate.exists(tenantCode, filename)) {
- logger.warn(
- "Resource file with same name exists so can not create again, tenantCode:{}, resourceName:{}.",
- tenantCode, RegexUtils.escapeNRT(filename));
- putMsg(result, Status.RESOURCE_FILE_EXIST, filename);
- }
-
- } catch (Exception e) {
- logger.error("Verify resource name failed, resourceName:{}.", RegexUtils.escapeNRT(fullName), e);
- putMsg(result, Status.STORE_OPERATE_CREATE_ERROR);
- }
- } else {
- logger.error(
- "Tenant does not exist or tenant of current login user does not specified, loginUserName:{}.",
- loginUser.getUserName());
- putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST);
- }
}
return result;
@@ -995,158 +1222,189 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
/**
* verify resource by full name or pid and type
*
- * @param fullName resource full name
- * @param id resource id
+ * @param fileName resource file name
* @param type resource type
+ * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource,
+ * can be different from the login user in the case of logging in as admin users.
* @return true if the resource full name or pid not exists, otherwise return false
*/
@Override
- public Result<Object> queryResource(User loginUser, String fullName, Integer id, ResourceType type) {
+ public Result<Object> queryResourceByFileName(User loginUser, String fileName, ResourceType type,
+ String resTenantCode) {
Result<Object> result = new Result<>();
- if (StringUtils.isBlank(fullName) && id == null) {
- logger.warn("Parameter fullName and id is invalid.");
+ if (StringUtils.isBlank(fileName)) {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR);
return result;
}
- Resource resource;
- if (StringUtils.isNotBlank(fullName)) {
- List<Resource> resourceList = resourcesMapper.queryResource(fullName, type.ordinal());
- if (CollectionUtils.isEmpty(resourceList)) {
- logger.error("Resources do not exist, fullName:{}.",
- RegexUtils.escapeNRT(fullName));
- putMsg(result, Status.RESOURCE_NOT_EXIST);
- return result;
- }
- resource = resourceList.get(0);
- } else {
- resource = resourcesMapper.selectById(id);
- if (resource == null) {
- logger.error("Resource does not exist, resourceId:{}.", id);
- putMsg(result, Status.RESOURCE_NOT_EXIST);
- return result;
- }
- int pid = resource.getPid();
- resource = resourcesMapper.selectById(pid);
- if (resource == null) {
- logger.error("Resource does not exist, resourceId:{}.", pid);
- putMsg(result, Status.RESOURCE_NOT_EXIST);
- return result;
- }
+
+ User user = userMapper.selectById(loginUser.getId());
+ if (user == null) {
+ logger.error("user {} not exists", loginUser.getId());
+ putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
+ return result;
+ }
+
+ Tenant tenant = tenantMapper.queryById(user.getTenantId());
+ if (tenant == null) {
+ logger.error("tenant not exists");
+ putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST);
+ return result;
}
- String funcPermissionKey = type.equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_VIEW
- : ApiFuncIdentificationConstant.UDF_FILE_VIEW;
- boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resource.getId()},
- checkResourceType(type), funcPermissionKey);
- if (!canOperatorPermissions) {
+
+ String tenantCode = tenant.getTenantCode();
+
+ if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) {
+ logger.error("current user does not have permission");
putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
return result;
}
+
+ String defaultPath = storageOperate.getResDir(resTenantCode);
+ if (type.equals(ResourceType.UDF)) {
+ defaultPath = storageOperate.getUdfDir(resTenantCode);
+ }
+
+ StorageEntity file;
+ try {
+ file = storageOperate.getFileStatus(defaultPath + fileName, defaultPath, resTenantCode, type);
+ } catch (Exception e) {
+ logger.error(e.getMessage() + " Resource path: {}", defaultPath + fileName, e);
+ putMsg(result, Status.RESOURCE_NOT_EXIST);
+ return result;
+ }
+
putMsg(result, Status.SUCCESS);
- result.setData(resource);
+ result.setData(file);
return result;
}
/**
* get resource by id
- * @param id resource id
+ * @param fullName resource full name
+ * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource,
+ * can be different from the login user in the case of logging in as admin users.
* @return resource
*/
@Override
- public Result<Object> queryResourceById(User loginUser, Integer id) {
+ public Result<Object> queryResourceByFullName(User loginUser, String fullName, String resTenantCode,
+ ResourceType type) throws IOException {
Result<Object> result = new Result<>();
- Resource resource = resourcesMapper.selectById(id);
- if (resource == null) {
- logger.error("Resource does not exist, resourceId:{}.", id);
- putMsg(result, Status.RESOURCE_NOT_EXIST);
+
+ User user = userMapper.selectById(loginUser.getId());
+ if (user == null) {
+ logger.error("user {} not exists", loginUser.getId());
+ putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
return result;
}
- String funcPermissionKey =
- resource.getType().equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_VIEW
- : ApiFuncIdentificationConstant.UDF_FILE_VIEW;
- boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{id},
- checkResourceType(resource.getType()), funcPermissionKey);
- if (!canOperatorPermissions) {
+
+ Tenant tenant = tenantMapper.queryById(user.getTenantId());
+ if (tenant == null) {
+ logger.error("tenant not exists");
+ putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST);
+ return result;
+ }
+
+ String tenantCode = tenant.getTenantCode();
+
+ if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) {
+ logger.error("current user does not have permission");
putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
return result;
}
+
+ String defaultPath = storageOperate.getResDir(resTenantCode);
+ if (type.equals(ResourceType.UDF)) {
+ defaultPath = storageOperate.getUdfDir(resTenantCode);
+ }
+
+ StorageEntity file;
+ try {
+ file = storageOperate.getFileStatus(fullName, defaultPath, resTenantCode, type);
+ } catch (Exception e) {
+ logger.error(e.getMessage() + " Resource path: {}", fullName, e);
+ putMsg(result, Status.RESOURCE_NOT_EXIST);
+ throw new ServiceException(String.format(e.getMessage() + " Resource path: %s", fullName));
+ }
+
putMsg(result, Status.SUCCESS);
- result.setData(resource);
+ result.setData(file);
return result;
}
/**
* view resource file online
*
- * @param resourceId resource id
+ * @param fullName resource fullName
+ * @param resTenantCode owner's tenant code of the resource
* @param skipLineNum skip line number
* @param limit limit
* @return resource content
*/
@Override
- public Result<Object> readResource(User loginUser, int resourceId, int skipLineNum, int limit) {
+ public Result<Object> readResource(User loginUser, String fullName, String resTenantCode,
+ int skipLineNum, int limit) {
Result<Object> result = checkResourceUploadStartupState();
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
return result;
}
- // get resource by id
- Resource resource = resourcesMapper.selectById(resourceId);
- if (resource == null) {
- logger.error("Resource does not exist, resourceId:{}.", resourceId);
- putMsg(result, Status.RESOURCE_NOT_EXIST);
+
+ User user = userMapper.selectById(loginUser.getId());
+ if (user == null) {
+ logger.error("user {} not exists", loginUser.getId());
+ putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
+ return result;
+ }
+
+ Tenant tenant = tenantMapper.queryById(user.getTenantId());
+ if (tenant == null) {
+ logger.error("tenant not exists");
+ putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST);
return result;
}
- String funcPermissionKey =
- resource.getType().equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_VIEW
- : ApiFuncIdentificationConstant.UDF_FILE_VIEW;
- boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resourceId},
- checkResourceType(resource.getType()), funcPermissionKey);
- if (!canOperatorPermissions) {
+
+ String tenantCode = tenant.getTenantCode();
+
+ if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) {
+ logger.error("current user does not have permission");
putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
return result;
}
+
// check preview or not by file suffix
- String nameSuffix = Files.getFileExtension(resource.getAlias());
+ String nameSuffix = Files.getFileExtension(fullName);
String resourceViewSuffixes = FileUtils.getResourceViewSuffixes();
if (StringUtils.isNotEmpty(resourceViewSuffixes)) {
List<String> strList = Arrays.asList(resourceViewSuffixes.split(","));
if (!strList.contains(nameSuffix)) {
- logger.warn("Resource suffix does not support view, resourceId:{}, suffix:{}.", resourceId, nameSuffix);
+ logger.error("Resource suffix does not support view,resourceFullName:{}, suffix:{}.", fullName,
+ nameSuffix);
putMsg(result, Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW);
return result;
}
}
- String tenantCode = getTenantCode(resource.getUserId(), result);
- if (StringUtils.isEmpty(tenantCode)) {
- logger.error("Tenant of resource user does not specified, resourceUserIde:{}.", resource.getUserId());
- return result;
- }
-
- // source path
- String resourceFileName = storageOperate.getResourceFileName(tenantCode, resource.getFullName());
- logger.info("Resource file path is {}.", resourceFileName);
+ List<String> content = new ArrayList<>();
try {
- if (storageOperate.exists(tenantCode, resourceFileName)) {
- List<String> content = storageOperate.vimFile(tenantCode, resourceFileName, skipLineNum, limit);
-
- logger.info(
- "Vim file content in path {} success, tenantCode:{}, fileName:{}, skipLineNum:{}, limit:{}.",
- resourceFileName, tenantCode, resourceFileName, skipLineNum, limit);
- putMsg(result, Status.SUCCESS);
- Map<String, Object> map = new HashMap<>();
- map.put(ALIAS, resource.getAlias());
- map.put(CONTENT, String.join("\n", content));
- result.setData(map);
+ if (storageOperate.exists(fullName)) {
+ content = storageOperate.vimFile(tenantCode, fullName, skipLineNum, limit);
} else {
- logger.error("File does not exist in storage, fileName:{}.", resourceFileName);
- putMsg(result, Status.RESOURCE_FILE_NOT_EXIST, resourceFileName);
+ logger.error("read file {} not exist in storage", fullName);
+ putMsg(result, Status.RESOURCE_FILE_NOT_EXIST, fullName);
+ return result;
}
} catch (Exception e) {
- logger.error("Resource file read error, fileName:{}.", resourceFileName, e);
+ logger.error("Resource {} read failed", fullName, e);
putMsg(result, Status.HDFS_OPERATION_ERROR);
+ return result;
}
+ putMsg(result, Status.SUCCESS);
+ Map<String, Object> map = new HashMap<>();
+ map.put(ALIAS, fullName);
+ map.put(CONTENT, String.join("\n", content));
+ result.setData(map);
+
return result;
}
@@ -1159,36 +1417,47 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
* @param fileSuffix file suffix
* @param desc description
* @param content content
- * @param pid pid
* @param currentDir current directory
* @return create result code
*/
@Override
@Transactional
public Result<Object> onlineCreateResource(User loginUser, ResourceType type, String fileName, String fileSuffix,
- String desc, String content, int pid, String currentDir) {
+ String desc, String content, String currentDir) {
Result<Object> result = new Result<>();
- boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.RESOURCE_FILE_ID,
- ApiFuncIdentificationConstant.FILE_ONLINE_CREATE);
- if (!canOperatorPermissions) {
- putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
- return result;
- }
result = checkResourceUploadStartupState();
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
return result;
}
+
+ User user = userMapper.selectById(loginUser.getId());
+ if (user == null) {
+ logger.error("user {} not exists", loginUser.getId());
+ putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
+ return result;
+ }
+
+ Tenant tenant = tenantMapper.queryById(user.getTenantId());
+ if (tenant == null) {
+ logger.error("tenant not exists");
+ putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST);
+ return result;
+ }
+
+ String tenantCode = tenant.getTenantCode();
+
+ if (!isUserTenantValid(isAdmin(loginUser), tenantCode, "")) {
+ logger.error("current user does not have permission");
+ putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
+ return result;
+ }
+
if (FileUtils.directoryTraversal(fileName)) {
logger.warn("File name verify failed, fileName:{}.", RegexUtils.escapeNRT(fileName));
putMsg(result, Status.VERIFY_PARAMETER_NAME_FAILED);
return result;
}
- if (checkDescriptionLength(desc)) {
- logger.warn("Parameter description is too long.");
- putMsg(result, Status.DESCRIPTION_TOO_LONG_ERROR);
- return result;
- }
// check file suffix
String nameSuffix = fileSuffix.trim();
@@ -1203,31 +1472,19 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
}
String name = fileName.trim() + "." + nameSuffix;
- String fullName = getFullName(currentDir, name);
- result = verifyResource(loginUser, type, fullName, pid);
- if (!result.getCode().equals(Status.SUCCESS.getCode())) {
- return result;
- }
-
- // save data
- Date now = new Date();
- Resource resource = new Resource(pid, name, fullName, false, desc, name, loginUser.getId(), type,
- content.getBytes().length, now, now);
- resourcesMapper.insert(resource);
- updateParentResourceSize(resource, resource.getSize());
-
- putMsg(result, Status.SUCCESS);
- permissionPostHandle(resource.getType(), loginUser, resource.getId());
- Map<String, Object> resultMap = new HashMap<>();
- for (Map.Entry<Object, Object> entry : new BeanMap(resource).entrySet()) {
- if (!Constants.CLASS.equalsIgnoreCase(entry.getKey().toString())) {
- resultMap.put(entry.getKey().toString(), entry.getValue());
- }
+ String fullName = "";
+ String userResRootPath = storageOperate.getResDir(tenantCode);
+ if (!currentDir.contains(userResRootPath)) {
+ fullName = userResRootPath + name;
+ } else {
+ fullName = currentDir + name;
}
- result.setData(resultMap);
- String tenantCode = tenantMapper.queryById(loginUser.getTenantId()).getTenantCode();
+ result = verifyResourceName(fullName, type, loginUser);
+ if (!result.getCode().equals(Status.SUCCESS.getCode())) {
+ return result;
+ }
result = uploadContentToStorage(loginUser, fullName, tenantCode, content);
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
@@ -1250,9 +1507,11 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
@Transactional
public Result<Object> onlineCreateOrUpdateResourceWithDir(User loginUser, String fileFullName, String desc,
String content) {
- if (checkResourceExists(fileFullName, ResourceType.FILE.ordinal())) {
+ // TODO: need update to third party service
+ if (checkResourceExists(fileFullName)) {
Resource resource = resourcesMapper.queryResource(fileFullName, ResourceType.FILE.ordinal()).get(0);
- Result<Object> result = this.updateResourceContent(loginUser, resource.getId(), content);
+ Result<Object> result = this.updateResourceContent(loginUser, fileFullName,
+ resource.getUserName(), content);
if (result.getCode() == Status.SUCCESS.getCode()) {
resource.setDescription(desc);
Map<String, Object> resultMap = new HashMap<>();
@@ -1279,7 +1538,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
}
}
return this.onlineCreateResource(
- loginUser, ResourceType.FILE, resourceName, resourceSuffix, desc, content, pid,
+ loginUser, ResourceType.FILE, resourceName, resourceSuffix, desc, content,
currDirPath.toString());
}
}
@@ -1311,7 +1570,7 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
private int queryOrCreateDirId(User user, int pid, String currentDir, String dirName) {
String dirFullName = currentDir + FOLDER_SEPARATOR + dirName;
- if (checkResourceExists(dirFullName, ResourceType.FILE.ordinal())) {
+ if (checkResourceExists(dirFullName)) {
List<Resource> resourceList = resourcesMapper.queryResource(dirFullName, ResourceType.FILE.ordinal());
return resourceList.get(0).getId();
} else {
@@ -1319,8 +1578,12 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
Result<Object> createDirResult = this.createDirectory(
user, dirName, EMPTY_STRING, ResourceType.FILE, pid, currentDir);
if (createDirResult.getCode() == Status.SUCCESS.getCode()) {
- Map<String, Object> resultMap = (Map<String, Object>) createDirResult.getData();
- return resultMap.get("id") == null ? -1 : (Integer) resultMap.get("id");
+ // Map<String, Object> resultMap = (Map<String, Object>) createDirResult.getData();
+ // return resultMap.get("id") == null ? -1 : (Integer) resultMap.get("id");
+
+ // Since resource is kept in third party services, its id will always be -1.
+ return -1;
+
} else {
String msg = String.format("Create dir error, dirFullName:%s.", dirFullName);
logger.error(msg);
@@ -1379,76 +1642,89 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
/**
* updateProcessInstance resource
*
- * @param resourceId resource id
+ * @param fullName resource full name
+ * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource,
+ * can be different from the login user in the case of logging in as admin users.
* @param content content
* @return update result cod
*/
@Override
@Transactional
- public Result<Object> updateResourceContent(User loginUser, int resourceId, String content) {
+ public Result<Object> updateResourceContent(User loginUser, String fullName, String resTenantCode,
+ String content) {
Result<Object> result = checkResourceUploadStartupState();
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
return result;
}
- Resource resource = resourcesMapper.selectById(resourceId);
- if (resource == null) {
- logger.error("Resource does not exist, resourceId:{}.", resourceId);
- putMsg(result, Status.RESOURCE_NOT_EXIST);
+ User user = userMapper.selectById(loginUser.getId());
+ if (user == null) {
+ logger.error("user {} not exists", loginUser.getId());
+ putMsg(result, Status.USER_NOT_EXIST, loginUser.getId());
return result;
}
- String funcPermissionKey =
- resource.getType().equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_UPDATE
- : ApiFuncIdentificationConstant.UDF_UPDATE;
- boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resourceId},
- checkResourceType(resource.getType()), funcPermissionKey);
- if (!canOperatorPermissions) {
+
+ Tenant tenant = tenantMapper.queryById(user.getTenantId());
+ if (tenant == null) {
+ logger.error("tenant not exists");
+ putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST);
+ return result;
+ }
+
+ String tenantCode = tenant.getTenantCode();
+
+ if (!isUserTenantValid(isAdmin(loginUser), tenantCode, resTenantCode)) {
+ logger.error("current user does not have permission");
putMsg(result, Status.NO_CURRENT_OPERATING_PERMISSION);
return result;
}
+
+ StorageEntity resource;
+ try {
+ resource = storageOperate.getFileStatus(fullName, "", resTenantCode, ResourceType.FILE);
+ } catch (Exception e) {
+ logger.error("error occurred when fetching resource information , resource full name {}", fullName);
+ putMsg(result, Status.RESOURCE_NOT_EXIST);
+ return result;
+ }
+
+ if (resource == null) {
+ logger.error("Resource does not exist, resource full name:{}.", fullName);
+ putMsg(result, Status.RESOURCE_NOT_EXIST);
+ return result;
+ }
+
// check can edit by file suffix
String nameSuffix = Files.getFileExtension(resource.getAlias());
String resourceViewSuffixes = FileUtils.getResourceViewSuffixes();
if (StringUtils.isNotEmpty(resourceViewSuffixes)) {
List<String> strList = Arrays.asList(resourceViewSuffixes.split(","));
if (!strList.contains(nameSuffix)) {
- logger.warn("Resource suffix does not support view, resourceId:{}, suffix:{}.", resourceId, nameSuffix);
+ logger.warn("Resource suffix does not support view, resource full name:{}, suffix:{}.",
+ fullName, nameSuffix);
putMsg(result, Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW);
return result;
}
}
- String tenantCode = getTenantCode(resource.getUserId(), result);
- if (StringUtils.isEmpty(tenantCode)) {
- logger.error("Tenant of resource user does not specified, resourceUserIde:{}.", resource.getUserId());
- return result;
- }
- long originFileSize = resource.getSize();
- resource.setSize(content.getBytes().length);
- resource.setUpdateTime(new Date());
- resourcesMapper.updateById(resource);
-
- result = uploadContentToStorage(loginUser, resource.getFullName(), tenantCode, content);
- updateParentResourceSize(resource, resource.getSize() - originFileSize);
+ result = uploadContentToStorage(loginUser, resource.getFullName(), resTenantCode, content);
if (!result.getCode().equals(Status.SUCCESS.getCode())) {
throw new ServiceException(result.getMsg());
} else
- logger.info("Update resource content complete, resourceId:{}.", resourceId);
+ logger.info("Update resource content complete, resource full name:{}.", fullName);
return result;
}
/**
- * @param resourceName resource name
+ * @param fullName resource full name
* @param tenantCode tenant code
* @param content content
* @return result
*/
- private Result<Object> uploadContentToStorage(User loginUser, String resourceName, String tenantCode,
- String content) {
+ private Result<Object> uploadContentToStorage(User loginUser, String fullName, String tenantCode, String content) {
Result<Object> result = new Result<>();
String localFilename = "";
- String storageFileName = "";
try {
localFilename = FileUtils.getUploadFilename(tenantCode, UUID.randomUUID().toString());
@@ -1461,42 +1737,40 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
}
// get resource file path
- storageFileName = storageOperate.getResourceFileName(tenantCode, resourceName);
String resourcePath = storageOperate.getResDir(tenantCode);
- logger.info("Resource file name is {}, resource dir is {}.", storageFileName, resourcePath);
+ logger.info("resource path is {}, resource dir is {}", fullName, resourcePath);
- if (!storageOperate.exists(tenantCode, resourcePath)) {
+ if (!storageOperate.exists(resourcePath)) {
// create if tenant dir not exists
storageOperate.createTenantDirIfNotExists(tenantCode);
logger.info("Create tenant dir because path {} does not exist, tenantCode:{}.", resourcePath,
tenantCode);
}
- if (storageOperate.exists(tenantCode, storageFileName)) {
- storageOperate.delete(tenantCode, storageFileName, false);
+ if (storageOperate.exists(fullName)) {
+ storageOperate.delete(fullName, false);
}
- storageOperate.upload(tenantCode, localFilename, storageFileName, true, true);
+ storageOperate.upload(tenantCode, localFilename, fullName, true, true);
} catch (Exception e) {
- logger.error("Upload content to storage error, tenantCode:{}, destFileName:{}.", tenantCode,
- storageFileName, e);
+ logger.error("Upload content to storage error, tenantCode:{}, destFileName:{}.", tenantCode, localFilename,
+ e);
result.setCode(Status.HDFS_OPERATION_ERROR.getCode());
- result.setMsg(String.format("copy %s to hdfs %s fail", localFilename, storageFileName));
+ result.setMsg(String.format("copy %s to hdfs %s fail", localFilename, fullName));
return result;
}
- logger.info("Upload content to storage complete, tenantCode:{}, destFileName:{}.", tenantCode, storageFileName);
+ logger.info("Upload content to storage complete, tenantCode:{}, destFileName:{}.", tenantCode, localFilename);
putMsg(result, Status.SUCCESS);
return result;
}
/**
* download file
- *
- * @param resourceId resource id
* @return resource content
* @throws IOException exception
*/
@Override
- public org.springframework.core.io.Resource downloadResource(User loginUser, int resourceId) throws IOException {
+ public org.springframework.core.io.Resource downloadResource(User loginUser,
+ String fullName) throws IOException {
// if resource upload startup
if (!PropertyUtils.getResUploadStartupState()) {
logger.warn("Storage does not start up, resource upload startup state: {}.",
@@ -1504,59 +1778,42 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
throw new ServiceException("hdfs not startup");
}
- Resource resource = resourcesMapper.selectById(resourceId);
- if (resource == null) {
- logger.error("Resource does not exist, resourceId:{}.", resourceId);
- return null;
- }
-
- String funcPermissionKey =
- resource.getType().equals(ResourceType.FILE) ? ApiFuncIdentificationConstant.FILE_DOWNLOAD
- : ApiFuncIdentificationConstant.UDF_DOWNLOAD;
- boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{resourceId},
- checkResourceType(resource.getType()), funcPermissionKey);
- if (!canOperatorPermissions) {
- logger.error("{}: {}", Status.NO_CURRENT_OPERATING_PERMISSION.getMsg(),
- PropertyUtils.getResUploadStartupState());
- throw new ServiceException(Status.NO_CURRENT_OPERATING_PERMISSION.getMsg());
- }
- if (resource.isDirectory()) {
- logger.warn("Resource is a directory so can not download it, resourceId:{}.", resourceId);
+ if (fullName.endsWith("/")) {
+ logger.error("resource id {} is directory,can't download it", fullName);
throw new ServiceException("can't download directory");
}
- int userId = resource.getUserId();
+ int userId = loginUser.getId();
User user = userMapper.selectById(userId);
if (user == null) {
logger.error("User does not exits, userId:{}.", userId);
- throw new ServiceException(String.format("resource owner id %d does not exist", userId));
- }
-
- Tenant tenant = tenantMapper.queryById(user.getTenantId());
- if (tenant == null) {
- logger.error("Tenant does not exists, tenantId:{}.", user.getTenantId());
- throw new ServiceException(
- String.format("The tenant id %d of resource owner does not exist", user.getTenantId()));
+ throw new ServiceException(String.format("Resource owner id %d does not exist", userId));
}
- String tenantCode = tenant.getTenantCode();
+ String tenantCode = "";
- String fileName = storageOperate.getFileName(resource.getType(), tenantCode, resource.getFullName());
+ if (user.getTenantId() != 0) {
+ Tenant tenant = tenantMapper.queryById(user.getTenantId());
+ if (tenant == null) {
+ logger.error("Tenant id {} not exists", user.getTenantId());
+ throw new ServiceException(
+ String.format("The tenant id %d of resource owner not exist", user.getTenantId()));
+ }
+ tenantCode = tenant.getTenantCode();
+ }
- String localFileName = FileUtils.getDownloadFilename(resource.getAlias());
- logger.info("Resource path is {}, download local filename is {}.", fileName, localFileName);
+ String[] aliasArr = fullName.split("/");
+ String alias = aliasArr[aliasArr.length - 1];
+ String localFileName = FileUtils.getDownloadFilename(alias);
+ logger.info("Resource path is {}, download local filename is {}", alias, localFileName);
try {
- storageOperate.download(tenantCode, fileName, localFileName, false, true);
- org.springframework.core.io.Resource file2Resource =
- org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(localFileName);
- if (file2Resource != null) {
- logger.info("Download resource complete, path:{}, localFileName:{}.", fileName, localFileName);
- }
- return file2Resource;
+ storageOperate.download(tenantCode, fullName, localFileName, false, true);
+ return org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(localFileName);
} catch (IOException e) {
- logger.error("Download resource error, path:{}, localFileName:{}.", fileName, localFileName, e);
- throw new ServerException("download the resource file failed ,it may be related to your storage");
+ logger.error("Download resource error, the path is {}, and local filename is {}, the error message is {}",
+ fullName, localFileName, e.getMessage());
+ throw new ServerException("Download the resource file failed ,it may be related to your storage");
}
}
@@ -1585,8 +1842,12 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
}
List<ResourceComponent> list;
if (CollectionUtils.isNotEmpty(resourceList)) {
- Visitor visitor = new ResourceTreeVisitor(resourceList);
- list = visitor.visit().getChildren();
+ // Transform into StorageEntity for compatibility
+ List<StorageEntity> transformedResourceList = resourceList.stream()
+ .map(this::createStorageEntityBasedOnResource)
+ .collect(Collectors.toList());
+ Visitor visitor = new ResourceTreeVisitor(transformedResourceList);
+ list = visitor.visit("").getChildren();
} else {
list = new ArrayList<>(0);
}
@@ -1597,12 +1858,13 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
}
@Override
- public Resource queryResourcesFileInfo(String userName, String fullName) {
+ public Resource queryResourcesFileInfo(String userName, String fileName) {
+ // TODO: It is used in PythonGateway, should be revised
User user = userMapper.queryByUserNameAccurately(userName);
- Result<Object> resourceResponse = this.queryResource(user, fullName, null, ResourceType.FILE);
+ Result<Object> resourceResponse = this.queryResourceByFileName(user, fileName, ResourceType.FILE, "");
if (resourceResponse.getCode() != Status.SUCCESS.getCode()) {
String msg =
- String.format("Query resource by fullName failed, userName:%s, fullName:%s", userName, fullName);
+ String.format("Query resource by fullName failed, userName:%s, fullName:%s", userName, fileName);
logger.error(msg);
throw new IllegalArgumentException(msg);
}
@@ -1637,8 +1899,12 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
} else {
list = new ArrayList<>(0);
}
- Visitor visitor = new ResourceTreeVisitor(list);
- result.put(Constants.DATA_LIST, visitor.visit().getChildren());
+ // Transform into StorageEntity for compatibility
+ List<StorageEntity> transformedResourceList = list.stream()
+ .map(this::createStorageEntityBasedOnResource)
+ .collect(Collectors.toList());
+ Visitor visitor = new ResourceTreeVisitor(transformedResourceList);
+ result.put(Constants.DATA_LIST, visitor.visit("").getChildren());
putMsg(result, Status.SUCCESS);
return result;
}
@@ -1717,13 +1983,17 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
}
List<Resource> authedResources = queryResourceList(userId, Constants.AUTHORIZE_WRITABLE_PERM);
- Visitor visitor = new ResourceTreeVisitor(authedResources);
- String visit = JSONUtils.toJsonString(visitor.visit(), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
+ // Transform into StorageEntity for compatibility
+ List<StorageEntity> transformedResourceList = authedResources.stream()
+ .map(this::createStorageEntityBasedOnResource)
+ .collect(Collectors.toList());
+ Visitor visitor = new ResourceTreeVisitor(transformedResourceList);
+ String visit = JSONUtils.toJsonString(visitor.visit(""), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
logger.info(visit);
String jsonTreeStr =
- JSONUtils.toJsonString(visitor.visit().getChildren(), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
+ JSONUtils.toJsonString(visitor.visit("").getChildren(), SerializationFeature.ORDER_MAP_ENTRIES_BY_KEYS);
logger.info(jsonTreeStr);
- result.put(Constants.DATA_LIST, visitor.visit().getChildren());
+ result.put(Constants.DATA_LIST, visitor.visit("").getChildren());
putMsg(result, Status.SUCCESS);
return result;
}
@@ -1742,30 +2012,6 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
}
}
- /**
- * get tenantCode by UserId
- *
- * @param userId user id
- * @param result return result
- * @return tenant code
- */
- private String getTenantCode(int userId, Result<Object> result) {
- User user = userMapper.selectById(userId);
- if (user == null) {
- logger.error("user {} not exists", userId);
- putMsg(result, Status.USER_NOT_EXIST, userId);
- return null;
- }
-
- Tenant tenant = tenantMapper.queryById(user.getTenantId());
- if (tenant == null) {
- logger.error("tenant not exists");
- putMsg(result, Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST);
- return null;
- }
- return tenant.getTenantCode();
- }
-
/**
* list all children id
*
@@ -1832,4 +2078,27 @@ public class ResourcesServiceImpl extends BaseServiceImpl implements ResourcesSe
private AuthorizationType checkResourceType(ResourceType type) {
return type.equals(ResourceType.FILE) ? AuthorizationType.RESOURCE_FILE_ID : AuthorizationType.UDF_FILE;
}
+
+ /**
+ * check permission by comparing login user's tenantCode with tenantCode in the request
+ *
+ * @param isAdmin is the login user admin
+ * @param userTenantCode loginUser's tenantCode
+ * @param resTenantCode tenantCode in the request field "resTenantCode" for tenant code owning the resource,
+ * can be different from the login user in the case of logging in as admin users.
+ * @return isValid
+ */
+ private boolean isUserTenantValid(boolean isAdmin, String userTenantCode,
+ String resTenantCode) throws ServiceException {
+ if (!isAdmin) {
+ resTenantCode = resTenantCode == null ? "" : resTenantCode;
+ if (!StringUtils.isBlank(resTenantCode) && !resTenantCode.equals(userTenantCode)) {
+ // if an ordinary user directly send a query API with a different tenantCode and fullName "",
+ // still he/she does not have read permission.
+ return false;
+ }
+ }
+
+ return true;
+ }
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java
index c5e5da007c..a7973eff94 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UdfFuncServiceImpl.java
@@ -25,15 +25,16 @@ import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.enums.AuthorizationType;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
-import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
+import org.apache.dolphinscheduler.service.storage.StorageOperate;
import org.apache.commons.lang3.StringUtils;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@@ -66,6 +67,9 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
@Autowired
private UDFUserMapper udfUserMapper;
+ @Autowired(required = false)
+ private StorageOperate storageOperate;
+
/**
* create udf function
*
@@ -75,7 +79,6 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
* @param argTypes argument types
* @param database database
* @param desc description
- * @param resourceId resource id
* @param className class name
* @return create result code
*/
@@ -84,11 +87,11 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
public Result<Object> createUdfFunction(User loginUser,
String funcName,
String className,
+ String fullName,
String argTypes,
String database,
String desc,
- UdfType type,
- int resourceId) {
+ UdfType type) {
Result<Object> result = new Result<>();
boolean canOperatorPermissions = canOperatorPermissions(loginUser, null, AuthorizationType.UDF,
@@ -117,9 +120,15 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
return result;
}
- Resource resource = resourceMapper.selectById(resourceId);
- if (resource == null) {
- logger.error("Resource does not exist, resourceId:{}.", resourceId);
+ Boolean existResource = false;
+ try {
+ existResource = storageOperate.exists(fullName);
+ } catch (IOException e) {
+ logger.error("Check resource error: {}", fullName, e);
+ }
+
+ if (!existResource) {
+ logger.error("resource full name {} is not exist", fullName);
putMsg(result, Status.RESOURCE_NOT_EXIST);
return result;
}
@@ -137,8 +146,9 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
udf.setDatabase(database);
}
udf.setDescription(desc);
- udf.setResourceId(resourceId);
- udf.setResourceName(resource.getFullName());
+ // set resourceId to -1 because we do not store resource to db anymore, instead we use fullName
+ udf.setResourceId(-1);
+ udf.setResourceName(fullName);
udf.setType(type);
udf.setCreateTime(now);
@@ -178,7 +188,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
}
UdfFunc udfFunc = udfFuncMapper.selectById(id);
if (udfFunc == null) {
- logger.error("Resource does not exist, resourceId:{}.", id);
+ logger.error("Resource does not exist, udf func id:{}.", id);
putMsg(result, Status.RESOURCE_NOT_EXIST);
return result;
}
@@ -196,7 +206,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
* @param argTypes argument types
* @param database data base
* @param desc description
- * @param resourceId resource id
+ * @param fullName resource full name
* @param className class name
* @return update result code
*/
@@ -209,7 +219,7 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
String database,
String desc,
UdfType type,
- int resourceId) {
+ String fullName) {
Result<Object> result = new Result<>();
boolean canOperatorPermissions = canOperatorPermissions(loginUser, new Object[]{udfFuncId},
@@ -251,13 +261,23 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
}
}
- Resource resource = resourceMapper.selectById(resourceId);
- if (resource == null) {
- logger.error("Resource does not exist, resourceId:{}.", resourceId);
+ Boolean doesResExist = false;
+ try {
+ doesResExist = storageOperate.exists(fullName);
+ } catch (Exception e) {
+ logger.error("udf resource checking error", fullName);
result.setCode(Status.RESOURCE_NOT_EXIST.getCode());
result.setMsg(Status.RESOURCE_NOT_EXIST.getMsg());
return result;
}
+
+ if (!doesResExist) {
+ logger.error("resource full name {} is not exist", fullName);
+ result.setCode(Status.RESOURCE_NOT_EXIST.getCode());
+ result.setMsg(Status.RESOURCE_NOT_EXIST.getMsg());
+ return result;
+ }
+
Date now = new Date();
udf.setFuncName(funcName);
udf.setClassName(className);
@@ -266,8 +286,9 @@ public class UdfFuncServiceImpl extends BaseServiceImpl implements UdfFuncServic
udf.setDatabase(database);
}
udf.setDescription(desc);
- udf.setResourceId(resourceId);
- udf.setResourceName(resource.getFullName());
+ // set resourceId to -1 because we do not store resource to db anymore, instead we use fullName
+ udf.setResourceId(-1);
+ udf.setResourceName(fullName);
udf.setType(type);
udf.setUpdateTime(now);
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
index 611f37bf30..3b65634979 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
@@ -1173,7 +1173,7 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
if (CollectionUtils.isNotEmpty(components)) {
for (ResourceComponent component : components) {
// verify whether exist
- if (!storageOperate.exists(oldTenantCode,
+ if (!storageOperate.exists(
String.format(Constants.FORMAT_S_S, srcBasePath, component.getFullName()))) {
logger.error("Resource file: {} does not exist, copy error.", component.getFullName());
throw new ServiceException(Status.RESOURCE_NOT_EXIST);
@@ -1188,8 +1188,8 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
if (CollectionUtils.isEmpty(component.getChildren())) {
// if not exist,need create it
- if (!storageOperate.exists(oldTenantCode,
- String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()))) {
+ if (!storageOperate
+ .exists(String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()))) {
storageOperate.mkdir(newTenantCode,
String.format(Constants.FORMAT_S_S, dstBasePath, component.getFullName()));
}
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
index 8c43deb997..6bfcddcb29 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
@@ -65,11 +65,15 @@ public class ResourcesControllerTest extends AbstractControllerTest {
public void testQuerytResourceList() throws Exception {
Map<String, Object> mockResult = new HashMap<>();
mockResult.put(Constants.STATUS, Status.SUCCESS);
- Mockito.when(resourcesService.queryResourceList(Mockito.any(), Mockito.any())).thenReturn(mockResult);
+ Mockito.when(resourcesService.queryResourceList(Mockito.any(), Mockito.any(), Mockito.anyString()))
+ .thenReturn(mockResult);
+ MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
+ paramsMap.add("fullName", "dolphinscheduler/resourcePath");
+ paramsMap.add("type", ResourceType.FILE.name());
MvcResult mvcResult = mockMvc.perform(get("/resources/list")
.header(SESSION_ID, sessionId)
- .param("type", ResourceType.FILE.name()))
+ .params(paramsMap))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
@@ -85,8 +89,8 @@ public class ResourcesControllerTest extends AbstractControllerTest {
Result mockResult = new Result<>();
mockResult.setCode(Status.SUCCESS.getCode());
Mockito.when(resourcesService.queryResourceListPaging(
- Mockito.any(), Mockito.anyInt(), Mockito.any(), Mockito.anyString(), Mockito.anyInt(),
- Mockito.anyInt()))
+ Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.any(),
+ Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt()))
.thenReturn(mockResult);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
@@ -95,6 +99,8 @@ public class ResourcesControllerTest extends AbstractControllerTest {
paramsMap.add("pageNo", "1");
paramsMap.add("searchVal", "test");
paramsMap.add("pageSize", "1");
+ paramsMap.add("fullName", "dolphinscheduler/resourcePath");
+ paramsMap.add("tenantCode", "123");
MvcResult mvcResult = mockMvc.perform(get("/resources")
.header(SESSION_ID, sessionId)
@@ -137,14 +143,17 @@ public class ResourcesControllerTest extends AbstractControllerTest {
public void testViewResource() throws Exception {
Result mockResult = new Result<>();
mockResult.setCode(Status.HDFS_NOT_STARTUP.getCode());
- Mockito.when(resourcesService.readResource(Mockito.any(), Mockito.anyInt(), Mockito.anyInt(), Mockito.anyInt()))
+ Mockito.when(resourcesService.readResource(Mockito.any(),
+ Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyInt()))
.thenReturn(mockResult);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("skipLineNum", "2");
paramsMap.add("limit", "100");
+ paramsMap.add("fullName", "dolphinscheduler/resourcePath");
+ paramsMap.add("tenantCode", "123");
- MvcResult mvcResult = mockMvc.perform(get("/resources/{id}/view", "5")
+ MvcResult mvcResult = mockMvc.perform(get("/resources/view")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isOk())
@@ -163,7 +172,7 @@ public class ResourcesControllerTest extends AbstractControllerTest {
mockResult.setCode(Status.TENANT_NOT_EXIST.getCode());
Mockito.when(resourcesService
.onlineCreateResource(Mockito.any(), Mockito.any(), Mockito.anyString(), Mockito.anyString(),
- Mockito.anyString(), Mockito.anyString(), Mockito.anyInt(), Mockito.anyString()))
+ Mockito.anyString(), Mockito.anyString(), Mockito.anyString()))
.thenReturn(mockResult);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
@@ -192,14 +201,17 @@ public class ResourcesControllerTest extends AbstractControllerTest {
public void testUpdateResourceContent() throws Exception {
Result mockResult = new Result<>();
mockResult.setCode(Status.TENANT_NOT_EXIST.getCode());
- Mockito.when(resourcesService.updateResourceContent(Mockito.any(), Mockito.anyInt(), Mockito.anyString()))
+ Mockito.when(resourcesService.updateResourceContent(Mockito.any(), Mockito.anyString(),
+ Mockito.anyString(), Mockito.anyString()))
.thenReturn(mockResult);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("id", "1");
paramsMap.add("content", "echo test_1111");
+ paramsMap.add("fullName", "dolphinscheduler/resourcePath");
+ paramsMap.add("tenantCode", "123");
- MvcResult mvcResult = mockMvc.perform(put("/resources/1/update-content")
+ MvcResult mvcResult = mockMvc.perform(put("/resources/update-content")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isOk())
@@ -215,9 +227,14 @@ public class ResourcesControllerTest extends AbstractControllerTest {
@Test
public void testDownloadResource() throws Exception {
- Mockito.when(resourcesService.downloadResource(Mockito.any(), Mockito.anyInt())).thenReturn(null);
+ Mockito.when(resourcesService.downloadResource(Mockito.any(), Mockito.anyString()))
+ .thenReturn(null);
- MvcResult mvcResult = mockMvc.perform(get("/resources/{id}/download", 5)
+ MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
+ paramsMap.add("fullName", "dolphinscheduler/resourcePath");
+
+ MvcResult mvcResult = mockMvc.perform(get("/resources/download")
+ .params(paramsMap)
.header(SESSION_ID, sessionId))
.andExpect(status().is(HttpStatus.BAD_REQUEST.value()))
.andReturn();
@@ -231,7 +248,7 @@ public class ResourcesControllerTest extends AbstractControllerTest {
mockResult.setCode(Status.TENANT_NOT_EXIST.getCode());
Mockito.when(udfFuncService
.createUdfFunction(Mockito.any(), Mockito.anyString(), Mockito.anyString(), Mockito.anyString(),
- Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.anyInt()))
+ Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.any()))
.thenReturn(mockResult);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
@@ -242,8 +259,9 @@ public class ResourcesControllerTest extends AbstractControllerTest {
paramsMap.add("database", "database");
paramsMap.add("description", "description");
paramsMap.add("resourceId", "1");
+ paramsMap.add("fullName", "dolphinscheduler/resourcePath");
- MvcResult mvcResult = mockMvc.perform(post("/resources/{resourceId}/udf-func", "123")
+ MvcResult mvcResult = mockMvc.perform(post("/resources/udf-func")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isCreated())
@@ -282,7 +300,8 @@ public class ResourcesControllerTest extends AbstractControllerTest {
mockResult.setCode(Status.TENANT_NOT_EXIST.getCode());
Mockito.when(udfFuncService
.updateUdfFunc(Mockito.any(), Mockito.anyInt(), Mockito.anyString(), Mockito.anyString(),
- Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.any(), Mockito.anyInt()))
+ Mockito.anyString(), Mockito.anyString(), Mockito.anyString(), Mockito.any(),
+ Mockito.anyString()))
.thenReturn(mockResult);
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
@@ -294,8 +313,9 @@ public class ResourcesControllerTest extends AbstractControllerTest {
paramsMap.add("database", "database");
paramsMap.add("description", "description");
paramsMap.add("resourceId", "1");
+ paramsMap.add("fullName", "dolphinscheduler/resourcePath");
- MvcResult mvcResult = mockMvc.perform(put("/resources/{resourceId}/udf-func/{id}", "123", "456")
+ MvcResult mvcResult = mockMvc.perform(put("/resources/udf-func/{id}", "456")
.header(SESSION_ID, sessionId)
.params(paramsMap))
.andExpect(status().isOk())
@@ -465,10 +485,15 @@ public class ResourcesControllerTest extends AbstractControllerTest {
public void testDeleteResource() throws Exception {
Result mockResult = new Result<>();
mockResult.setCode(Status.SUCCESS.getCode());
- Mockito.when(resourcesService.delete(Mockito.any(), Mockito.anyInt())).thenReturn(mockResult);
-
- MvcResult mvcResult = mockMvc.perform(delete("/resources/{id}", "123")
- .header(SESSION_ID, sessionId))
+ Mockito.when(resourcesService.delete(Mockito.any(), Mockito.anyString(),
+ Mockito.anyString()))
+ .thenReturn(mockResult);
+ MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
+ paramsMap.add("fullName", "dolphinscheduler/resourcePath");
+ paramsMap.add("tenantCode", "123");
+ MvcResult mvcResult = mockMvc.perform(delete("/resources")
+ .header(SESSION_ID, sessionId)
+ .params(paramsMap))
.andExpect(status().isOk())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java
index 7178cbfce3..9d239c911b 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/TenantControllerTest.java
@@ -27,9 +27,12 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
+import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.http.MediaType;
@@ -41,17 +44,21 @@ public class TenantControllerTest extends AbstractControllerTest {
private static final Logger logger = LoggerFactory.getLogger(TenantControllerTest.class);
+ private MockedStatic<PropertyUtils> mockedStaticPropertyUtils;
+
@Test
public void testCreateTenant() throws Exception {
+ mockedStaticPropertyUtils = Mockito.mockStatic(PropertyUtils.class);
+
MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
paramsMap.add("tenantCode", "hayden");
paramsMap.add("queueId", "1");
paramsMap.add("description", "tenant description");
+ Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
- MvcResult mvcResult = mockMvc.perform(post("/tenants/")
+ MvcResult mvcResult = mockMvc.perform(post("/tenants")
.header(SESSION_ID, sessionId)
.params(paramsMap))
- .andExpect(status().isCreated())
.andExpect(content().contentType(MediaType.APPLICATION_JSON))
.andReturn();
@@ -59,6 +66,7 @@ public class TenantControllerTest extends AbstractControllerTest {
Assertions.assertEquals(Status.SUCCESS.getCode(), result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
+ mockedStaticPropertyUtils.close();
}
@Test
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitorTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitorTest.java
index f5ca3c559d..c51145b3a9 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitorTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitorTest.java
@@ -17,7 +17,7 @@
package org.apache.dolphinscheduler.api.dto.resources.visitor;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
-import org.apache.dolphinscheduler.dao.entity.Resource;
+import org.apache.dolphinscheduler.service.storage.StorageEntity;
import java.util.ArrayList;
import java.util.List;
@@ -31,53 +31,43 @@ import org.junit.jupiter.api.Test;
public class ResourceTreeVisitorTest {
@Test
- public void visit() throws Exception {
- List<Resource> resourceList = new ArrayList<>();
+ public void visit() {
+ List<StorageEntity> resourceList = new ArrayList<>();
+
+ StorageEntity resource1 = new StorageEntity();
+ resource1.setFullName("/default/a");
+ resource1.setPfullName("/default");
+
+ StorageEntity resource2 = new StorageEntity();
+ resource1.setFullName("/default/a/a1.txt");
+ resource1.setPfullName("/default/a");
- Resource resource1 = new Resource(3, -1, "b", "/b", true);
- Resource resource2 = new Resource(4, 2, "a1.txt", "/a/a1.txt", false);
- Resource resource3 = new Resource(5, 3, "b1.txt", "/b/b1.txt", false);
- Resource resource4 = new Resource(6, 3, "b2.jar", "/b/b2.jar", false);
- Resource resource5 = new Resource(7, -1, "b2", "/b2", true);
- Resource resource6 = new Resource(8, -1, "b2", "/b/b2", true);
- Resource resource7 = new Resource(9, 8, "c2.jar", "/b/b2/c2.jar", false);
resourceList.add(resource1);
resourceList.add(resource2);
- resourceList.add(resource3);
- resourceList.add(resource4);
- resourceList.add(resource5);
- resourceList.add(resource6);
- resourceList.add(resource7);
ResourceTreeVisitor resourceTreeVisitor = new ResourceTreeVisitor(resourceList);
- ResourceComponent resourceComponent = resourceTreeVisitor.visit();
+ ResourceComponent resourceComponent = resourceTreeVisitor.visit("/default");
Assertions.assertNotNull(resourceComponent.getChildren());
}
@Test
- public void rootNode() throws Exception {
- List<Resource> resourceList = new ArrayList<>();
+ public void rootNode() {
+ List<StorageEntity> resourceList = new ArrayList<>();
+
+ StorageEntity resource1 = new StorageEntity();
+ resource1.setFullName("/default/a");
+ resource1.setPfullName("/default");
+
+ StorageEntity resource2 = new StorageEntity();
+ resource1.setFullName("/default/a/a1.txt");
+ resource1.setPfullName("/default/a");
- Resource resource1 = new Resource(3, -1, "b", "/b", true);
- Resource resource2 = new Resource(4, 2, "a1.txt", "/a/a1.txt", false);
- Resource resource3 = new Resource(5, 3, "b1.txt", "/b/b1.txt", false);
- Resource resource4 = new Resource(6, 3, "b2.jar", "/b/b2.jar", false);
- Resource resource5 = new Resource(7, -1, "b2", "/b2", true);
- Resource resource6 = new Resource(8, -1, "b2", "/b/b2", true);
- Resource resource7 = new Resource(9, 8, "c2.jar", "/b/b2/c2.jar", false);
resourceList.add(resource1);
resourceList.add(resource2);
- resourceList.add(resource3);
- resourceList.add(resource4);
- resourceList.add(resource5);
- resourceList.add(resource6);
- resourceList.add(resource7);
ResourceTreeVisitor resourceTreeVisitor = new ResourceTreeVisitor(resourceList);
- Assertions.assertTrue(resourceTreeVisitor.rootNode(resource1));
- Assertions.assertTrue(resourceTreeVisitor.rootNode(resource2));
- Assertions.assertFalse(resourceTreeVisitor.rootNode(resource3));
-
+ Assertions.assertTrue(resourceTreeVisitor.rootNode(resource1, "/default"));
+ Assertions.assertFalse(resourceTreeVisitor.rootNode(resource2, "/default"));
}
}
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
index a4b0d79111..ecfbc72548 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/ResourcesServiceTest.java
@@ -19,7 +19,6 @@ package org.apache.dolphinscheduler.api.service;
import static org.mockito.ArgumentMatchers.eq;
-import org.apache.dolphinscheduler.api.constants.ApiFuncIdentificationConstant;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.permission.ResourcePermissionCheckService;
import org.apache.dolphinscheduler.api.service.impl.BaseServiceImpl;
@@ -27,7 +26,6 @@ import org.apache.dolphinscheduler.api.service.impl.ResourcesServiceImpl;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.constants.Constants;
-import org.apache.dolphinscheduler.common.enums.AuthorizationType;
import org.apache.dolphinscheduler.common.enums.UserType;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
@@ -41,6 +39,7 @@ import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
+import org.apache.dolphinscheduler.service.storage.StorageEntity;
import org.apache.dolphinscheduler.service.storage.StorageOperate;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
@@ -145,11 +144,6 @@ public class ResourcesServiceTest {
@Test
public void testCreateResource() {
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- null, 1, ApiFuncIdentificationConstant.FILE_UPLOAD, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- null, 1, serviceLogger)).thenReturn(true);
-
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
User user = new User();
user.setId(1);
@@ -158,10 +152,10 @@ public class ResourcesServiceTest {
// CURRENT_LOGIN_USER_TENANT_NOT_EXIST
MockMultipartFile mockMultipartFile = new MockMultipartFile("test.pdf", "test.pdf", "pdf", "test".getBytes());
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
- Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
+ Mockito.when(userMapper.selectById(user.getId())).thenReturn(getUser());
Mockito.when(tenantMapper.queryById(1)).thenReturn(null);
Result result = resourcesService.createResource(user, "ResourcesServiceTest", "ResourcesServiceTest",
- ResourceType.FILE, mockMultipartFile, -1, "/");
+ ResourceType.FILE, mockMultipartFile, "/");
logger.info(result.toString());
Assertions.assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), result.getMsg());
// set tenant for user
@@ -171,7 +165,7 @@ public class ResourcesServiceTest {
// HDFS_NOT_STARTUP
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
result = resourcesService.createResource(user, "ResourcesServiceTest", "ResourcesServiceTest",
- ResourceType.FILE, null, -1, "/");
+ ResourceType.FILE, null, "/");
logger.info(result.toString());
Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg());
@@ -179,7 +173,7 @@ public class ResourcesServiceTest {
mockMultipartFile = new MockMultipartFile("test.pdf", "".getBytes());
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
result = resourcesService.createResource(user, "ResourcesServiceTest", "ResourcesServiceTest",
- ResourceType.FILE, mockMultipartFile, -1, "/");
+ ResourceType.FILE, mockMultipartFile, "/");
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_FILE_IS_EMPTY.getMsg(), result.getMsg());
@@ -188,47 +182,33 @@ public class ResourcesServiceTest {
Mockito.when(Files.getFileExtension("test.pdf")).thenReturn("pdf");
Mockito.when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar");
result = resourcesService.createResource(user, "ResourcesServiceTest.jar", "ResourcesServiceTest",
- ResourceType.FILE, mockMultipartFile, -1, "/");
+ ResourceType.FILE, mockMultipartFile, "/");
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_SUFFIX_FORBID_CHANGE.getMsg(), result.getMsg());
// UDF_RESOURCE_SUFFIX_NOT_JAR
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- null, 1, ApiFuncIdentificationConstant.UDF_UPLOAD, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- null, 1, serviceLogger)).thenReturn(true);
-
mockMultipartFile = new MockMultipartFile("ResourcesServiceTest.pdf", "ResourcesServiceTest.pdf",
"pdf", "test".getBytes());
Mockito.when(Files.getFileExtension("ResourcesServiceTest.pdf")).thenReturn("pdf");
result = resourcesService.createResource(user, "ResourcesServiceTest.pdf", "ResourcesServiceTest",
- ResourceType.UDF, mockMultipartFile, -1, "/");
+ ResourceType.UDF, mockMultipartFile, "/");
logger.info(result.toString());
Assertions.assertEquals(Status.UDF_RESOURCE_SUFFIX_NOT_JAR.getMsg(), result.getMsg());
// FULL_FILE_NAME_TOO_LONG
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- null, 1, ApiFuncIdentificationConstant.FILE_UPLOAD, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- null, 1, serviceLogger)).thenReturn(true);
-
String tooLongFileName = getRandomStringWithLength(Constants.RESOURCE_FULL_NAME_MAX_LENGTH) + ".pdf";
mockMultipartFile = new MockMultipartFile(tooLongFileName, tooLongFileName, "pdf", "test".getBytes());
- Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(Files.getFileExtension(tooLongFileName)).thenReturn("pdf");
+ // '/databasePath/tenantCode/RESOURCE/'
+ Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
result = resourcesService.createResource(user, tooLongFileName, tooLongFileName, ResourceType.FILE,
- mockMultipartFile, -1, "/");
+ mockMultipartFile, "/");
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_FULL_NAME_TOO_LONG_ERROR.getMsg(), result.getMsg());
}
@Test
public void testCreateDirecotry() {
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- null, 1, ApiFuncIdentificationConstant.FOLDER_ONLINE_CREATE, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- null, 1, serviceLogger)).thenReturn(true);
-
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
User user = new User();
user.setId(1);
@@ -239,23 +219,18 @@ public class ResourcesServiceTest {
logger.info(result.toString());
Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg());
- // PARENT_RESOURCE_NOT_EXIST
+ // RESOURCE_EXIST
user.setId(1);
user.setTenantId(1);
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
+ Mockito.when(userMapper.selectById(user.getId())).thenReturn(getUser());
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
- Mockito.when(resourcesMapper.selectById(Mockito.anyInt())).thenReturn(null);
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- null, 1, ApiFuncIdentificationConstant.FOLDER_ONLINE_CREATE, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- null, 1, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- null, 1, ApiFuncIdentificationConstant.FILE_RENAME, serviceLogger)).thenReturn(true);
-
- result = resourcesService.createDirectory(user, "directoryTest", "directory test", ResourceType.FILE, 1, "/");
- logger.info(result.toString());
- Assertions.assertEquals(Status.PARENT_RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
- // RESOURCE_EXIST
+ Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
+ try {
+ Mockito.when(storageOperate.exists("/dolphinscheduler/123/resources/directoryTest")).thenReturn(true);
+ } catch (IOException e) {
+ logger.error(e.getMessage(), e);
+ }
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(resourcesMapper.existResource("/directoryTest", 0)).thenReturn(true);
result = resourcesService.createDirectory(user, "directoryTest", "directory test", ResourceType.FILE, -1, "/");
@@ -272,131 +247,126 @@ public class ResourcesServiceTest {
@Test
public void testUpdateResource() {
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- null, 1, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- new Object[]{1}, 1, serviceLogger)).thenReturn(true);
-
- Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
User user = new User();
user.setId(1);
user.setUserType(UserType.GENERAL_USER);
+ user.setTenantId(1);
+
+ Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
+ Mockito.when(userMapper.selectById(user.getId())).thenReturn(getUser());
+ Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
+ Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
+
// HDFS_NOT_STARTUP
- Result result = resourcesService.updateResource(user, 1, "ResourcesServiceTest", "ResourcesServiceTest",
- ResourceType.FILE, null);
+ Result result = resourcesService.updateResource(user, "ResourcesServiceTest",
+ "123", "ResourcesServiceTest", "", ResourceType.FILE, null);
logger.info(result.toString());
Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg());
- // RESOURCE_NOT_EXIST
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- null, 1, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- new Object[]{0}, 1, serviceLogger)).thenReturn(true);
- Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource());
- Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
- result = resourcesService.updateResource(user, 0, "ResourcesServiceTest", "ResourcesServiceTest",
- ResourceType.FILE, null);
- logger.info(result.toString());
- Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
-
// USER_NO_OPERATION_PERM
- user.setId(2);
+ Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
user.setUserType(UserType.GENERAL_USER);
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- null, 2, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- new Object[]{1}, 2, serviceLogger)).thenReturn(false);
- result = resourcesService.updateResource(user, 1, "ResourcesServiceTest", "ResourcesServiceTest",
- ResourceType.FILE, null);
+ // tenant who have access to resource is 123,
+ Tenant tenantWNoPermission = new Tenant();
+ tenantWNoPermission.setTenantCode("321");
+ Mockito.when(tenantMapper.queryById(1)).thenReturn(tenantWNoPermission);
+ result = resourcesService.updateResource(user,
+ "/dolphinscheduler/123/resources/ResourcesServiceTest",
+ "123",
+ "ResourcesServiceTest", "", ResourceType.FILE, null);
logger.info(result.toString());
Assertions.assertEquals(Status.NO_CURRENT_OPERATING_PERMISSION.getMsg(), result.getMsg());
- // RESOURCE_NOT_EXIST
- user.setId(1);
- Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
+ // SUCCESS
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
- Mockito.when(storageOperate.getFileName(Mockito.any(), Mockito.any(), Mockito.anyString())).thenReturn("test1");
-
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.UDF_FILE, null,
- 1, ApiFuncIdentificationConstant.UDF_UPDATE, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.UDF_FILE, new Object[]{1},
- 1, serviceLogger)).thenReturn(true);
try {
- Mockito.when(storageOperate.exists(Mockito.any(), Mockito.any())).thenReturn(false);
+ Mockito.when(storageOperate.exists(Mockito.any())).thenReturn(false);
} catch (IOException e) {
logger.error(e.getMessage(), e);
}
- result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest",
- ResourceType.UDF, null);
- Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
- // SUCCESS
- user.setId(1);
- Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
try {
- Mockito.when(storageOperate.exists(Mockito.any(), Mockito.any())).thenReturn(true);
- } catch (IOException e) {
- logger.error(e.getMessage(), e);
+ Mockito.when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest",
+ "/dolphinscheduler/123/resources/",
+ "123", ResourceType.FILE)).thenReturn(getStorageEntityResource());
+ result = resourcesService.updateResource(user, "/dolphinscheduler/123/resources/ResourcesServiceTest",
+ "123",
+ "ResourcesServiceTest", "", ResourceType.FILE, null);
+ logger.info(result.toString());
+ Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
+ } catch (Exception e) {
+ logger.error(e.getMessage() + " Resource path: {}", "/dolphinscheduler/123/resources/ResourcesServiceTest",
+ e);
}
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
- 1, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- new Object[]{1},
- 1, serviceLogger)).thenReturn(true);
- result = resourcesService.updateResource(user, 1, "ResourcesServiceTest.jar", "ResourcesServiceTest",
- ResourceType.FILE, null);
- logger.info(result.toString());
- Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
-
+ // Tests for udf resources.
// RESOURCE_EXIST
- Mockito.when(resourcesMapper.existResource("/ResourcesServiceTest1.jar", 0)).thenReturn(true);
- result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest",
- ResourceType.FILE, null);
- logger.info(result.toString());
- Assertions.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg());
- // USER_NOT_EXIST
+ try {
+ Mockito.when(storageOperate.exists("/dolphinscheduler/123/resources/ResourcesServiceTest2.jar"))
+ .thenReturn(true);
+ } catch (IOException e) {
+ logger.error("error occurred when checking resource: "
+ + "/dolphinscheduler/123/resources/ResourcesServiceTest2.jar");
+ }
- Mockito.when(userMapper.selectById(Mockito.anyInt())).thenReturn(null);
- result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest",
- ResourceType.UDF, null);
+ try {
+ Mockito.when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest1.jar",
+ "/dolphinscheduler/123/resources/",
+ "123", ResourceType.UDF)).thenReturn(getStorageEntityUdfResource());
+ } catch (Exception e) {
+ logger.error(e.getMessage() + " Resource path: {}",
+ "/dolphinscheduler/123/resources/ResourcesServiceTest1.jar", e);
+ }
+ result = resourcesService.updateResource(user, "/dolphinscheduler/123/resources/ResourcesServiceTest1.jar",
+ "123", "ResourcesServiceTest2.jar", "", ResourceType.UDF, null);
logger.info(result.toString());
- Assertions.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode());
+ Assertions.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg());
// TENANT_NOT_EXIST
- Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
Mockito.when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(null);
- result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest",
- ResourceType.UDF, null);
+ result = resourcesService.updateResource(user, "ResourcesServiceTest1.jar",
+ "", "ResourcesServiceTest", "", ResourceType.UDF, null);
logger.info(result.toString());
Assertions.assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), result.getMsg());
// SUCCESS
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
- result = resourcesService.updateResource(user, 1, "ResourcesServiceTest1.jar", "ResourcesServiceTest1.jar",
- ResourceType.UDF, null);
+ result = resourcesService.updateResource(user, "/dolphinscheduler/123/resources/ResourcesServiceTest1.jar",
+ "123", "ResourcesServiceTest1.jar", "", ResourceType.UDF, null);
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
-
}
@Test
public void testQueryResourceListPaging() {
User loginUser = new User();
loginUser.setId(1);
+ loginUser.setTenantId(1);
+ loginUser.setTenantCode("tenant1");
loginUser.setUserType(UserType.ADMIN_USER);
IPage<Resource> resourcePage = new Page<>(1, 10);
resourcePage.setTotal(1);
resourcePage.setRecords(getResourceList());
- Mockito.when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.RESOURCE_FILE_ID,
- 1, resourceLogger)).thenReturn(getSetIds());
+ List<StorageEntity> mockResList = new ArrayList<StorageEntity>();
+ mockResList.add(getStorageEntityResource());
+ List<User> mockUserList = new ArrayList<User>();
+ mockUserList.add(getUser());
+ Mockito.when(userMapper.selectList(null)).thenReturn(mockUserList);
+ Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
+ Mockito.when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant());
+ Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
- Mockito.when(
- resourcesMapper.queryResourcePaging(Mockito.any(Page.class), eq(-1), eq(0), eq("test"), Mockito.any()))
- .thenReturn(resourcePage);
- Result result = resourcesService.queryResourceListPaging(loginUser, -1, ResourceType.FILE, "test", 1, 10);
+ try {
+ Mockito.when(storageOperate.listFilesStatus("/dolphinscheduler/123/resources/",
+ "/dolphinscheduler/123/resources/",
+ "123", ResourceType.FILE)).thenReturn(mockResList);
+ } catch (Exception e) {
+ logger.error("QueryResourceListPaging Error");
+ }
+ Result result = resourcesService.queryResourceListPaging(loginUser, "", "",
+ ResourceType.FILE, "Test", 1, 10);
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS.getCode(), (int) result.getCode());
PageInfo pageInfo = (PageInfo) result.getData();
@@ -410,30 +380,33 @@ public class ResourcesServiceTest {
loginUser.setId(0);
loginUser.setUserType(UserType.ADMIN_USER);
- Mockito.when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.RESOURCE_FILE_ID,
- 0, resourceLogger)).thenReturn(getSetIds());
- Mockito.when(resourcesMapper.selectBatchIds(Mockito.anySet())).thenReturn(getResourceList());
-
- Map<String, Object> result = resourcesService.queryResourceList(loginUser, ResourceType.FILE);
+ Mockito.when(userMapper.selectList(null)).thenReturn(Arrays.asList(loginUser));
+ Mockito.when(userMapper.selectById(loginUser.getId())).thenReturn(loginUser);
+ Mockito.when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(getTenant());
+ Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
+ Mockito.when(storageOperate.listFilesStatusRecursively("/dolphinscheduler/123/resources/",
+ "/dolphinscheduler/123/resources/",
+ "123",
+ ResourceType.FILE)).thenReturn(Arrays.asList(getStorageEntityResource()));
+ Map<String, Object> result = resourcesService.queryResourceList(loginUser, ResourceType.FILE, "");
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
List<Resource> resourceList = (List<Resource>) result.get(Constants.DATA_LIST);
Assertions.assertTrue(CollectionUtils.isNotEmpty(resourceList));
// test udf
- Mockito.when(resourcePermissionCheckService.userOwnedResourceIdsAcquisition(AuthorizationType.UDF_FILE, 0,
- resourceLogger)).thenReturn(getSetIds());
- Mockito.when(resourcesMapper.selectBatchIds(Mockito.anySet()))
- .thenReturn(Arrays.asList(getResource(11, ResourceType.UDF),
- getResource(10, ResourceType.UDF), getResource(9, ResourceType.UDF),
- getResource(8, ResourceType.UDF)));
-
+ Mockito.when(storageOperate.getUdfDir("123")).thenReturn("/dolphinscheduler/123/udfs/");
+ Mockito.when(storageOperate.listFilesStatusRecursively("/dolphinscheduler/123/udfs/",
+ "/dolphinscheduler/123/udfs/",
+ "123",
+ ResourceType.UDF))
+ .thenReturn(Arrays.asList(getStorageEntityUdfResource()));
loginUser.setUserType(UserType.GENERAL_USER);
- result = resourcesService.queryResourceList(loginUser, ResourceType.UDF);
+ result = resourcesService.queryResourceList(loginUser, ResourceType.UDF, "");
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS, result.get(Constants.STATUS));
resourceList = (List<Resource>) result.get(Constants.DATA_LIST);
- Assertions.assertTrue(resourceList.size() == 4);
+ Assertions.assertTrue(CollectionUtils.isNotEmpty(resourceList));
}
@Test
@@ -443,59 +416,34 @@ public class ResourcesServiceTest {
loginUser.setId(0);
loginUser.setUserType(UserType.GENERAL_USER);
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
- Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource());
- Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
-
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
- 0, ApiFuncIdentificationConstant.FILE_DELETE, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- new Object[]{1},
- 0, serviceLogger)).thenReturn(true);
try {
// HDFS_NOT_STARTUP
- Result result = resourcesService.delete(loginUser, 1);
+ Result result = resourcesService.delete(loginUser, "", "");
logger.info(result.toString());
Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg());
- // RESOURCE_NOT_EXIST
- Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
- Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource());
-
- Mockito.when(
- resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
- 0, ApiFuncIdentificationConstant.FILE_DELETE, serviceLogger))
- .thenReturn(true);
-
- result = resourcesService.delete(loginUser, 2);
- logger.info(result.toString());
- Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
-
- // USER_NO_OPERATION_PERM
- result = resourcesService.delete(loginUser, 2);
- logger.info(result.toString());
- Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
-
// TENANT_NOT_EXIST
+ Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
loginUser.setUserType(UserType.ADMIN_USER);
loginUser.setTenantId(2);
- Mockito.when(userMapper.selectById(Mockito.anyInt())).thenReturn(loginUser);
- Mockito.when(
- resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
- 0, ApiFuncIdentificationConstant.FILE_DELETE, serviceLogger))
- .thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- new Object[]{1},
- 0, serviceLogger)).thenReturn(true);
- result = resourcesService.delete(loginUser, 1);
+ Mockito.when(userMapper.selectById(loginUser.getId())).thenReturn(loginUser);
+ result = resourcesService.delete(loginUser, "", "");
logger.info(result.toString());
Assertions.assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), result.getMsg());
+ // RESOURCE_NOT_EXIST
+ Mockito.when(tenantMapper.queryById(Mockito.anyInt())).thenReturn(getTenant());
+ Mockito.when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest",
+ null, "123", null))
+ .thenReturn(getStorageEntityResource());
+ result = resourcesService.delete(loginUser, "/dolphinscheduler/123/resources/ResNotExist", "123");
+ logger.info(result.toString());
+ Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
+
// SUCCESS
loginUser.setTenantId(1);
- Mockito.when(processDefinitionMapper.listResources()).thenReturn(getResources());
- Mockito.when(resourcesMapper.deleteIds(Mockito.any())).thenReturn(1);
- Mockito.when(resourceUserMapper.deleteResourceUserArray(Mockito.anyInt(), Mockito.any())).thenReturn(1);
- result = resourcesService.delete(loginUser, 1);
+ result = resourcesService.delete(loginUser, "/dolphinscheduler/123/resources/ResourcesServiceTest",
+ "123");
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
@@ -508,29 +456,19 @@ public class ResourcesServiceTest {
@Test
public void testVerifyResourceName() {
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
- 1, ApiFuncIdentificationConstant.FILE_RENAME, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
- 1, serviceLogger)).thenReturn(true);
User user = new User();
user.setId(1);
user.setUserType(UserType.GENERAL_USER);
- Mockito.when(resourcesMapper.existResource("/ResourcesServiceTest.jar", 0)).thenReturn(true);
+ try {
+ Mockito.when(storageOperate.exists("/ResourcesServiceTest.jar")).thenReturn(true);
+ } catch (IOException e) {
+ logger.error("error occurred when checking resource: /ResourcesServiceTest.jar\"");
+ }
Result result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar", ResourceType.FILE, user);
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_EXIST.getMsg(), result.getMsg());
- // TENANT_NOT_EXIST
- Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
- String unExistFullName = "/test.jar";
-
- result = resourcesService.verifyResourceName("/test.jar", ResourceType.FILE, user);
- logger.info(result.toString());
- Assertions.assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), result.getMsg());
-
// RESOURCE_FILE_EXIST
- user.setTenantId(1);
-
result = resourcesService.verifyResourceName("/ResourcesServiceTest.jar", ResourceType.FILE, user);
logger.info(result.toString());
Assertions.assertTrue(Status.RESOURCE_EXIST.getCode() == result.getCode());
@@ -544,123 +482,89 @@ public class ResourcesServiceTest {
@Test
public void testReadResource() {
-
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
- 1, ApiFuncIdentificationConstant.FILE_VIEW, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- new Object[]{1},
- 1, serviceLogger)).thenReturn(true);
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
// HDFS_NOT_STARTUP
- Result result = resourcesService.readResource(getUser(), 1, 1, 10);
+ Result result = resourcesService.readResource(getUser(), "", "", 1, 10);
logger.info(result.toString());
Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg());
// RESOURCE_NOT_EXIST
- Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource());
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
- 1, ApiFuncIdentificationConstant.FILE_VIEW, serviceLogger)).thenReturn(true);
-
- result = resourcesService.readResource(getUser(), 2, 1, 10);
+ Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
+ Mockito.when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant());
+ result = resourcesService.readResource(getUser(), "", "", 1, 10);
logger.info(result.toString());
- Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
+ Assertions.assertEquals(Status.RESOURCE_FILE_NOT_EXIST.getCode(), (int) result.getCode());
// RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
- 1, ApiFuncIdentificationConstant.FILE_VIEW, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- new Object[]{1},
- 1, serviceLogger)).thenReturn(true);
Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("class");
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
- result = resourcesService.readResource(getUser(), 1, 1, 10);
+ result = resourcesService.readResource(getUser(), "", "", 1, 10);
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg());
// USER_NOT_EXIST
+ Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(null);
Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("jar");
Mockito.when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar");
- result = resourcesService.readResource(getUser(), 1, 1, 10);
+ result = resourcesService.readResource(getUser(), "", "", 1, 10);
logger.info(result.toString());
Assertions.assertEquals(Status.USER_NOT_EXIST.getCode(), (int) result.getCode());
// TENANT_NOT_EXIST
- Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
- result = resourcesService.readResource(getUser(), 1, 1, 10);
+ Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
+ Mockito.when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(null);
+ result = resourcesService.readResource(getUser(), "", "", 1, 10);
logger.info(result.toString());
Assertions.assertEquals(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getMsg(), result.getMsg());
- // RESOURCE_FILE_NOT_EXIST
- Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
-
- result = resourcesService.readResource(getUser(), 1, 1, 10);
- logger.info(result.toString());
- Assertions.assertEquals(Status.RESOURCE_FILE_NOT_EXIST.getCode(), (int) result.getCode());
-
// SUCCESS
+ Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
+ Mockito.when(tenantMapper.queryById(getUser().getTenantId())).thenReturn(getTenant());
try {
- Mockito.when(storageOperate.exists(Mockito.any(), Mockito.any())).thenReturn(true);
+ Mockito.when(storageOperate.exists(Mockito.any())).thenReturn(true);
Mockito.when(storageOperate.vimFile(Mockito.any(), Mockito.any(), eq(1), eq(10))).thenReturn(getContent());
} catch (IOException e) {
logger.error("storage error", e);
}
- result = resourcesService.readResource(getUser(), 1, 1, 10);
+ Mockito.when(Files.getFileExtension("test.jar")).thenReturn("jar");
+ result = resourcesService.readResource(getUser(), "test.jar", "", 1, 10);
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
-
}
@Test
public void testOnlineCreateResource() {
-
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
- 1, ApiFuncIdentificationConstant.FILE_ONLINE_CREATE, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
- 1, serviceLogger)).thenReturn(true);
-
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
User user = getUser();
user.setId(1);
+ Mockito.when(userMapper.selectById(user.getId())).thenReturn(getUser());
+ Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
+
// HDFS_NOT_STARTUP
Result result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content",
- -1, "/");
+ "/");
logger.info(result.toString());
Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg());
// RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("class");
- result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", -1,
+ result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content",
"/");
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg());
- // RuntimeException
- try {
- Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("jar");
- Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
- result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content",
- -1, "/");
- } catch (RuntimeException ex) {
- logger.info(result.toString());
- Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), ex.getMessage());
- }
-
// SUCCESS
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
- 1, ApiFuncIdentificationConstant.FILE_RENAME, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
- 1, serviceLogger)).thenReturn(true);
-
+ Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("jar");
+ Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test");
Mockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true);
- result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content", -1,
+ result = resourcesService.onlineCreateResource(user, ResourceType.FILE, "test", "jar", "desc", "content",
"/");
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
-
}
@Test
@@ -684,141 +588,138 @@ public class ResourcesServiceTest {
Resource dir2 = new Resource();
dir2.setFullName(resourceDir);
dir2.setUserId(user.getId());
- Mockito.when(resourcesMapper.queryResource(dir1.getFullName(), ResourceType.FILE.ordinal()))
- .thenReturn(Collections.singletonList(dir1));
- Mockito.when(resourcesMapper.queryResource(resourceDir, ResourceType.FILE.ordinal())).thenReturn(null);
-
- Tenant tenant = getTenant();
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(
- AuthorizationType.RESOURCE_FILE_ID, null, 1, ApiFuncIdentificationConstant.FOLDER_ONLINE_CREATE,
- serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(
- AuthorizationType.RESOURCE_FILE_ID, null, 1, serviceLogger)).thenReturn(true);
+
+ Mockito.when(storageOperate.getDir(ResourceType.FILE, "123")).thenReturn("/dolphinscheduler/123/resources/");
+ Mockito.when(storageOperate.getResDir("123")).thenReturn("/dolphinscheduler/123/resources/");
+ Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test");
+ Mockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true);
try {
- Mockito.when(storageOperate.mkdir(tenant.getTenantCode(), null)).thenReturn(true);
+ Mockito.when(storageOperate.mkdir("123", "/dolphinscheduler/123/resources" + dir1Path)).thenReturn(true);
+ Mockito.when(storageOperate.mkdir("123", "/dolphinscheduler/123/resources" + dir2Path)).thenReturn(true);
} catch (IOException e) {
- logger.error("storage error", e);
+ logger.error("create resource directory {} failed", fullName);
}
-
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(
- AuthorizationType.RESOURCE_FILE_ID, null, 1, ApiFuncIdentificationConstant.FILE_ONLINE_CREATE,
- serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(
- AuthorizationType.RESOURCE_FILE_ID, null, 1, serviceLogger)).thenReturn(true);
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(
- AuthorizationType.RESOURCE_FILE_ID, null, 1, ApiFuncIdentificationConstant.FILE_RENAME, serviceLogger))
- .thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(
- AuthorizationType.RESOURCE_FILE_ID, null, 1, serviceLogger)).thenReturn(true);
- Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
- Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test");
- Mockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true);
-
+ Mockito.when(userMapper.selectById(user.getId())).thenReturn(getUser());
+ Mockito.when(tenantMapper.queryById(user.getTenantId())).thenReturn(getTenant());
Result<Object> result = resourcesService.onlineCreateOrUpdateResourceWithDir(user, fullName, desc, content);
Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
- @Test
- public void testQueryResourcesFileInfo() {
- User user = getUser();
- String userName = "test-user";
- Mockito.when(userMapper.queryByUserNameAccurately(userName)).thenReturn(user);
- Resource file = new Resource();
- file.setFullName("/dir/file1.py");
- file.setId(1);
- Mockito.when(resourcesMapper.queryResource(file.getFullName(), ResourceType.FILE.ordinal()))
- .thenReturn(Collections.singletonList(file));
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(
- AuthorizationType.RESOURCE_FILE_ID, null, user.getId(), ApiFuncIdentificationConstant.FILE_VIEW,
- serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(
- AuthorizationType.RESOURCE_FILE_ID, new Object[]{file.getId()}, user.getId(), serviceLogger))
- .thenReturn(true);
- Resource result = resourcesService.queryResourcesFileInfo(userName, file.getFullName());
- Assertions.assertEquals(file.getFullName(), result.getFullName());
- }
+ // TODO: revise this testcase after modifying PythonGateway.java
+ // @Test
+ // public void testQueryResourcesFileInfo() {
+ // User user = getUser();
+ // String userName = "test-user";
+ // Mockito.when(userMapper.queryByUserNameAccurately(userName)).thenReturn(user);
+ // Resource file = new Resource();
+ // file.setFullName("/dir/file1.py");
+ // file.setId(1);
+ // Mockito.when(resourcesMapper.queryResource(file.getFullName(), ResourceType.FILE.ordinal()))
+ // .thenReturn(Collections.singletonList(file));
+ // Mockito.when(resourcePermissionCheckService.operationPermissionCheck(
+ // AuthorizationType.RESOURCE_FILE_ID, null, user.getId(), ApiFuncIdentificationConstant.FILE_VIEW,
+ // serviceLogger)).thenReturn(true);
+ // Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(
+ // AuthorizationType.RESOURCE_FILE_ID, new Object[]{file.getId()}, user.getId(), serviceLogger))
+ // .thenReturn(true);
+ // Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
+ // Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
+ // Resource result = resourcesService.queryResourcesFileInfo(userName, file.getFullName());
+ // Assertions.assertEquals(file.getFullName(), result.getFullName());
+ // }
@Test
public void testUpdateResourceContent() {
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
// HDFS_NOT_STARTUP
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
- 1, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- new Object[]{1},
- 1, serviceLogger)).thenReturn(true);
-
- Result result = resourcesService.updateResourceContent(getUser(), 1, "content");
+ Result result = resourcesService.updateResourceContent(getUser(), "", "", "content");
logger.info(result.toString());
Assertions.assertEquals(Status.STORAGE_NOT_STARTUP.getMsg(), result.getMsg());
// RESOURCE_NOT_EXIST
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
- 1, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true);
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
- Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource());
- result = resourcesService.updateResourceContent(getUser(), 2, "content");
+ Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
+ Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
+
+ try {
+ Mockito.when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest.jar",
+ "",
+ "123", ResourceType.FILE)).thenReturn(null);
+ } catch (Exception e) {
+ logger.error(e.getMessage() + " Resource path: {}", "", e);
+ }
+
+ result = resourcesService.updateResourceContent(getUser(),
+ "/dolphinscheduler/123/resources/ResourcesServiceTest.jar",
+ "123", "content");
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
// RESOURCE_SUFFIX_NOT_SUPPORT_VIEW
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
- 1, ApiFuncIdentificationConstant.FILE_UPDATE, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- new Object[]{1},
- 1, serviceLogger)).thenReturn(true);
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("class");
- result = resourcesService.updateResourceContent(getUser(), 1, "content");
+ try {
+ Mockito.when(storageOperate.getFileStatus("", "", "123", ResourceType.FILE))
+ .thenReturn(getStorageEntityResource());
+ } catch (Exception e) {
+ logger.error(e.getMessage() + " Resource path: {}", "", e);
+ }
+
+ result = resourcesService.updateResourceContent(getUser(), "", "123", "content");
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_SUFFIX_NOT_SUPPORT_VIEW.getMsg(), result.getMsg());
// USER_NOT_EXIST
- Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("jar");
- Mockito.when(Files.getFileExtension("ResourcesServiceTest.jar")).thenReturn("jar");
- result = resourcesService.updateResourceContent(getUser(), 1, "content");
+ Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(null);
+ result = resourcesService.updateResourceContent(getUser(), "", "123", "content");
logger.info(result.toString());
Assertions.assertTrue(Status.USER_NOT_EXIST.getCode() == result.getCode());
// TENANT_NOT_EXIST
- Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
- result = resourcesService.updateResourceContent(getUser(), 1, "content");
+ Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
+ Mockito.when(tenantMapper.queryById(1)).thenReturn(null);
+ result = resourcesService.updateResourceContent(getUser(), "", "123", "content");
logger.info(result.toString());
Assertions.assertTrue(Status.CURRENT_LOGIN_USER_TENANT_NOT_EXIST.getCode() == result.getCode());
// SUCCESS
+ try {
+ Mockito.when(storageOperate.getFileStatus("/dolphinscheduler/123/resources/ResourcesServiceTest.jar",
+ "",
+ "123", ResourceType.FILE)).thenReturn(getStorageEntityResource());
+ } catch (Exception e) {
+ logger.error(e.getMessage() + " Resource path: {}", "", e);
+ }
+
+ Mockito.when(Files.getFileExtension(Mockito.anyString())).thenReturn("jar");
+ Mockito.when(FileUtils.getResourceViewSuffixes()).thenReturn("jar");
+ Mockito.when(userMapper.selectById(getUser().getId())).thenReturn(getUser());
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
Mockito.when(FileUtils.getUploadFilename(Mockito.anyString(), Mockito.anyString())).thenReturn("test");
Mockito.when(FileUtils.writeContent2File(Mockito.anyString(), Mockito.anyString())).thenReturn(true);
- result = resourcesService.updateResourceContent(getUser(), 1, "content");
+ result = resourcesService.updateResourceContent(getUser(),
+ "/dolphinscheduler/123/resources/ResourcesServiceTest.jar",
+ "123", "content");
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
@Test
public void testDownloadResource() {
-
- Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.RESOURCE_FILE_ID, null,
- 1, ApiFuncIdentificationConstant.FILE_DOWNLOAD, serviceLogger)).thenReturn(true);
- Mockito.when(resourcePermissionCheckService.resourcePermissionCheck(AuthorizationType.RESOURCE_FILE_ID,
- new Object[]{1},
- 1, serviceLogger)).thenReturn(true);
-
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
Mockito.when(tenantMapper.queryById(1)).thenReturn(getTenant());
Mockito.when(userMapper.selectById(1)).thenReturn(getUser());
org.springframework.core.io.Resource resourceMock = Mockito.mock(org.springframework.core.io.Resource.class);
try {
// resource null
- org.springframework.core.io.Resource resource = resourcesService.downloadResource(getUser(), 1);
+ org.springframework.core.io.Resource resource = resourcesService.downloadResource(getUser(), "");
Assertions.assertNull(resource);
- Mockito.when(resourcesMapper.selectById(1)).thenReturn(getResource());
Mockito.when(org.apache.dolphinscheduler.api.utils.FileUtils.file2Resource(Mockito.any()))
.thenReturn(resourceMock);
- resource = resourcesService.downloadResource(getUser(), 1);
+ resource = resourcesService.downloadResource(getUser(), "");
Assertions.assertNotNull(resource);
} catch (Exception e) {
logger.error("DownloadResource error", e);
@@ -1016,7 +917,6 @@ public class ResourcesServiceTest {
}
private Resource getResource() {
-
Resource resource = new Resource();
resource.setPid(-1);
resource.setUserId(1);
@@ -1027,6 +927,19 @@ public class ResourcesServiceTest {
return resource;
}
+ private StorageEntity getStorageEntityResource() {
+ StorageEntity entity = new StorageEntity();
+ entity.setAlias("ResourcesServiceTest");
+ entity.setFileName("ResourcesServiceTest");
+ entity.setDirectory(false);
+ entity.setDescription("");
+ entity.setUserName("123");
+ entity.setType(ResourceType.FILE);
+ entity.setFullName("/dolphinscheduler/123/resources/ResourcesServiceTest");
+
+ return entity;
+ }
+
private Resource getResource(int resourceId) {
Resource resource = new Resource();
@@ -1064,6 +977,19 @@ public class ResourcesServiceTest {
return resource;
}
+ private StorageEntity getStorageEntityUdfResource() {
+ StorageEntity entity = new StorageEntity();
+ entity.setAlias("ResourcesServiceTest1.jar");
+ entity.setFileName("ResourcesServiceTest1.jar");
+ entity.setDirectory(false);
+ entity.setDescription("");
+ entity.setUserName("123");
+ entity.setType(ResourceType.UDF);
+ entity.setFullName("/dolphinscheduler/123/resources/ResourcesServiceTest1.jar");
+
+ return entity;
+ }
+
private UdfFunc getUdfFunc() {
UdfFunc udfFunc = new UdfFunc();
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java
index 0607d95c9c..32f24a69f9 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/TenantServiceTest.java
@@ -40,6 +40,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.TenantMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
+import org.apache.dolphinscheduler.service.storage.StorageOperate;
import org.apache.commons.collections.CollectionUtils;
@@ -96,6 +97,9 @@ public class TenantServiceTest {
@Mock
private ResourcePermissionCheckService resourcePermissionCheckService;
+ @Mock
+ private StorageOperate storageOperate;
+
private static final String tenantCode = "hayden";
private static final String tenantDesc = "This is the tenant desc";
private static final String queue = "queue";
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java
index 29ce54afad..9f12cc23c2 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/service/UdfFuncServiceTest.java
@@ -34,9 +34,11 @@ import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
import org.apache.dolphinscheduler.dao.mapper.UDFUserMapper;
import org.apache.dolphinscheduler.dao.mapper.UdfFuncMapper;
+import org.apache.dolphinscheduler.service.storage.StorageOperate;
import org.apache.commons.collections.CollectionUtils;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
@@ -85,6 +87,9 @@ public class UdfFuncServiceTest {
@Mock
private UDFUserMapper udfUserMapper;
+ @Mock
+ private StorageOperate storageOperate;
+
@BeforeEach
public void setUp() {
mockedStaticPropertyUtils = Mockito.mockStatic(PropertyUtils.class);
@@ -108,21 +113,27 @@ public class UdfFuncServiceTest {
// hdfs not start
Result result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest",
"org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String",
- "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, Integer.MAX_VALUE);
+ "UdfFuncServiceTest", "UdfFuncServiceTest", "", UdfType.HIVE);
logger.info(result.toString());
Assertions.assertEquals(Status.HDFS_NOT_STARTUP.getMsg(), result.getMsg());
// resource not exist
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
+
result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest",
"org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String",
- "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, Integer.MAX_VALUE);
+ "UdfFuncServiceTest", "UdfFuncServiceTest", "", UdfType.HIVE);
logger.info(result.toString());
Assertions.assertEquals(Status.RESOURCE_NOT_EXIST.getMsg(), result.getMsg());
// success
- Mockito.when(resourceMapper.selectById(1)).thenReturn(getResource());
+ try {
+ Mockito.when(storageOperate.exists("String")).thenReturn(true);
+ } catch (IOException e) {
+ logger.error("AmazonServiceException when checking resource: String");
+ }
+
result = udfFuncService.createUdfFunction(getLoginUser(), "UdfFuncServiceTest",
"org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String",
- "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1);
+ "UdfFuncServiceTest", "UdfFuncServiceTest", "", UdfType.HIVE);
logger.info(result.toString());
Assertions.assertEquals(Status.SUCCESS.getMsg(), result.getMsg());
}
@@ -154,7 +165,6 @@ public class UdfFuncServiceTest {
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(false);
Mockito.when(udfFuncMapper.selectUdfById(1)).thenReturn(getUdfFunc());
- Mockito.when(resourceMapper.selectById(1)).thenReturn(getResource());
// UDF_FUNCTION_NOT_EXIST
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.UDF, null, 1,
@@ -163,7 +173,7 @@ public class UdfFuncServiceTest {
serviceLogger)).thenReturn(true);
Result<Object> result = udfFuncService.updateUdfFunc(getLoginUser(), 12, "UdfFuncServiceTest",
"org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String",
- "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1);
+ "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, "");
logger.info(result.toString());
Assertions.assertTrue(Status.UDF_FUNCTION_NOT_EXIST.getCode() == result.getCode());
@@ -172,7 +182,7 @@ public class UdfFuncServiceTest {
serviceLogger)).thenReturn(true);
result = udfFuncService.updateUdfFunc(getLoginUser(), 1, "UdfFuncServiceTest",
"org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String",
- "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1);
+ "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, "");
logger.info(result.toString());
Assertions.assertTrue(Status.HDFS_NOT_STARTUP.getCode() == result.getCode());
@@ -185,16 +195,22 @@ public class UdfFuncServiceTest {
Mockito.when(PropertyUtils.getResUploadStartupState()).thenReturn(true);
result = udfFuncService.updateUdfFunc(getLoginUser(), 11, "UdfFuncServiceTest",
"org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String",
- "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 12);
+ "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, "");
logger.info(result.toString());
Assertions.assertTrue(Status.RESOURCE_NOT_EXIST.getCode() == result.getCode());
// success
Mockito.when(resourcePermissionCheckService.operationPermissionCheck(AuthorizationType.UDF, null, 1,
ApiFuncIdentificationConstant.UDF_FUNCTION_UPDATE, serviceLogger)).thenReturn(true);
+ try {
+ Mockito.when(storageOperate.exists("")).thenReturn(true);
+ } catch (IOException e) {
+ logger.error("AmazonServiceException when checking resource: ");
+ }
+
result = udfFuncService.updateUdfFunc(getLoginUser(), 11, "UdfFuncServiceTest",
"org.apache.dolphinscheduler.api.service.UdfFuncServiceTest", "String",
- "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, 1);
+ "UdfFuncServiceTest", "UdfFuncServiceTest", UdfType.HIVE, "");
logger.info(result.toString());
Assertions.assertTrue(Status.SUCCESS.getCode() == result.getCode());
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesTask.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesTask.java
new file mode 100644
index 0000000000..799aa3e901
--- /dev/null
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResourcesTask.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.dolphinscheduler.dao.entity;
+
+import org.apache.dolphinscheduler.spi.enums.ResourceType;
+
+import lombok.Data;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+
+@Data
+@TableName("t_ds_relation_resources_task")
+public class ResourcesTask {
+
+ @TableId(value = "id", type = IdType.AUTO)
+ private Integer id;
+
+ private String fullName;
+
+ private int taskId;
+
+ private ResourceType type;
+
+ public ResourcesTask(int id, String fullName, int taskId, ResourceType type) {
+ this.id = id;
+ this.fullName = fullName;
+ this.taskId = taskId;
+ this.type = type;
+ }
+
+ public ResourcesTask(int taskId, String fullName, ResourceType type) {
+ this.taskId = taskId;
+ this.fullName = fullName;
+ this.type = type;
+ }
+}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java
index 45c70cb723..c90d4ea068 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceMapper.java
@@ -54,7 +54,6 @@ public interface ResourceMapper extends BaseMapper<Resource> {
/**
* resource page
* @param page page
- * @param userId userId
* @param id id
* @param type type
* @param searchVal searchVal
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceTaskMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceTaskMapper.java
new file mode 100644
index 0000000000..bdd1542c49
--- /dev/null
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/ResourceTaskMapper.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.dolphinscheduler.dao.mapper;
+
+import org.apache.dolphinscheduler.dao.entity.ResourcesTask;
+
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+/**
+ * resource task relation mapper interface
+ */
+public interface ResourceTaskMapper extends BaseMapper<ResourcesTask> {
+
+ Integer existResourceByTaskIdNFullName(@Param("taskId") int task_id, @Param("fullName") String fullName);
+
+ int deleteIds(@Param("resIds") Integer[] resIds);
+
+ int updateResource(@Param("id") int id, @Param("fullName") String fullName);
+
+ List<ResourcesTask> selectBatchFullNames(@Param("fullNameArr") String[] fullNameArr);
+
+ List<ResourcesTask> selectSubfoldersFullNames(@Param("folderPath") String folderPath);
+}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.java
index eb3d1f7e40..629cd9bf7b 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.java
@@ -75,7 +75,7 @@ public interface TaskDefinitionMapper extends BaseMapper<TaskDefinition> {
List<DefinitionGroupByUser> countDefinitionGroupByUser(@Param("projectCodes") Long[] projectCodes);
/**
- * list all resource ids
+ * list all resource ids and task_params containing resourceList
*
* @return task ids list
*/
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java
index 5573aedd8d..6bc8049c7d 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.java
@@ -49,7 +49,7 @@ public interface UdfFuncMapper extends BaseMapper<UdfFunc> {
/**
* udf function page
* @param page page
- * @param userId userId
+ * @param ids userId
* @param searchVal searchVal
* @return udf function IPage
*/
@@ -59,7 +59,7 @@ public interface UdfFuncMapper extends BaseMapper<UdfFunc> {
/**
* query udf function by type
- * @param userId userId
+ * @param ids userId
* @param type type
* @return udf function list
*/
@@ -95,6 +95,13 @@ public interface UdfFuncMapper extends BaseMapper<UdfFunc> {
*/
List<UdfFunc> listUdfByResourceId(@Param("resourceIds") Integer[] resourceIds);
+ /**
+ * list UDF by resource fullName
+ * @param resourceFullNames resource fullName array
+ * @return UDF function list
+ */
+ List<UdfFunc> listUdfByResourceFullName(@Param("resourceFullNames") String[] resourceFullNames);
+
/**
* list authorized UDF by resource id
* @param resourceIds resource id array
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java
index 69d9788b32..91484a476f 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/utils/ResourceProcessDefinitionUtils.java
@@ -66,4 +66,34 @@ public class ResourceProcessDefinitionUtils {
return resourceResult;
}
+
+ public static <T> Map<Integer, Set<T>> getResourceObjectMap(List<Map<String, Object>> resourceList,
+ String objectName, Class<T> clazz) {
+ // resourceId -> task ids or code depends on the objectName
+ Map<Integer, Set<T>> resourceResult = new HashMap<>();
+
+ if (CollectionUtils.isNotEmpty(resourceList)) {
+ for (Map<String, Object> resourceMap : resourceList) {
+ // get resName and resource_id_news, td_id
+ T taskId = (T) resourceMap.get(objectName);
+ String[] resourceIds = ((String) resourceMap.get("resource_ids"))
+ .split(",");
+
+ Set<Integer> resourceIdSet =
+ Arrays.stream(resourceIds).map(Integer::parseInt).collect(Collectors.toSet());
+ for (Integer resourceId : resourceIdSet) {
+ Set<T> codeSet;
+ if (resourceResult.containsKey(resourceId)) {
+ codeSet = resourceResult.get(resourceId);
+ } else {
+ codeSet = new HashSet<>();
+ }
+ codeSet.add(taskId);
+ resourceResult.put(resourceId, codeSet);
+ }
+ }
+ }
+
+ return resourceResult;
+ }
}
diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml
index a69150b3f7..f654b0bd77 100644
--- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml
+++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ProcessDefinitionMapper.xml
@@ -168,7 +168,7 @@
</select>
<select id="listResources" resultType="java.util.HashMap">
- SELECT distinct pd.code,td.resource_ids
+ SELECT distinct pd.code, td.id as td_id, td.resource_ids
FROM t_ds_process_task_relation ptr
join t_ds_process_definition pd
on ptr.process_definition_code=pd.code and ptr.process_definition_version = pd.version
@@ -180,7 +180,7 @@
</select>
<select id="listResourcesByUser" resultType="java.util.HashMap">
- SELECT distinct pd.code,td.resource_ids
+ SELECT distinct pd.code, td.id as td_id, td.resource_ids
FROM t_ds_process_task_relation ptr
join t_ds_process_definition pd
on ptr.process_definition_code=pd.code and ptr.process_definition_version = pd.version
diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceTaskMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceTaskMapper.xml
new file mode 100644
index 0000000000..f5131405de
--- /dev/null
+++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/ResourceTaskMapper.xml
@@ -0,0 +1,58 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+<mapper namespace="org.apache.dolphinscheduler.dao.mapper.ResourceTaskMapper">
+ <sql id="baseSqlV2">
+ ${alias}.id, ${alias}.full_name, ${alias}.task_id, ${alias}.type
+ </sql>
+ <select id="existResourceByTaskIdNFullName" resultType="java.lang.Integer">
+ select
+ id
+ from t_ds_relation_resources_task
+ where full_name = #{fullName} and task_id = #{taskId}
+ </select>
+
+ <select id="selectBatchFullNames" resultType="org.apache.dolphinscheduler.dao.entity.ResourcesTask">
+ select
+ id, full_name, task_id, type
+ from t_ds_relation_resources_task
+ where full_name in
+ <foreach collection="fullNameArr" item="i" open="(" close=")" separator=",">
+ #{i}
+ </foreach>
+ </select>
+
+ <update id="updateResource" >
+ UPDATE t_ds_relation_resources_task SET full_name=#{fullName} WHERE id=#{id}
+ </update>
+
+ <delete id="deleteIds" parameterType="java.lang.Integer">
+ delete from t_ds_relation_resources_task where id in
+ <foreach collection="resIds" item="i" open="(" close=")" separator=",">
+ #{i}
+ </foreach>
+ </delete>
+
+ <select id="selectSubfoldersFullNames" resultType="org.apache.dolphinscheduler.dao.entity.ResourcesTask">
+ select
+ id, full_name, task_id, type
+ from t_ds_relation_resources_task
+ where full_name like concat(#{folderPath}, '%')
+ </select>
+</mapper>
\ No newline at end of file
diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.xml
index 9c7d6a7b59..2c6a5cbe8a 100644
--- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.xml
+++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskDefinitionMapper.xml
@@ -96,8 +96,8 @@
#{taskDefinition.projectCode},#{taskDefinition.userId},#{taskDefinition.taskType},#{taskDefinition.taskParams},#{taskDefinition.flag},
#{taskDefinition.taskPriority},#{taskDefinition.workerGroup},#{taskDefinition.environmentCode},#{taskDefinition.failRetryTimes},
#{taskDefinition.failRetryInterval},#{taskDefinition.timeoutFlag},#{taskDefinition.timeoutNotifyStrategy},#{taskDefinition.timeout},
- #{taskDefinition.delayTime},#{taskDefinition.resourceIds},#{taskDefinition.createTime},#{taskDefinition.updateTime}, #{taskDefinition.taskGroupId},
- #{taskDefinition.taskExecuteType})
+ #{taskDefinition.delayTime},#{taskDefinition.resourceIds},#{taskDefinition.createTime},#{taskDefinition.updateTime},
+ #{taskDefinition.taskGroupId}, #{taskDefinition.taskExecuteType})
</foreach>
</insert>
<select id="queryDefineListPaging" resultType="org.apache.dolphinscheduler.dao.entity.TaskMainInfo">
diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml
index a981b3026b..24cb8de956 100644
--- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml
+++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/UdfFuncMapper.xml
@@ -141,6 +141,20 @@
</foreach>
</if>
</select>
+ <select id="listUdfByResourceFullName" resultType="org.apache.dolphinscheduler.dao.entity.UdfFunc">
+ select
+ <include refid="baseSql">
+ <property name="alias" value="udf"/>
+ </include>
+ from t_ds_udfs udf
+ where 1=1
+ <if test="resourceFullNames != null and resourceFullNames.length > 0">
+ and udf.resource_name in
+ <foreach collection="resourceFullNames" item="i" open="(" close=")" separator=",">
+ #{i}
+ </foreach>
+ </if>
+ </select>
<select id="listAuthorizedUdfByResourceId" resultType="org.apache.dolphinscheduler.dao.entity.UdfFunc">
select
<include refid="baseSql">
diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql
index a1eb4a82d1..d396b5c5e8 100644
--- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql
+++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_h2.sql
@@ -793,6 +793,24 @@ CREATE TABLE t_ds_resources
-- Records of t_ds_resources
-- ----------------------------
+-- ----------------------------
+-- Table structure for t_ds_relation_resources_task
+-- ----------------------------
+DROP TABLE IF EXISTS t_ds_relation_resources_task CASCADE;
+CREATE TABLE t_ds_relation_resources_task
+(
+ id int(11) NOT NULL AUTO_INCREMENT,
+ task_id int(11) DEFAULT NULL,
+ full_name varchar(255) DEFAULT NULL,
+ type tinyint(4) DEFAULT NULL,
+ PRIMARY KEY (id),
+ UNIQUE KEY t_ds_relation_resources_task_un (task_id, full_name)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
+
+-- ----------------------------
+-- Records of t_ds_relation_resources_task
+-- ----------------------------
+
-- ----------------------------
-- Table structure for t_ds_schedules
-- ----------------------------
diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql
index 8ce188c945..9c690ddb45 100644
--- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql
+++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_mysql.sql
@@ -791,6 +791,23 @@ CREATE TABLE `t_ds_resources` (
-- Records of t_ds_resources
-- ----------------------------
+-- ----------------------------
+-- Table structure for t_ds_relation_resources_task
+-- ----------------------------
+DROP TABLE IF EXISTS `t_ds_relation_resources_task`;
+CREATE TABLE `t_ds_relation_resources_task` (
+ `id` int NOT NULL AUTO_INCREMENT COMMENT 'key',
+ `task_id` int(11) DEFAULT NULL COMMENT 'task id',
+ `full_name` varchar(255) DEFAULT NULL,
+ `type` tinyint DEFAULT NULL COMMENT 'resource type,0:FILE,1:UDF',
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `t_ds_relation_resources_task_un` (`task_id`, `full_name`)
+) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
+
+-- ----------------------------
+-- Records of t_ds_relation_resources_task
+-- ----------------------------
+
-- ----------------------------
-- Table structure for t_ds_schedules
-- ----------------------------
diff --git a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql
index a818a8082a..89f3d06a74 100644
--- a/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql
+++ b/dolphinscheduler-dao/src/main/resources/sql/dolphinscheduler_postgresql.sql
@@ -700,6 +700,18 @@ CREATE TABLE t_ds_resources (
CONSTRAINT t_ds_resources_un UNIQUE (full_name, type)
) ;
+--
+-- Table structure for table t_ds_relation_resources_task
+--
+DROP TABLE IF EXISTS t_ds_relation_resources_task;
+CREATE TABLE t_ds_relation_resources_task (
+ id SERIAL NOT NULL,
+ task_id int DEFAULT NULL,
+ full_name varchar(255) DEFAULT NULL,
+ type int DEFAULT NULL,
+ PRIMARY KEY (id),
+ CONSTRAINT t_ds_relation_resources_task_un UNIQUE (task_id, full_name)
+);
--
-- Table structure for table t_ds_schedules
diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
index ff39833142..31050fb163 100644
--- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
+++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/runner/task/BaseTaskProcessor.java
@@ -44,7 +44,6 @@ import org.apache.dolphinscheduler.dao.entity.DqRule;
import org.apache.dolphinscheduler.dao.entity.DqRuleExecuteSql;
import org.apache.dolphinscheduler.dao.entity.DqRuleInputEntry;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
-import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.Tenant;
import org.apache.dolphinscheduler.dao.entity.UdfFunc;
@@ -92,9 +91,6 @@ import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Properties;
-import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
import lombok.NonNull;
@@ -624,26 +620,9 @@ public abstract class BaseTaskProcessor implements ITaskProcessor {
if (baseParam != null) {
List<ResourceInfo> projectResourceFiles = baseParam.getResourceFilesList();
if (CollectionUtils.isNotEmpty(projectResourceFiles)) {
-
- // filter the resources that the resource id equals 0
- Set<ResourceInfo> oldVersionResources =
- projectResourceFiles.stream().filter(t -> t.getId() == null).collect(Collectors.toSet());
- if (CollectionUtils.isNotEmpty(oldVersionResources)) {
- oldVersionResources.forEach(t -> resourcesMap.put(t.getRes(),
- processService.queryTenantCodeByResName(t.getRes(), ResourceType.FILE)));
- }
-
- // get the resource id in order to get the resource names in batch
- Stream<Integer> resourceIdStream = projectResourceFiles.stream().map(ResourceInfo::getId);
- Set<Integer> resourceIdsSet = resourceIdStream.collect(Collectors.toSet());
-
- if (CollectionUtils.isNotEmpty(resourceIdsSet)) {
- Integer[] resourceIds = resourceIdsSet.toArray(new Integer[resourceIdsSet.size()]);
-
- List<Resource> resources = processService.listResourceByIds(resourceIds);
- resources.forEach(t -> resourcesMap.put(t.getFullName(),
- processService.queryTenantCodeByResName(t.getFullName(), ResourceType.FILE)));
- }
+ // TODO: Modify this part to accomodate(migrate) oldversionresources in the future.
+ projectResourceFiles.forEach(file -> resourcesMap.put(file.getResourceName(),
+ processService.queryTenantCodeByResName(file.getResourceName(), ResourceType.FILE)));
}
}
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
index ea3f5fd7a1..5c6a149657 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/process/ProcessServiceImpl.java
@@ -69,6 +69,7 @@ import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
import org.apache.dolphinscheduler.dao.entity.Project;
import org.apache.dolphinscheduler.dao.entity.ProjectUser;
import org.apache.dolphinscheduler.dao.entity.Resource;
+import org.apache.dolphinscheduler.dao.entity.ResourcesTask;
import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
@@ -97,6 +98,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationLogMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
+import org.apache.dolphinscheduler.dao.mapper.ResourceTaskMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceUserMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
@@ -142,9 +144,11 @@ import org.apache.dolphinscheduler.spi.enums.ResourceType;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
@@ -229,6 +233,9 @@ public class ProcessServiceImpl implements ProcessService {
@Autowired
private ResourceMapper resourceMapper;
+ @Autowired
+ private ResourceTaskMapper resourceTaskMapper;
+
@Autowired
private ResourceUserMapper resourceUserMapper;
@@ -1376,7 +1383,7 @@ public class ProcessServiceImpl implements ProcessService {
ResourceInfo mainJar = JSONUtils.parseObject(
JSONUtils.toJsonString(mainJarObj),
ResourceInfo.class);
- ResourceInfo resourceInfo = updateResourceInfo(mainJar);
+ ResourceInfo resourceInfo = updateResourceInfo(taskDefinition.getId(), mainJar);
if (resourceInfo != null) {
taskParameters.put("mainJar", resourceInfo);
}
@@ -1387,7 +1394,7 @@ public class ProcessServiceImpl implements ProcessService {
List<ResourceInfo> resourceInfos = JSONUtils.toList(resourceListStr, ResourceInfo.class);
List<ResourceInfo> updatedResourceInfos = resourceInfos
.stream()
- .map(this::updateResourceInfo)
+ .map(resourceInfo -> updateResourceInfo(taskDefinition.getId(), resourceInfo))
.filter(Objects::nonNull)
.collect(Collectors.toList());
taskParameters.put("resourceList", updatedResourceInfos);
@@ -1403,21 +1410,23 @@ public class ProcessServiceImpl implements ProcessService {
* @param res origin resource info
* @return {@link ResourceInfo}
*/
- protected ResourceInfo updateResourceInfo(ResourceInfo res) {
+ protected ResourceInfo updateResourceInfo(int task_id, ResourceInfo res) {
ResourceInfo resourceInfo = null;
- // only if mainJar is not null and does not contains "resourceName" field
+ // only if mainJar is not null and does not contain "resourceName" field
if (res != null) {
- Integer resourceId = res.getId();
- if (resourceId == null) {
- logger.error("invalid resourceId, {}", resourceId);
- return null;
+ String resourceFullName = res.getResourceName();
+ if (StringUtils.isBlank(resourceFullName)) {
+ logger.error("invalid resource full name, {}", resourceFullName);
+ return new ResourceInfo();
}
resourceInfo = new ResourceInfo();
// get resource from database, only one resource should be returned
- Resource resource = getResourceById(resourceId);
- resourceInfo.setId(resourceId);
- resourceInfo.setRes(resource.getFileName());
- resourceInfo.setResourceName(resource.getFullName());
+ Integer resultList = resourceTaskMapper.existResourceByTaskIdNFullName(task_id, resourceFullName);
+ if (resultList != null) {
+ resourceInfo.setId(resultList);
+ resourceInfo.setRes(res.getRes());
+ resourceInfo.setResourceName(resourceFullName);
+ }
if (logger.isInfoEnabled()) {
logger.info("updated resource info {}",
JSONUtils.toJsonString(resourceInfo));
@@ -2033,7 +2042,6 @@ public class ProcessServiceImpl implements ProcessService {
taskDefinitionLog.setUpdateTime(now);
taskDefinitionLog.setOperateTime(now);
taskDefinitionLog.setOperator(operator.getId());
- taskDefinitionLog.setResourceIds(getResourceIds(taskDefinitionLog));
if (taskDefinitionLog.getCode() == 0) {
taskDefinitionLog.setCode(CodeGenerateUtils.getInstance().genCode());
}
@@ -2074,6 +2082,8 @@ public class ProcessServiceImpl implements ProcessService {
TaskDefinition task = taskDefinitionMap.get(taskDefinitionToUpdate.getCode());
if (task == null) {
newTaskDefinitionLogs.add(taskDefinitionToUpdate);
+ } else {
+ taskDefinitionToUpdate.setId(task.getId());
}
}
}
@@ -2091,9 +2101,42 @@ public class ProcessServiceImpl implements ProcessService {
if (CollectionUtils.isNotEmpty(newTaskDefinitionLogs) && Boolean.TRUE.equals(syncDefine)) {
updateResult += taskDefinitionMapper.batchInsert(newTaskDefinitionLogs);
+
+ for (TaskDefinitionLog newTaskDefinitionLog : newTaskDefinitionLogs) {
+ Set<String> resourceFullNameSet = getResourceFullNames(newTaskDefinitionLog);
+ for (String resourceFullName : resourceFullNameSet) {
+ List<TaskDefinition> taskDefinitionList = taskDefinitionMapper.selectByMap(
+ Collections.singletonMap("code", newTaskDefinitionLog.getCode()));
+ if (taskDefinitionList.size() > 0) {
+ createRelationTaskResourcesIfNotExist(
+ taskDefinitionList.get(0).getId(), resourceFullName);
+ }
+ }
+ }
+
}
if (CollectionUtils.isNotEmpty(updateTaskDefinitionLogs) && Boolean.TRUE.equals(syncDefine)) {
for (TaskDefinitionLog taskDefinitionLog : updateTaskDefinitionLogs) {
+ Set<String> resourceFullNameSet = getResourceFullNames(taskDefinitionLog);
+
+ // remove resources that user deselected.
+ for (ResourcesTask resourcesTask : resourceTaskMapper.selectByMap(
+ Collections.singletonMap("task_id",
+ taskDefinitionMapper.queryByCode(taskDefinitionLog.getCode()).getId()))) {
+ if (!resourceFullNameSet.contains(resourcesTask.getFullName())) {
+ resourceTaskMapper.deleteById(resourcesTask.getId());
+ }
+ }
+
+ for (String resourceFullName : resourceFullNameSet) {
+ List<TaskDefinition> taskDefinitionList = taskDefinitionMapper.selectByMap(
+ Collections.singletonMap("code", taskDefinitionLog.getCode()));
+ if (taskDefinitionList.size() > 0) {
+ createRelationTaskResourcesIfNotExist(
+ taskDefinitionList.get(0).getId(), resourceFullName);
+ }
+ }
+
updateResult += taskDefinitionMapper.updateById(taskDefinitionLog);
}
}
@@ -2682,4 +2725,36 @@ public class ProcessServiceImpl implements ProcessService {
return testDataSourceId;
return null;
}
+
+ private Set<String> getResourceFullNames(TaskDefinition taskDefinition) {
+ Set<String> resourceFullNames = null;
+ AbstractParameters params = taskPluginManager.getParameters(ParametersNode.builder()
+ .taskType(taskDefinition.getTaskType()).taskParams(taskDefinition.getTaskParams()).build());
+
+ if (params != null && CollectionUtils.isNotEmpty(params.getResourceFilesList())) {
+ resourceFullNames = params.getResourceFilesList().stream()
+ .filter(t -> !StringUtils.isBlank(t.getResourceName()))
+ .map(ResourceInfo::getResourceName)
+ .collect(toSet());
+ }
+
+ if (CollectionUtils.isEmpty(resourceFullNames)) {
+ return new HashSet<String>();
+ }
+
+ return resourceFullNames;
+ }
+
+ private Integer createRelationTaskResourcesIfNotExist(int taskId, String resourceFullName) {
+
+ Integer resourceId = resourceTaskMapper.existResourceByTaskIdNFullName(taskId, resourceFullName);
+ if (null == resourceId) {
+ // create the relation if not exist
+ ResourcesTask resourcesTask = new ResourcesTask(taskId, resourceFullName, ResourceType.FILE);
+ resourceTaskMapper.insert(resourcesTask);
+ return resourcesTask.getId();
+ }
+
+ return resourceId;
+ }
}
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/StorageEntity.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/StorageEntity.java
new file mode 100644
index 0000000000..e8ae58df48
--- /dev/null
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/StorageEntity.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.dolphinscheduler.service.storage;
+
+import org.apache.dolphinscheduler.spi.enums.ResourceType;
+
+import java.util.Date;
+
+import lombok.Data;
+
+// StorageEneity is an entity representing a resource in the third-part storage service.
+// It is only stored in t_ds_relation_resources_task if the resource is used by a task.
+// It is not put in the model module because it has more attributes than corresponding objects stored
+// in table t_ds_relation_resources_task.
+
+@Data
+public class StorageEntity {
+
+ /**
+ * exist only if it is stored in t_ds_relation_resources_task.
+ *
+ */
+ private int id;
+ /**
+ * fullname is in a format of basepath + tenantCode + res/udf + filename
+ */
+ private String fullName;
+ /**
+ * filename is in a format of possible parent folders + alias
+ */
+ private String fileName;
+ /**
+ * the name of the file
+ */
+ private String alias;
+ /**
+ * parent folder time
+ */
+ private String pfullName;
+ private boolean isDirectory;
+ private String description;
+ private int userId;
+ private String userName;
+ private ResourceType type;
+ private long size;
+ private Date createTime;
+ private Date updateTime;
+}
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/StorageOperate.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/StorageOperate.java
index 358030161f..e194811291 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/StorageOperate.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/StorageOperate.java
@@ -68,6 +68,13 @@ public interface StorageOperate {
*/
String getResourceFileName(String tenantCode, String fullName);
+ /**
+ * get the path of the resource file excluding the base path.
+ * @param fullName
+ * @return
+ */
+ String getResourceFileName(String fullName);
+
/**
* get the path of the file
* @param resourceType
@@ -79,23 +86,23 @@ public interface StorageOperate {
/**
* predicate if the resource of tenant exists
- * @param tenantCode
- * @param fileName
+ * @param fullName
* @return
* @throws IOException
*/
- boolean exists(String tenantCode, String fileName) throws IOException;
+ boolean exists(String fullName) throws IOException;
/**
* delete the resource of filePath
* todo if the filePath is the type of directory ,the files in the filePath need to be deleted at all
- * @param tenantCode
* @param filePath
* @param recursive
* @return
* @throws IOException
*/
- boolean delete(String tenantCode, String filePath, boolean recursive) throws IOException;
+ boolean delete(String filePath, boolean recursive) throws IOException;
+
+ boolean delete(String filePath, List<String> childrenPathArray, boolean recursive) throws IOException;
/**
* copy the file from srcPath to dstPath
@@ -167,4 +174,21 @@ public interface StorageOperate {
*/
ResUploadType returnStorageType();
+ /**
+ * return files and folders in the current directory and subdirectories
+ * */
+ public List<StorageEntity> listFilesStatusRecursively(String path, String defaultPath, String tenantCode,
+ ResourceType type);
+
+ /**
+ * return files and folders in the current directory
+ * */
+ public List<StorageEntity> listFilesStatus(String path, String defaultPath, String tenantCode,
+ ResourceType type) throws Exception;
+
+ /**
+ * return a file status
+ * */
+ public StorageEntity getFileStatus(String path, String defaultPath, String tenantCode,
+ ResourceType type) throws Exception;
}
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtils.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtils.java
index 0eef3f4c43..b0e0614a42 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtils.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/HadoopUtils.java
@@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.KerberosHttpClient;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
+import org.apache.dolphinscheduler.service.storage.StorageEntity;
import org.apache.dolphinscheduler.service.storage.StorageOperate;
import org.apache.dolphinscheduler.service.utils.CommonUtils;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
@@ -48,12 +49,17 @@ import org.apache.hadoop.security.UserGroupInformation;
import java.io.BufferedReader;
import java.io.Closeable;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.Date;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@@ -64,6 +70,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.common.base.Joiner;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
@@ -71,6 +78,7 @@ import com.google.common.cache.LoadingCache;
/**
* hadoop utils
* single instance
+ * By default, directory path does NOT end with '/'
*/
public class HadoopUtils implements Closeable, StorageOperate {
@@ -185,7 +193,11 @@ public class HadoopUtils implements Closeable, StorageOperate {
* @return DefaultFS
*/
public String getDefaultFS() {
- return getConfiguration().get(Constants.FS_DEFAULT_FS);
+ String defaultFS = getConfiguration().get(Constants.FS_DEFAULT_FS);
+ if (StringUtils.isBlank(defaultFS)) {
+ defaultFS = PropertyUtils.getString(Constants.FS_DEFAULT_FS);
+ }
+ return defaultFS;
}
/**
@@ -271,12 +283,12 @@ public class HadoopUtils implements Closeable, StorageOperate {
@Override
public String getResDir(String tenantCode) {
- return getHdfsResDir(tenantCode);
+ return getHdfsResDir(tenantCode) + FOLDER_SEPARATOR;
}
@Override
public String getUdfDir(String tenantCode) {
- return getHdfsUdfDir(tenantCode);
+ return getHdfsUdfDir(tenantCode) + FOLDER_SEPARATOR;
}
/**
@@ -290,7 +302,7 @@ public class HadoopUtils implements Closeable, StorageOperate {
*/
@Override
public boolean mkdir(String tenantCode, String hdfsPath) throws IOException {
- return fs.mkdirs(new Path(hdfsPath));
+ return fs.mkdirs(new Path(addFolderSeparatorIfNotExisted(hdfsPath)));
}
@Override
@@ -298,6 +310,26 @@ public class HadoopUtils implements Closeable, StorageOperate {
return getHdfsResourceFileName(tenantCode, fullName);
}
+ @Override
+ public String getResourceFileName(String fullName) {
+ // here is a quick fix here to get fileName. We get the resource upload path and
+ // get the index of the first appearance of resource upload path. The index is put
+ // in the start index of the substring function and get the result substring containing
+ // tenantcode and "resource" directory and the fileName.
+ // Then we split the result substring
+ // with "/" and join all elements except the first two elements because they are
+ // tenantCode and "resource" directory.
+ String resourceUploadPath =
+ RESOURCE_UPLOAD_PATH.endsWith(FOLDER_SEPARATOR) ? StringUtils.chop(RESOURCE_UPLOAD_PATH)
+ : RESOURCE_UPLOAD_PATH;
+ // +1 because we want to skip the "/" after resource upload path as well.
+ String pathContainingTenantNResource = fullName.substring(
+ fullName.indexOf(resourceUploadPath)
+ + resourceUploadPath.length() + 1);
+ String[] fileNameArr = pathContainingTenantNResource.split(FOLDER_SEPARATOR);
+ return Joiner.on(FOLDER_SEPARATOR).join(Arrays.stream(fileNameArr).skip(2).collect(Collectors.toList()));
+ }
+
@Override
public String getFileName(ResourceType resourceType, String tenantCode, String fileName) {
return getHdfsFileName(resourceType, tenantCode, fileName);
@@ -351,7 +383,7 @@ public class HadoopUtils implements Closeable, StorageOperate {
return copyLocalToHdfs(srcFile, dstPath, deleteSource, overwrite);
}
- /*
+ /**
* copy hdfs file to local
*
* @param srcHdfsFilePath source hdfs file path
@@ -368,6 +400,7 @@ public class HadoopUtils implements Closeable, StorageOperate {
*/
public boolean copyHdfsToLocal(String srcHdfsFilePath, String dstFile, boolean deleteSource,
boolean overwrite) throws IOException {
+
Path srcPath = new Path(srcHdfsFilePath);
File dstPath = new File(dstFile);
@@ -399,10 +432,29 @@ public class HadoopUtils implements Closeable, StorageOperate {
* @throws IOException errors
*/
@Override
- public boolean delete(String tenantCode, String hdfsFilePath, boolean recursive) throws IOException {
+ public boolean delete(String hdfsFilePath, boolean recursive) throws IOException {
return fs.delete(new Path(hdfsFilePath), recursive);
}
+ /**
+ * delete a list of files
+ *
+ * @param filePath the path to delete, usually it is a directory.
+ * @param recursive if path is a directory and set to
+ * true, the directory is deleted else throws an exception. In
+ * case of a file the recursive can be set to either true or false.
+ * @return true if delete is successful else false.
+ * @throws IOException errors
+ */
+
+ @Override
+ public boolean delete(String filePath, List<String> childrenPathArray, boolean recursive) throws IOException {
+ if (filePath.endsWith("/")) {
+ return fs.delete(new Path(filePath), true);
+ }
+ return fs.delete(new Path(filePath), recursive);
+ }
+
/**
* check if exists
*
@@ -411,23 +463,119 @@ public class HadoopUtils implements Closeable, StorageOperate {
* @throws IOException errors
*/
@Override
- public boolean exists(String tenantCode, String hdfsFilePath) throws IOException {
+ public boolean exists(String hdfsFilePath) throws IOException {
return fs.exists(new Path(hdfsFilePath));
}
/**
* Gets a list of files in the directory
*
- * @param filePath file path
+ * @param path file fullName path
* @return {@link FileStatus} file status
* @throws IOException errors
*/
- public FileStatus[] listFileStatus(String filePath) throws IOException {
+ @Override
+ public List<StorageEntity> listFilesStatus(String path, String defaultPath, String tenantCode,
+ ResourceType type) throws IOException {
+ // TODO: Does listStatus truncate resultList if its size goes above certain threshold (like a 1000 in S3)
+ // TODO: add hdfs prefix getFile
+ List<StorageEntity> storageEntityList = new ArrayList<>();
+ try {
+ FileStatus[] fileStatuses = fs.listStatus(new Path(path));
+
+ // transform FileStatusArray into the StorageEntity List
+ for (FileStatus fileStatus : fileStatuses) {
+ if (fileStatus.isDirectory()) {
+ // the path is a directory
+ String fullName = fileStatus.getPath().toString();
+ fullName = addFolderSeparatorIfNotExisted(fullName);
+
+ String suffix = StringUtils.difference(path, fullName);
+ String fileName = StringUtils.difference(defaultPath, fullName);
+
+ StorageEntity entity = new StorageEntity();
+ entity.setAlias(suffix);
+ entity.setFileName(fileName);
+ entity.setFullName(fullName);
+ entity.setDirectory(true);
+ entity.setDescription("");
+ entity.setUserName(tenantCode);
+ entity.setType(type);
+ entity.setSize(fileStatus.getLen());
+ entity.setCreateTime(new Date(fileStatus.getModificationTime()));
+ entity.setUpdateTime(new Date(fileStatus.getModificationTime()));
+ entity.setPfullName(path);
+
+ storageEntityList.add(entity);
+ } else {
+ // the path is a file
+ String fullName = fileStatus.getPath().toString();
+ String[] aliasArr = fullName.split("/");
+ String alias = aliasArr[aliasArr.length - 1];
+
+ String fileName = StringUtils.difference(defaultPath, fullName);
+
+ StorageEntity entity = new StorageEntity();
+ entity.setAlias(alias);
+ entity.setFileName(fileName);
+ entity.setFullName(fullName);
+ entity.setDirectory(false);
+ entity.setDescription("");
+ entity.setUserName(tenantCode);
+ entity.setType(type);
+ entity.setSize(fileStatus.getLen());
+ entity.setCreateTime(new Date(fileStatus.getModificationTime()));
+ entity.setUpdateTime(new Date(fileStatus.getModificationTime()));
+ entity.setPfullName(path);
+
+ storageEntityList.add(entity);
+ }
+ }
+ } catch (FileNotFoundException e) {
+ throw new FileNotFoundException("The path does not exist.");
+ } catch (IOException e) {
+ throw new IOException("Get file list exception.", e);
+ }
+
+ return storageEntityList;
+ }
+
+ @Override
+ public StorageEntity getFileStatus(String path, String prefix, String tenantCode,
+ ResourceType type) throws IOException {
try {
- return fs.listStatus(new Path(filePath));
+ FileStatus fileStatus = fs.getFileStatus(new Path(path));
+ String alias = "";
+ String fileName = "";
+ String fullName = fileStatus.getPath().toString();
+ if (fileStatus.isDirectory()) {
+ fullName = addFolderSeparatorIfNotExisted(fullName);
+ alias = findDirAlias(fullName);
+ fileName = StringUtils.difference(prefix, fullName);
+ } else {
+ String[] aliasArr = fileStatus.getPath().toString().split("/");
+ alias = aliasArr[aliasArr.length - 1];
+ fileName = StringUtils.difference(prefix, fileStatus.getPath().toString());
+ }
+
+ StorageEntity entity = new StorageEntity();
+ entity.setAlias(alias);
+ entity.setFileName(fileName);
+ entity.setFullName(fullName);
+ entity.setDirectory(fileStatus.isDirectory());
+ entity.setDescription("");
+ entity.setUserName(tenantCode);
+ entity.setType(type);
+ entity.setSize(fileStatus.getLen());
+ entity.setCreateTime(new Date(fileStatus.getModificationTime()));
+ entity.setUpdateTime(new Date(fileStatus.getModificationTime()));
+ entity.setPfullName(path);
+
+ return entity;
+ } catch (FileNotFoundException e) {
+ throw new FileNotFoundException("The path does not exist.");
} catch (IOException e) {
- logger.error("Get file list exception", e);
- throw new IOException("Get file list exception", e);
+ throw new IOException("Get file exception.", e);
}
}
@@ -528,10 +676,12 @@ public class HadoopUtils implements Closeable, StorageOperate {
* @return data hdfs path
*/
public static String getHdfsDataBasePath() {
+ String defaultFS = PropertyUtils.getString(Constants.FS_DEFAULT_FS);
+ defaultFS = defaultFS.endsWith("/") ? StringUtils.chop(defaultFS) : defaultFS;
if (FOLDER_SEPARATOR.equals(RESOURCE_UPLOAD_PATH)) {
- return "";
+ return defaultFS + "";
} else {
- return RESOURCE_UPLOAD_PATH;
+ return defaultFS + RESOURCE_UPLOAD_PATH;
}
}
@@ -739,8 +889,8 @@ public class HadoopUtils implements Closeable, StorageOperate {
public void deleteTenant(String tenantCode) throws Exception {
String tenantPath = getHdfsDataBasePath() + FOLDER_SEPARATOR + tenantCode;
- if (exists(tenantCode, tenantPath)) {
- delete(tenantCode, tenantPath, true);
+ if (exists(tenantPath)) {
+ delete(tenantPath, true);
}
}
@@ -749,4 +899,67 @@ public class HadoopUtils implements Closeable, StorageOperate {
public ResUploadType returnStorageType() {
return ResUploadType.HDFS;
}
+
+ @Override
+ public List<StorageEntity> listFilesStatusRecursively(String path, String defaultPath, String tenantCode,
+ ResourceType type) {
+ List<StorageEntity> storageEntityList = new ArrayList<>();
+
+ LinkedList<StorageEntity> foldersToFetch = new LinkedList<>();
+
+ do {
+ String pathToExplore = "";
+ if (foldersToFetch.size() == 0) {
+ pathToExplore = path;
+ } else {
+ pathToExplore = foldersToFetch.pop().getFullName();
+ }
+
+ try {
+ List<StorageEntity> tempList = listFilesStatus(pathToExplore, defaultPath, tenantCode, type);
+
+ for (StorageEntity temp : tempList) {
+ if (temp.isDirectory()) {
+ foldersToFetch.add(temp);
+ }
+ }
+
+ storageEntityList.addAll(tempList);
+ } catch (FileNotFoundException e) {
+ logger.error("Resource path: {}", pathToExplore, e);
+ // return the resources fetched before error occurs.
+ return storageEntityList;
+ } catch (IOException e) {
+ logger.error("Resource path: {}", pathToExplore, e);
+ // return the resources fetched before error occurs.
+ return storageEntityList;
+ }
+
+ } while (foldersToFetch.size() != 0);
+
+ return storageEntityList;
+
+ }
+
+ /**
+ * find alias for directories, NOT for files
+ * a directory is a path ending with "/"
+ */
+ private String findDirAlias(String myStr) {
+ if (!myStr.endsWith("/")) {
+ // Make sure system won't crush down if someone accidentally misuse the function.
+ return myStr;
+ }
+ int lastIndex = myStr.lastIndexOf("/");
+ String subbedString = myStr.substring(0, lastIndex);
+ int secondLastIndex = subbedString.lastIndexOf("/");
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append(myStr, secondLastIndex + 1, lastIndex + 1);
+
+ return stringBuilder.toString();
+ }
+
+ private String addFolderSeparatorIfNotExisted(String fullName) {
+ return fullName.endsWith(FOLDER_SEPARATOR) ? fullName : fullName + FOLDER_SEPARATOR;
+ }
}
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/OssOperator.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/OssOperator.java
index 338b2dd100..d076112927 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/OssOperator.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/OssOperator.java
@@ -28,6 +28,7 @@ import org.apache.dolphinscheduler.common.factory.OssClientFactory;
import org.apache.dolphinscheduler.common.model.OssConnection;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
+import org.apache.dolphinscheduler.service.storage.StorageEntity;
import org.apache.dolphinscheduler.service.storage.StorageOperate;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
@@ -175,6 +176,11 @@ public class OssOperator implements Closeable, StorageOperate {
return String.format(FORMAT_S_S, getOssResDir(tenantCode), fileName);
}
+ @Override
+ public String getResourceFileName(String fullName) {
+ return null;
+ }
+
@Override
public String getFileName(ResourceType resourceType, String tenantCode, String fileName) {
if (fileName.startsWith(FOLDER_SEPARATOR)) {
@@ -183,6 +189,11 @@ public class OssOperator implements Closeable, StorageOperate {
return getDir(resourceType, tenantCode) + fileName;
}
+ @Override
+ public boolean delete(String filePath, List<String> childrenPathArray, boolean recursive) throws IOException {
+ return false;
+ }
+
@Override
public void download(String tenantCode, String srcFilePath, String dstFilePath, boolean deleteSource,
boolean overwrite) throws IOException {
@@ -210,12 +221,12 @@ public class OssOperator implements Closeable, StorageOperate {
}
@Override
- public boolean exists(String tenantCode, String fileName) throws IOException {
+ public boolean exists(String fileName) throws IOException {
return ossClient.doesObjectExist(bucketName, fileName);
}
@Override
- public boolean delete(String tenantCode, String filePath, boolean recursive) throws IOException {
+ public boolean delete(String filePath, boolean recursive) throws IOException {
try {
ossClient.deleteObject(bucketName, filePath);
return true;
@@ -274,6 +285,24 @@ public class OssOperator implements Closeable, StorageOperate {
return ResUploadType.OSS;
}
+ @Override
+ public List<StorageEntity> listFilesStatusRecursively(String path, String defaultPath, String tenantCode,
+ ResourceType type) {
+ return null;
+ }
+
+ @Override
+ public List<StorageEntity> listFilesStatus(String path, String defaultPath, String tenantCode,
+ ResourceType type) throws Exception {
+ return null;
+ }
+
+ @Override
+ public StorageEntity getFileStatus(String path, String defaultPath, String tenantCode,
+ ResourceType type) throws Exception {
+ return null;
+ }
+
@Override
public void deleteTenant(String tenantCode) throws Exception {
deleteTenantCode(tenantCode);
diff --git a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/S3Utils.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/S3Utils.java
index 56f904d8e4..f8ffafae89 100644
--- a/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/S3Utils.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/storage/impl/S3Utils.java
@@ -29,6 +29,7 @@ import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.ResUploadType;
import org.apache.dolphinscheduler.common.utils.PropertyUtils;
import org.apache.dolphinscheduler.plugin.task.api.TaskConstants;
+import org.apache.dolphinscheduler.service.storage.StorageEntity;
import org.apache.dolphinscheduler.service.storage.StorageOperate;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
@@ -44,7 +45,10 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.LinkedList;
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -61,14 +65,22 @@ import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.Bucket;
+import com.amazonaws.services.s3.model.DeleteObjectsRequest;
+import com.amazonaws.services.s3.model.ListObjectsV2Request;
+import com.amazonaws.services.s3.model.ListObjectsV2Result;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.MultipleFileDownload;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
+import com.google.common.base.Joiner;
+/**
+ * By default, directory path does end with '/'
+ */
public class S3Utils implements Closeable, StorageOperate {
private static final Logger logger = LoggerFactory.getLogger(S3Utils.class);
@@ -171,6 +183,26 @@ public class S3Utils implements Closeable, StorageOperate {
return String.format(FORMAT_S_S, getS3ResDir(tenantCode), fileName);
}
+ @Override
+ public String getResourceFileName(String fullName) {
+ // here is a quick fix here to get fileName. We get the resource upload path and
+ // get the index of the first appearance of resource upload path. The index is put
+ // in the start index of the substring function and get the result substring containing
+ // tenantcode and "resource" directory and the fileName.
+ // Then we split the result substring
+ // with "/" and join all elements except the first two elements because they are
+ // tenantCode and "resource" directory.
+ String resourceUploadPath =
+ RESOURCE_UPLOAD_PATH.endsWith(FOLDER_SEPARATOR) ? StringUtils.chop(RESOURCE_UPLOAD_PATH)
+ : RESOURCE_UPLOAD_PATH;
+ // +1 because we want to skip the "/" after resource upload path as well.
+ String pathContainingTenantNResource = fullName.substring(
+ fullName.indexOf(resourceUploadPath)
+ + resourceUploadPath.length() + 1);
+ String[] fileNameArr = pathContainingTenantNResource.split(FOLDER_SEPARATOR);
+ return Joiner.on(FOLDER_SEPARATOR).join(Arrays.stream(fileNameArr).skip(2).collect(Collectors.toList()));
+ }
+
@Override
public String getFileName(ResourceType resourceType, String tenantCode, String fileName) {
if (fileName.startsWith(FOLDER_SEPARATOR)) {
@@ -206,21 +238,38 @@ public class S3Utils implements Closeable, StorageOperate {
}
@Override
- public boolean exists(String tenantCode, String fileName) throws IOException {
- return s3Client.doesObjectExist(BUCKET_NAME, fileName);
+ public boolean exists(String fullName) throws IOException {
+ return s3Client.doesObjectExist(BUCKET_NAME, fullName);
}
@Override
- public boolean delete(String tenantCode, String filePath, boolean recursive) throws IOException {
+ public boolean delete(String fullName, boolean recursive) throws IOException {
try {
- s3Client.deleteObject(BUCKET_NAME, filePath);
+ s3Client.deleteObject(BUCKET_NAME, fullName);
return true;
} catch (AmazonServiceException e) {
- logger.error("delete the object error,the resource path is {}", filePath);
+ logger.error("delete the object error,the resource path is {}", fullName);
return false;
}
}
+ @Override
+ public boolean delete(String fullName, List<String> childrenPathList, boolean recursive) throws IOException {
+ // append the resource fullName to the list for deletion.
+ childrenPathList.add(fullName);
+
+ DeleteObjectsRequest deleteObjectsRequest = new DeleteObjectsRequest(BUCKET_NAME)
+ .withKeys(childrenPathList.stream().toArray(String[]::new));
+ try {
+ s3Client.deleteObjects(deleteObjectsRequest);
+ } catch (AmazonServiceException e) {
+ logger.error("delete objects error", e);
+ return false;
+ }
+
+ return true;
+ }
+
@Override
public boolean copy(String srcPath, String dstPath, boolean deleteSource, boolean overwrite) throws IOException {
s3Client.copyObject(BUCKET_NAME, srcPath, BUCKET_NAME, dstPath);
@@ -384,4 +433,197 @@ public class S3Utils implements Closeable, StorageOperate {
public ResUploadType returnStorageType() {
return ResUploadType.S3;
}
+
+ @Override
+ public List<StorageEntity> listFilesStatusRecursively(String path, String defaultPath, String tenantCode,
+ ResourceType type) {
+ List<StorageEntity> storageEntityList = new ArrayList<>();
+
+ LinkedList<StorageEntity> foldersToFetch = new LinkedList<>();
+
+ do {
+ String pathToExplore = "";
+ if (foldersToFetch.size() == 0) {
+ pathToExplore = path;
+ } else {
+ pathToExplore = foldersToFetch.pop().getFullName();
+ }
+
+ try {
+ List<StorageEntity> tempList = listFilesStatus(pathToExplore, defaultPath, tenantCode, type);
+
+ for (StorageEntity temp : tempList) {
+ if (temp.isDirectory()) {
+ foldersToFetch.add(temp);
+ }
+ }
+
+ storageEntityList.addAll(tempList);
+ } catch (AmazonServiceException e) {
+ logger.error("Resource path: {}", pathToExplore, e);
+ // return the resources fetched before error occurs.
+ return storageEntityList;
+ }
+
+ } while (foldersToFetch.size() != 0);
+
+ return storageEntityList;
+
+ }
+
+ @Override
+ public List<StorageEntity> listFilesStatus(String path, String defaultPath, String tenantCode,
+ ResourceType type) throws AmazonServiceException {
+ List<StorageEntity> storageEntityList = new ArrayList<>();
+
+ // TODO: optimize pagination
+ ListObjectsV2Request request = new ListObjectsV2Request();
+ request.setBucketName(BUCKET_NAME);
+ request.setPrefix(path);
+ request.setDelimiter("/");
+
+ ListObjectsV2Result v2Result;
+ do {
+ try {
+ v2Result = s3Client.listObjectsV2(request);
+ } catch (AmazonServiceException e) {
+ throw new AmazonServiceException("Get S3 file list exception, error type:" + e.getErrorType(), e);
+ }
+
+ List<S3ObjectSummary> summaries = v2Result.getObjectSummaries();
+
+ for (S3ObjectSummary summary : summaries) {
+ if (!summary.getKey().endsWith("/")) {
+ // the path is a file
+ String[] aliasArr = summary.getKey().split("/");
+ String alias = aliasArr[aliasArr.length - 1];
+ String fileName = StringUtils.difference(defaultPath, summary.getKey());
+
+ StorageEntity entity = new StorageEntity();
+ entity.setAlias(alias);
+ entity.setFileName(fileName);
+ entity.setFullName(summary.getKey());
+ entity.setDirectory(false);
+ entity.setDescription("");
+ entity.setUserName(tenantCode);
+ entity.setType(type);
+ entity.setSize(summary.getSize());
+ entity.setCreateTime(summary.getLastModified());
+ entity.setUpdateTime(summary.getLastModified());
+ entity.setPfullName(path);
+
+ storageEntityList.add(entity);
+ }
+ }
+
+ for (String commonPrefix : v2Result.getCommonPrefixes()) {
+ // the paths in commonPrefix are directories
+ String suffix = StringUtils.difference(path, commonPrefix);
+ String fileName = StringUtils.difference(defaultPath, commonPrefix);
+
+ StorageEntity entity = new StorageEntity();
+ entity.setAlias(suffix);
+ entity.setFileName(fileName);
+ entity.setFullName(commonPrefix);
+ entity.setDirectory(true);
+ entity.setDescription("");
+ entity.setUserName(tenantCode);
+ entity.setType(type);
+ entity.setSize(0);
+ entity.setCreateTime(null);
+ entity.setUpdateTime(null);
+ entity.setPfullName(path);
+
+ storageEntityList.add(entity);
+ }
+
+ request.setContinuationToken(v2Result.getContinuationToken());
+
+ } while (v2Result.isTruncated());
+
+ return storageEntityList;
+ }
+
+ @Override
+ public StorageEntity getFileStatus(String path, String defaultPath, String tenantCode,
+ ResourceType type) throws AmazonServiceException, FileNotFoundException {
+ // Notice: we do not use getObject here because intermediate directories
+ // may not exist in S3, which can cause getObject to throw exception.
+ // Since we still want to access it on frontend, this is a workaround using listObjects.
+
+ ListObjectsV2Request request = new ListObjectsV2Request();
+ request.setBucketName(BUCKET_NAME);
+ request.setPrefix(path);
+ request.setDelimiter("/");
+
+ ListObjectsV2Result v2Result;
+ try {
+ v2Result = s3Client.listObjectsV2(request);
+ } catch (AmazonServiceException e) {
+ throw new AmazonServiceException("Get S3 file list exception, error type:" + e.getErrorType(), e);
+ }
+
+ List<S3ObjectSummary> summaries = v2Result.getObjectSummaries();
+
+ if (path.endsWith("/")) {
+ // the path is a directory that may or may not exist in S3
+ String alias = findDirAlias(path);
+ String fileName = StringUtils.difference(defaultPath, path);
+
+ StorageEntity entity = new StorageEntity();
+ entity.setAlias(alias);
+ entity.setFileName(fileName);
+ entity.setFullName(path);
+ entity.setDirectory(true);
+ entity.setDescription("");
+ entity.setUserName(tenantCode);
+ entity.setType(type);
+ entity.setSize(0);
+
+ return entity;
+
+ } else {
+ // the path is a file
+ if (summaries.size() > 0) {
+ S3ObjectSummary summary = summaries.get(0);
+ String[] aliasArr = summary.getKey().split("/");
+ String alias = aliasArr[aliasArr.length - 1];
+ String fileName = StringUtils.difference(defaultPath, summary.getKey());
+
+ StorageEntity entity = new StorageEntity();
+ entity.setAlias(alias);
+ entity.setFileName(fileName);
+ entity.setFullName(summary.getKey());
+ entity.setDirectory(false);
+ entity.setDescription("");
+ entity.setUserName(tenantCode);
+ entity.setType(type);
+ entity.setSize(summary.getSize());
+ entity.setCreateTime(summary.getLastModified());
+ entity.setUpdateTime(summary.getLastModified());
+
+ return entity;
+ }
+ }
+
+ throw new FileNotFoundException("Object is not found in S3 Bucket: " + BUCKET_NAME);
+ }
+
+ /**
+ * find alias for directories, NOT for files
+ * a directory is a path ending with "/"
+ */
+ private String findDirAlias(String myStr) {
+ if (!myStr.endsWith("/")) {
+ // Make sure system won't crush down if someone accidentally misuse the function.
+ return myStr;
+ }
+ int lastIndex = myStr.lastIndexOf("/");
+ String subbedString = myStr.substring(0, lastIndex);
+ int secondLastIndex = subbedString.lastIndexOf("/");
+ StringBuilder stringBuilder = new StringBuilder();
+ stringBuilder.append(myStr, secondLastIndex + 1, lastIndex + 1);
+
+ return stringBuilder.toString();
+ }
}
diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
index 5ad590a24f..12fb2c69a8 100644
--- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
+++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/process/ProcessServiceTest.java
@@ -40,7 +40,6 @@ import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
-import org.apache.dolphinscheduler.dao.entity.Resource;
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.TaskGroupQueue;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
@@ -59,6 +58,7 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationLogMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessTaskRelationMapper;
import org.apache.dolphinscheduler.dao.mapper.ResourceMapper;
+import org.apache.dolphinscheduler.dao.mapper.ResourceTaskMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionLogMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskGroupMapper;
@@ -149,6 +149,8 @@ public class ProcessServiceTest {
@Mock
private ResourceMapper resourceMapper;
@Mock
+ private ResourceTaskMapper resourceTaskMapper;
+ @Mock
private TaskGroupMapper taskGroupMapper;
@Mock
private DataSourceMapper dataSourceMapper;
@@ -616,6 +618,7 @@ public class ProcessServiceTest {
Mockito.when(taskDefinitionLogMapper.queryMaxVersionForDefinition(taskDefinition.getCode())).thenReturn(1);
Mockito.when(taskDefinitionMapper.queryByCodeList(Collections.singletonList(taskDefinition.getCode())))
.thenReturn(Collections.singletonList(taskDefinition));
+ Mockito.when(taskDefinitionMapper.queryByCode(Mockito.anyLong())).thenReturn(taskDefinition);
int result = processService.saveTaskDefine(operator, projectCode, taskDefinitionLogs, Boolean.TRUE);
Assertions.assertEquals(0, result);
}
@@ -693,24 +696,22 @@ public class ProcessServiceTest {
public void testUpdateResourceInfo() throws Exception {
// test if input is null
ResourceInfo resourceInfoNull = null;
- ResourceInfo updatedResourceInfo1 = processService.updateResourceInfo(resourceInfoNull);
+ ResourceInfo updatedResourceInfo1 = processService.updateResourceInfo(0, resourceInfoNull);
Assertions.assertNull(updatedResourceInfo1);
// test if resource id less than 1
ResourceInfo resourceInfoVoid = new ResourceInfo();
- ResourceInfo updatedResourceInfo2 = processService.updateResourceInfo(resourceInfoVoid);
- Assertions.assertNull(updatedResourceInfo2);
+ ResourceInfo updatedResourceInfo2 = processService.updateResourceInfo(0, resourceInfoVoid);
+ Assertions.assertNull(updatedResourceInfo2.getResourceName());
// test normal situation
ResourceInfo resourceInfoNormal = new ResourceInfo();
resourceInfoNormal.setId(1);
- Resource resource = new Resource();
- resource.setId(1);
- resource.setFileName("test.txt");
- resource.setFullName("/test.txt");
- Mockito.when(resourceMapper.selectById(1)).thenReturn(resource);
+ resourceInfoNormal.setRes("test.txt");
+ resourceInfoNormal.setResourceName("/test.txt");
+ Mockito.when(resourceTaskMapper.existResourceByTaskIdNFullName(0, "/test.txt")).thenReturn(1);
- ResourceInfo updatedResourceInfo3 = processService.updateResourceInfo(resourceInfoNormal);
+ ResourceInfo updatedResourceInfo3 = processService.updateResourceInfo(0, resourceInfoNormal);
Assertions.assertEquals(1, updatedResourceInfo3.getId().intValue());
Assertions.assertEquals("test.txt", updatedResourceInfo3.getRes());
diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/OssOperatorTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/OssOperatorTest.java
index da20bf7b32..4d35807d6e 100644
--- a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/OssOperatorTest.java
+++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/storage/impl/OssOperatorTest.java
@@ -169,7 +169,7 @@ public class OssOperatorTest {
boolean doesExist = false;
doReturn(true).when(ossClientMock).doesObjectExist(BUCKET_NAME_MOCK, FILE_NAME_MOCK);
try {
- doesExist = ossOperator.exists(TENANT_CODE_MOCK, FILE_NAME_MOCK);
+ doesExist = ossOperator.exists(FILE_NAME_MOCK);
} catch (IOException e) {
Assertions.fail("unexpected IO exception in unit test");
}
@@ -183,7 +183,7 @@ public class OssOperatorTest {
boolean isDeleted = false;
doReturn(null).when(ossClientMock).deleteObject(anyString(), anyString());
try {
- isDeleted = ossOperator.delete(TENANT_CODE_MOCK, FILE_NAME_MOCK, true);
+ isDeleted = ossOperator.delete(FILE_NAME_MOCK, true);
} catch (IOException e) {
Assertions.fail("unexpected IO exception in unit test");
}
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java
index 97463ec7df..391899dc56 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sql/src/main/java/org/apache/dolphinscheduler/plugin/task/sql/SqlTask.java
@@ -572,9 +572,7 @@ public class SqlTask extends AbstractTask {
String prefixPath = defaultFS.startsWith("file://") ? "file://" : defaultFS;
String uploadPath = CommonUtils.getHdfsUdfDir(value.getTenantCode());
String resourceFullName = value.getResourceName();
- resourceFullName =
- resourceFullName.startsWith("/") ? resourceFullName : String.format("/%s", resourceFullName);
- return String.format("add jar %s%s%s", prefixPath, uploadPath, resourceFullName);
+ return String.format("add jar %s", resourceFullName);
}).collect(Collectors.toList());
}
diff --git a/dolphinscheduler-ui/src/locales/en_US/resource.ts b/dolphinscheduler-ui/src/locales/en_US/resource.ts
index 297a3217ea..dbb9a0f4a4 100644
--- a/dolphinscheduler-ui/src/locales/en_US/resource.ts
+++ b/dolphinscheduler-ui/src/locales/en_US/resource.ts
@@ -24,6 +24,7 @@ export default {
enter_keyword_tips: 'Please enter keyword',
name: 'Name',
user_name: 'Resource userName',
+ tenant_name: 'Resource tenantName',
whether_directory: 'Whether directory',
file_name: 'File Name',
description: 'Description',
@@ -58,6 +59,7 @@ export default {
upload_udf_resources: 'Upload UDF Resources',
udf_source_name: 'UDF Resource Name',
user_name: 'Resource userName',
+ tenant_name: 'Resource tenantName',
whether_directory: 'Whether directory',
file_name: 'File Name',
file_size: 'File Size',
@@ -68,6 +70,7 @@ export default {
yes: 'Yes',
no: 'No',
edit: 'Edit',
+ rename: 'Rename',
download: 'Download',
delete: 'Delete',
delete_confirm: 'Delete?',
@@ -83,7 +86,7 @@ export default {
function: {
udf_function: 'UDF Function',
create_udf_function: 'Create UDF Function',
- edit_udf_function: 'Create UDF Function',
+ edit_udf_function: 'Edit UDF Function',
udf_function_name: 'UDF Function Name',
user_name: 'Resource userName',
class_name: 'Class Name',
diff --git a/dolphinscheduler-ui/src/router/modules/resources.ts b/dolphinscheduler-ui/src/router/modules/resources.ts
index cc47e58377..0e33de52bd 100644
--- a/dolphinscheduler-ui/src/router/modules/resources.ts
+++ b/dolphinscheduler-ui/src/router/modules/resources.ts
@@ -53,7 +53,7 @@ export default {
}
},
{
- path: '/resource/file/edit/:id',
+ path: '/resource/file/edit',
name: 'resource-file-edit',
component: components['resource-file-edit'],
meta: {
@@ -65,7 +65,7 @@ export default {
}
},
{
- path: '/resource/file/subdirectory/:id',
+ path: '/resource/file/subdirectory',
name: 'resource-file-subdirectory',
component: components['resource-file'],
meta: {
@@ -77,7 +77,7 @@ export default {
}
},
{
- path: '/resource/file/list/:id',
+ path: '/resource/file/list',
name: 'resource-file-list',
component: components['resource-file-edit'],
meta: {
@@ -89,7 +89,7 @@ export default {
}
},
{
- path: '/resource/file/create/:id',
+ path: '/resource/file/create',
name: 'resource-subfile-create',
component: components['resource-file-create'],
meta: {
@@ -112,7 +112,7 @@ export default {
}
},
{
- path: '/resource/resource-manage/:id',
+ path: '/resource/resource-manage',
name: 'resource-sub-manage',
component: components['resource-udf-resource'],
meta: {
diff --git a/dolphinscheduler-ui/src/service/modules/resources/index.ts b/dolphinscheduler-ui/src/service/modules/resources/index.ts
index b461f5f3d7..9a7fbe7b68 100644
--- a/dolphinscheduler-ui/src/service/modules/resources/index.ts
+++ b/dolphinscheduler-ui/src/service/modules/resources/index.ts
@@ -22,6 +22,7 @@ import {
NameReq,
FileNameReq,
FullNameReq,
+ TenantCodeReq,
IdReq,
ContentReq,
DescriptionReq,
@@ -31,12 +32,11 @@ import {
ProgramTypeReq,
ListReq,
ViewResourceReq,
- ResourceIdReq,
UdfFuncReq
} from './types'
export function queryResourceListPaging(
- params: ListReq & IdReq & ResourceTypeReq
+ params: ListReq & ResourceTypeReq & FullNameReq & TenantCodeReq
): any {
return axios({
url: '/resources',
@@ -45,21 +45,23 @@ export function queryResourceListPaging(
})
}
-export function queryResourceById(
- params: ResourceTypeReq & FullNameReq & IdReq,
- id: number
+export function queryCurrentResourceByFileName(
+ params: ResourceTypeReq & FileNameReq & TenantCodeReq,
): any {
return axios({
- url: `/resources/${id}`,
+ url: `/resources/query-file-name`,
method: 'get',
params
})
}
-export function queryCurrentResourceById(id: number): any {
+export function queryCurrentResourceByFullName(
+ params: ResourceTypeReq & FullNameReq & TenantCodeReq,
+): any {
return axios({
- url: `/resources/${id}/query`,
- method: 'get'
+ url: `/resources/query-full-name`,
+ method: 'get',
+ params
})
}
@@ -107,7 +109,7 @@ export function createDirectory(
})
}
-export function queryResourceList(params: ResourceTypeReq): any {
+export function queryResourceList(params: ResourceTypeReq & FullNameReq): any {
return axios({
url: '/resources/list',
method: 'get',
@@ -159,10 +161,11 @@ export function verifyUdfFuncName(params: NameReq): any {
})
}
-export function deleteUdfFunc(id: number): any {
+export function deleteUdfFunc(id: number, params: FullNameReq): any {
return axios({
url: `/resources/udf-func/${id}`,
- method: 'delete'
+ method: 'delete',
+ params
})
}
@@ -182,37 +185,36 @@ export function verifyResourceName(params: FullNameReq & ResourceTypeReq): any {
})
}
-export function queryResource(
+export function doesResourceExist(
params: FullNameReq & ResourceTypeReq,
- id: IdReq
): any {
return axios({
- url: `/resources/verify-name/${id}`,
+ url: `/resources/verify-name`,
method: 'get',
params
})
}
export function updateResource(
- data: NameReq & ResourceTypeReq & IdReq & DescriptionReq,
- id: number
+ data: NameReq & ResourceTypeReq & DescriptionReq & FullNameReq & TenantCodeReq,
): any {
return axios({
- url: `/resources/${id}`,
+ url: `/resources`,
method: 'put',
data
})
}
-export function deleteResource(id: number): any {
+export function deleteResource(params: FullNameReq & TenantCodeReq): any {
return axios({
- url: `/resources/${id}`,
- method: 'delete'
+ url: `/resources`,
+ method: 'delete',
+ params
})
}
-export function downloadResource(id: number): void {
- utils.downloadFile(`resources/${id}/download`)
+export function downloadResource(params: FullNameReq): void {
+ utils.downloadFile(`resources/download`, params)
}
export function viewUIUdfFunction(id: IdReq): any {
@@ -222,28 +224,27 @@ export function viewUIUdfFunction(id: IdReq): any {
})
}
-export function updateResourceContent(data: ContentReq, id: number): any {
+export function updateResourceContent(data: ContentReq & TenantCodeReq & FullNameReq): any {
return axios({
- url: `/resources/${id}/update-content`,
+ url: `/resources/update-content`,
method: 'put',
data
})
}
-export function viewResource(params: ViewResourceReq, id: number): any {
+export function viewResource(params: ViewResourceReq & FullNameReq & TenantCodeReq): any {
return axios({
- url: `/resources/${id}/view`,
+ url: `/resources/view`,
method: 'get',
params
})
}
export function createUdfFunc(
- data: UdfFuncReq,
- resourceId: ResourceIdReq
+ data: UdfFuncReq
): any {
return axios({
- url: `/resources/${resourceId}/udf-func`,
+ url: `/resources/udf-func`,
method: 'post',
data
})
@@ -251,11 +252,10 @@ export function createUdfFunc(
export function updateUdfFunc(
data: UdfFuncReq,
- resourceId: ResourceIdReq,
id: number
): any {
return axios({
- url: `/resources/${resourceId}/udf-func/${id}`,
+ url: `/resources/udf-func/${id}`,
method: 'put',
data
})
diff --git a/dolphinscheduler-ui/src/service/modules/resources/types.ts b/dolphinscheduler-ui/src/service/modules/resources/types.ts
index 85c6fef9f8..9df6386b09 100644
--- a/dolphinscheduler-ui/src/service/modules/resources/types.ts
+++ b/dolphinscheduler-ui/src/service/modules/resources/types.ts
@@ -36,6 +36,10 @@ interface FileNameReq {
fileName: string
}
+interface TenantCodeReq{
+ tenantCode: string
+}
+
interface FullNameReq {
fullName: string
}
@@ -80,11 +84,7 @@ interface ViewResourceReq {
skipLineNum: number
}
-interface ResourceIdReq {
- resourceId: number
-}
-
-interface UdfFuncReq extends UdfTypeReq, DescriptionReq, ResourceIdReq {
+interface UdfFuncReq extends UdfTypeReq, DescriptionReq, FullNameReq {
className: string
funcName: string
argTypes?: string
@@ -126,6 +126,7 @@ export {
NameReq,
FileNameReq,
FullNameReq,
+ TenantCodeReq,
IdReq,
ContentReq,
DescriptionReq,
@@ -135,7 +136,6 @@ export {
ProgramTypeReq,
ListReq,
ViewResourceReq,
- ResourceIdReq,
UdfFuncReq,
ResourceListRes,
ResourceViewRes,
diff --git a/dolphinscheduler-ui/src/utils/tree-format.ts b/dolphinscheduler-ui/src/utils/tree-format.ts
index 5f7cea50ad..9168ead429 100644
--- a/dolphinscheduler-ui/src/utils/tree-format.ts
+++ b/dolphinscheduler-ui/src/utils/tree-format.ts
@@ -16,7 +16,7 @@
*/
const removeUselessChildren = (
- list: { children?: []; dirctory?: boolean; disabled?: boolean }[]
+ list: { children?: []; directory?: boolean; disabled?: boolean; dirctory?: boolean }[]
) => {
if (!list.length) return
list.forEach((item) => {
diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-resources.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-resources.ts
index e4dd9f54ef..8b0f741963 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-resources.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-resources.ts
@@ -41,7 +41,7 @@ export function useResources(
}
if (resourcesLoading.value) return
resourcesLoading.value = true
- const res = await queryResourceList({ type: 'FILE' })
+ const res = await queryResourceList({ type: 'FILE', fullName:"" })
utils.removeUselessChildren(res)
resourcesOptions.value = res || []
resourcesLoading.value = false
@@ -65,7 +65,7 @@ export function useResources(
showPath: true,
checkStrategy: 'child',
placeholder: t('project.node.resources_tips'),
- keyField: 'id',
+ keyField: 'fullName',
labelField: 'name',
loading: resourcesLoading
},
diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
index a11ea1da0c..6896260e2a 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
@@ -42,7 +42,7 @@ export function formatParams(data: INodeData): {
taskParams.jvmArgs = data.jvmArgs
taskParams.isModulePath = data.isModulePath
if (data.runType === 'JAR' && data.mainJar) {
- taskParams.mainJar = { id: data.mainJar }
+ taskParams.mainJar = { resourceName: data.mainJar }
}
}
@@ -53,7 +53,7 @@ export function formatParams(data: INodeData): {
taskParams.programType = data.programType
taskParams.mainClass = data.mainClass
if (data.mainJar) {
- taskParams.mainJar = { id: data.mainJar }
+ taskParams.mainJar = { resourceName: data.mainJar }
}
taskParams.deployMode = data.deployMode
taskParams.appName = data.appName
@@ -488,7 +488,7 @@ export function formatParams(data: INodeData): {
initScript: data.initScript,
rawScript: data.rawScript,
resourceList: data.resourceList?.length
- ? data.resourceList.map((id: number) => ({ id }))
+ ? data.resourceList.map((fullName: string) => ({ resourceName: `${fullName}` }))
: [],
...taskParams
},
@@ -511,7 +511,6 @@ export function formatParams(data: INodeData): {
params.taskDefinitionJsonObj.timeout = 0
params.taskDefinitionJsonObj.timeoutNotifyStrategy = ''
}
-
return params
}
@@ -537,11 +536,11 @@ export function formatModel(data: ITaskData) {
}
if (data.taskParams?.resourceList) {
params.resourceList = data.taskParams.resourceList.map(
- (item: { id: number }) => item.id
+ (item: { resourceName: string }) => (`${item.resourceName}`)
)
}
if (data.taskParams?.mainJar) {
- params.mainJar = data.taskParams?.mainJar.id
+ params.mainJar = data.taskParams?.mainJar.resourceName
}
if (data.taskParams?.method) {
diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts
index 49eccf4c53..23e1d618f9 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/types.ts
@@ -90,8 +90,14 @@ interface ISwitchResult {
nextNode?: number
}
+/*
+ * resourceName: resource full name
+ * res: resource file name
+ */
interface ISourceItem {
- id: number
+ id?: number,
+ resourceName: string,
+ res?: string
}
interface ISqoopTargetData {
@@ -429,8 +435,8 @@ interface INodeData
preTasks?: number[]
preTaskOptions?: []
postTaskOptions?: []
- resourceList?: number[]
- mainJar?: number
+ resourceList?: string[]
+ mainJar?: string
timeoutSetting?: boolean
isCustomTask?: boolean
method?: string
diff --git a/dolphinscheduler-ui/src/views/projects/workflow/definition/components/use-modal.ts b/dolphinscheduler-ui/src/views/projects/workflow/definition/components/use-modal.ts
index df24da14c0..9767f51534 100644
--- a/dolphinscheduler-ui/src/views/projects/workflow/definition/components/use-modal.ts
+++ b/dolphinscheduler-ui/src/views/projects/workflow/definition/components/use-modal.ts
@@ -125,7 +125,6 @@ export function useModal(
params.startParams = !_.isEmpty(startParams)
? JSON.stringify(startParams)
: ''
-
await startProcessInstance(params, variables.projectCode)
window.$message.success(t('project.workflow.success'))
state.saving = false
diff --git a/dolphinscheduler-ui/src/views/resource/file/create/index.tsx b/dolphinscheduler-ui/src/views/resource/file/create/index.tsx
index 4465714dcf..10dacd3ec6 100644
--- a/dolphinscheduler-ui/src/views/resource/file/create/index.tsx
+++ b/dolphinscheduler-ui/src/views/resource/file/create/index.tsx
@@ -49,9 +49,7 @@ export default defineComponent({
}
const handleReturn = () => {
- const { id } = router.currentRoute.value.params
- const name = id ? 'resource-file-subdirectory' : 'file'
- router.push({ name, params: { id } })
+ router.go(-1)
}
const trim = getCurrentInstance()?.appContext.config.globalProperties.trim
diff --git a/dolphinscheduler-ui/src/views/resource/file/create/use-create.ts b/dolphinscheduler-ui/src/views/resource/file/create/use-create.ts
index c080f945ca..c387723274 100644
--- a/dolphinscheduler-ui/src/views/resource/file/create/use-create.ts
+++ b/dolphinscheduler-ui/src/views/resource/file/create/use-create.ts
@@ -27,18 +27,17 @@ export function useCreate(state: any) {
const fileStore = useFileStore()
const handleCreateFile = () => {
- const pid = router.currentRoute.value.params.id || -1
+ // no more pid, as currentDir acts as the pid or parent path right now.
const currentDir = fileStore.getCurrentDir || '/'
state.fileFormRef.validate(async (valid: any) => {
if (!valid) {
await onlineCreateResource({
...state.fileForm,
- ...{ pid, currentDir }
+ ...{ currentDir }
})
window.$message.success(t('resource.file.success'))
- const name = pid ? 'resource-file-subdirectory' : 'file'
- router.push({ name, params: { id: pid } })
+ router.go(-1)
}
})
}
diff --git a/dolphinscheduler-ui/src/views/resource/file/edit/index.tsx b/dolphinscheduler-ui/src/views/resource/file/edit/index.tsx
index 73686ba053..36e112b409 100644
--- a/dolphinscheduler-ui/src/views/resource/file/edit/index.tsx
+++ b/dolphinscheduler-ui/src/views/resource/file/edit/index.tsx
@@ -32,21 +32,23 @@ export default defineComponent({
const router = useRouter()
const componentName = route.name
- const fileId = Number(route.params.id)
+ // fullname is now the id of resources
+ const fullName = String(router.currentRoute.value.query.prefix || "")
+ const tenantCode = String(router.currentRoute.value.query.tenantCode || "")
const { state } = useForm()
const { getResourceView, handleUpdateContent } = useEdit(state)
const handleFileContent = () => {
state.fileForm.content = resourceViewRef.state.value.content
- handleUpdateContent(fileId)
+ handleUpdateContent(fullName, tenantCode)
}
const handleReturn = () => {
router.go(-1)
}
- const resourceViewRef = getResourceView(fileId)
+ const resourceViewRef = getResourceView(fullName, tenantCode)
watch(
() => resourceViewRef.state.value.content,
() => (state.fileForm.content = resourceViewRef.state.value.content)
diff --git a/dolphinscheduler-ui/src/views/resource/file/edit/use-edit.ts b/dolphinscheduler-ui/src/views/resource/file/edit/use-edit.ts
index d2edaa9665..cb01b46895 100644
--- a/dolphinscheduler-ui/src/views/resource/file/edit/use-edit.ts
+++ b/dolphinscheduler-ui/src/views/resource/file/edit/use-edit.ts
@@ -28,25 +28,28 @@ export function useEdit(state: any) {
const { t } = useI18n()
const router: Router = useRouter()
- const getResourceView = (id: number) => {
+ const getResourceView = (fullName: string, tenantCode: string) => {
const params = {
skipLineNum: 0,
- limit: 3000
+ limit: 3000,
+ fullName: fullName,
+ tenantCode: tenantCode
}
- return useAsyncState(viewResource(params, id), {
+ return useAsyncState(viewResource(params), {
alias: '',
content: ''
})
}
- const handleUpdateContent = (id: number) => {
+ const handleUpdateContent = (fullName: string, tenantCode: string) => {
state.fileFormRef.validate(async (valid: any) => {
if (!valid) {
await updateResourceContent(
{
- ...state.fileForm
+ ...state.fileForm,
+ tenantCode: tenantCode,
+ fullName: fullName,
},
- id
)
window.$message.success(t('resource.file.success'))
diff --git a/dolphinscheduler-ui/src/views/resource/file/index.tsx b/dolphinscheduler-ui/src/views/resource/file/index.tsx
index 450bd95e31..1943a5d94e 100644
--- a/dolphinscheduler-ui/src/views/resource/file/index.tsx
+++ b/dolphinscheduler-ui/src/views/resource/file/index.tsx
@@ -42,8 +42,8 @@ import { useFileState } from './use-file'
import { BreadcrumbItem, IRenameFile } from './types'
import { useFileStore } from '@/store/file/file'
import {
- queryCurrentResourceById,
- queryResourceById
+ queryCurrentResourceByFullName,
+ queryCurrentResourceByFileName
} from '@/service/modules/resources'
import Card from '@/components/card'
import ResourceFolderModal from './folder'
@@ -57,8 +57,8 @@ export default defineComponent({
name: 'File',
setup() {
const router: Router = useRouter()
- const fileId = ref(Number(router.currentRoute.value.params.id) || -1)
-
+ const fullName = ref(String(router.currentRoute.value.query.prefix || ""))
+ const tenantCode = ref(String(router.currentRoute.value.query.tenantCode || ""))
const resourceListRef = ref()
const folderShowRef = ref(false)
const uploadShowRef = ref(false)
@@ -66,9 +66,10 @@ export default defineComponent({
const searchRef = ref()
const renameInfo = reactive({
- id: -1,
name: '',
- description: ''
+ description: '',
+ fullName: '',
+ user_name: ''
})
const paginationReactive = reactive({
@@ -81,7 +82,8 @@ export default defineComponent({
const handleUpdatePage = (page: number) => {
paginationReactive.page = page
resourceListRef.value = getResourceListState(
- fileId.value,
+ fullName.value,
+ tenantCode.value,
searchRef.value,
paginationReactive.page,
paginationReactive.pageSize
@@ -92,7 +94,8 @@ export default defineComponent({
paginationReactive.page = 1
paginationReactive.pageSize = pageSize
resourceListRef.value = getResourceListState(
- fileId.value,
+ fullName.value,
+ tenantCode.value,
searchRef.value,
paginationReactive.page,
paginationReactive.pageSize
@@ -111,7 +114,8 @@ export default defineComponent({
const handleConditions = () => {
resourceListRef.value = getResourceListState(
- fileId.value,
+ fullName.value,
+ tenantCode.value,
searchRef.value
)
}
@@ -121,12 +125,12 @@ export default defineComponent({
}
const handleCreateFile = () => {
- const name = fileId.value
+ const name = fullName.value
? 'resource-subfile-create'
: 'resource-file-create'
router.push({
name,
- params: { id: fileId.value }
+ params: { id: fullName.value }
})
}
@@ -134,10 +138,11 @@ export default defineComponent({
handleShowModal(uploadShowRef)
}
- const handleRenameFile: IRenameFile = (id, name, description) => {
- renameInfo.id = id
+ const handleRenameFile: IRenameFile = (name: string, description: string, fullName: string, user_name: string) => {
+ renameInfo.fullName = fullName
renameInfo.name = name
renameInfo.description = description
+ renameInfo.user_name = user_name
handleShowModal(renameShowRef)
}
@@ -149,75 +154,75 @@ export default defineComponent({
const updateList = () => {
resourceListRef.value = getResourceListState(
- fileId.value,
+ fullName.value,
+ tenantCode.value,
searchRef.value
)
}
const fileStore = useFileStore()
onMounted(() => {
- resourceListRef.value = getResourceListState(fileId.value)
+ resourceListRef.value = getResourceListState(fullName.value, tenantCode.value,searchRef.value)
})
const breadcrumbItemsRef: Ref<Array<BreadcrumbItem> | undefined> = ref([
{
id: 1,
- fullName: 'l1'
+ fullName: 'l1',
+ userName: 'u1'
},
{
id: 2,
- fullName: 'l2'
+ fullName: 'l2',
+ userName: 'u2'
},
{
id: 4,
- fullName: 'l3'
+ fullName: 'l3',
+ userName: 'u3'
}
])
const trim = getCurrentInstance()?.appContext.config.globalProperties.trim
onMounted(() => {
- const currFileId = Number(router.currentRoute.value.params.id) || -1
- if (currFileId === -1) {
+ const currfullName = String(router.currentRoute.value.query.prefix || "")
+ if (currfullName === "") {
fileStore.setCurrentDir('/')
} else {
- queryCurrentResourceById(currFileId).then((res: ResourceFile) => {
- if (res.fullName) {
- fileStore.setCurrentDir(res.fullName)
- }
- })
+ fileStore.setCurrentDir(currfullName)
}
})
const initBreadcrumb = async (dirs: string[]) => {
- let index = 0
- for (const dir of dirs) {
+ for (let index = 0; index < dirs.length; index ++) {
const newDir = dirs.slice(0, index + 1).join('/')
- if (newDir) {
- const id = 0
- const resource = await queryResourceById(
- {
- id,
- type: 'FILE',
- fullName: newDir
- },
- id
- )
- breadcrumbItemsRef.value?.push({ id: resource.id, fullName: dir })
- } else {
- breadcrumbItemsRef.value?.push({ id: 0, fullName: 'Root' })
+ const resource = await queryCurrentResourceByFileName(
+ {
+ type: 'FILE',
+ fileName: newDir+"/",
+ tenantCode: tenantCode.value
+ }
+ )
+ breadcrumbItemsRef.value?.push({ id: resource.fullName, fullName: resource.alias, userName: resource.userName })
}
- index = index + 1
- }
}
onMounted(() => {
breadcrumbItemsRef.value = []
- if (fileId.value != -1) {
- queryCurrentResourceById(fileId.value).then((res: ResourceFile) => {
- if (res.fullName) {
- const dirs = res.fullName.split('/')
+ if (fullName.value != "") {
+ breadcrumbItemsRef.value?.push({ id: 0, fullName: 'Root', userName: '' })
+ queryCurrentResourceByFullName(
+ {
+ type: 'FILE',
+ fullName: fullName.value,
+ tenantCode: tenantCode.value,
+ }
+ ).then((res: ResourceFile) => {
+ if (res.fileName) {
+ const dirs = res.fileName.split('/')
if (dirs && dirs.length > 1) {
+ dirs.pop()
initBreadcrumb(dirs)
}
}
@@ -226,7 +231,7 @@ export default defineComponent({
})
return {
- fileId,
+ fullName,
searchRef,
folderShowRef,
uploadShowRef,
@@ -307,7 +312,7 @@ export default defineComponent({
)
} else {
return (
- <NBreadcrumbItem href={item.id.toString()}>
+ <NBreadcrumbItem href={"0?prefix=" + item.id.toString() + "&tenantCode=" + item.userName}>
{item.fullName}
</NBreadcrumbItem>
)
@@ -353,9 +358,10 @@ export default defineComponent({
/>
<ResourceRenameModal
v-model:show={this.renameShowRef}
- id={this.renameInfo.id}
name={this.renameInfo.name}
+ fullName={this.renameInfo.fullName}
description={this.renameInfo.description}
+ userName={this.renameInfo.user_name}
onUpdateList={this.updateList}
/>
</NSpace>
diff --git a/dolphinscheduler-ui/src/views/resource/file/rename/index.tsx b/dolphinscheduler-ui/src/views/resource/file/rename/index.tsx
index 8124d29f14..b9f4ec01c5 100644
--- a/dolphinscheduler-ui/src/views/resource/file/rename/index.tsx
+++ b/dolphinscheduler-ui/src/views/resource/file/rename/index.tsx
@@ -32,10 +32,6 @@ const props = {
type: Boolean as PropType<boolean>,
default: false
},
- id: {
- type: Number as PropType<number>,
- default: -1
- },
name: {
type: String as PropType<string>,
default: ''
@@ -43,7 +39,15 @@ const props = {
description: {
type: String as PropType<string>,
default: ''
- }
+ },
+ fullName: {
+ type: String as PropType<string>,
+ default: ''
+ },
+ userName: {
+ type: String as PropType<string>,
+ default: ''
+ },
}
export default defineComponent({
@@ -51,9 +55,8 @@ export default defineComponent({
props,
emits: ['updateList', 'update:show'],
setup(props, ctx) {
- const { state, resetForm } = useForm(props.name, props.description)
+ const { state, resetForm } = useForm(props.fullName, props.name, props.description, props.userName)
const { handleRenameFile } = useRename(state)
-
const hideModal = () => {
ctx.emit('update:show', false)
}
@@ -67,9 +70,10 @@ export default defineComponent({
watch(
() => props.show,
() => {
- state.renameForm.id = props.id
+ state.renameForm.fullName = props.fullName
state.renameForm.name = props.name
state.renameForm.description = props.description
+ state.renameForm.user_name = props.userName
}
)
diff --git a/dolphinscheduler-ui/src/views/resource/file/rename/use-form.ts b/dolphinscheduler-ui/src/views/resource/file/rename/use-form.ts
index 567b23d14d..a849592c5c 100644
--- a/dolphinscheduler-ui/src/views/resource/file/rename/use-form.ts
+++ b/dolphinscheduler-ui/src/views/resource/file/rename/use-form.ts
@@ -19,23 +19,24 @@ import { reactive, ref, unref } from 'vue'
import { useI18n } from 'vue-i18n'
import type { FormRules } from 'naive-ui'
-const defaultValue = (name = '', description = '') => ({
+const defaultValue = (fullName = '',name = '', description = '', user_name = '') => ({
id: -1,
+ fullName,
name,
type: 'FILE',
- description
+ description,
+ user_name
})
-export function useForm(name: string, description: string) {
+export function useForm(fullName: string, name: string, description: string, user_name: string) {
const { t } = useI18n()
-
const resetForm = () => {
state.renameForm = Object.assign(unref(state.renameForm), defaultValue())
}
const state = reactive({
renameFormRef: ref(),
- renameForm: defaultValue(name, description),
+ renameForm: defaultValue(fullName, name, description, user_name),
saving: false,
rules: {
name: {
diff --git a/dolphinscheduler-ui/src/views/resource/file/rename/use-rename.ts b/dolphinscheduler-ui/src/views/resource/file/rename/use-rename.ts
index 103bbbe8e3..bd3a064c38 100644
--- a/dolphinscheduler-ui/src/views/resource/file/rename/use-rename.ts
+++ b/dolphinscheduler-ui/src/views/resource/file/rename/use-rename.ts
@@ -28,16 +28,14 @@ export function useRename(state: any) {
resetForm: () => void
) => {
await state.renameFormRef.validate()
-
if (state.saving) return
state.saving = true
-
try {
await updateResource(
{
- ...state.renameForm
- },
- state.renameForm.id
+ ...state.renameForm,
+ tenantCode: state.renameForm.user_name,
+ }
)
window.$message.success(t('resource.file.success'))
state.saving = false
diff --git a/dolphinscheduler-ui/src/views/resource/file/table/table-action.tsx b/dolphinscheduler-ui/src/views/resource/file/table/table-action.tsx
index e224e27645..ff4c5152d5 100644
--- a/dolphinscheduler-ui/src/views/resource/file/table/table-action.tsx
+++ b/dolphinscheduler-ui/src/views/resource/file/table/table-action.tsx
@@ -27,10 +27,9 @@ import {
} from '@vicons/antd'
import _ from 'lodash'
import { useI18n } from 'vue-i18n'
-import { ResourceFileTableData } from '../types'
+import { ResourceFileTableData, IRenameFile, IRtDisb } from '../types'
import { fileTypeArr } from '@/common/common'
import { downloadResource, deleteResource } from '@/service/modules/resources'
-import { IRenameFile, IRtDisb } from '../types'
import type { Router } from 'vue-router'
const props = {
@@ -41,9 +40,9 @@ const props = {
row: {
type: Object as PropType<ResourceFileTableData>,
default: {
- id: -1,
name: '',
- description: ''
+ description: '',
+ user_name: '',
}
}
}
@@ -63,16 +62,16 @@ export default defineComponent({
return !(flag && size < 1000000)
}
- const handleEditFile = (item: { id: number }) => {
- router.push({ name: 'resource-file-edit', params: { id: item.id } })
+ const handleEditFile = (item: {fullName: string, user_name: string }) => {
+ router.push({ name: 'resource-file-edit', query: {prefix: item.fullName, tenantCode: item.user_name} })
}
- const handleDeleteFile = (id: number) => {
- deleteResource(id).then(() => emit('updateList'))
+ const handleDeleteFile = (fullNameObj: {fullName: string, tenantCode: string}) => {
+ deleteResource(fullNameObj).then(() => emit('updateList'))
}
- const handleRenameFile: IRenameFile = (id, name, description) => {
- emit('renameResource', id, name, description)
+ const handleRenameFile: IRenameFile = (name: string, description: string, fullName: string, user_name: string) => {
+ emit('renameResource', name, description, fullName, user_name)
}
return {
@@ -98,7 +97,7 @@ export default defineComponent({
disabled={this.rtDisb(this.row.name, this.row.size)}
tag='div'
onClick={() => {
- this.handleEditFile(this.row)
+ this.handleEditFile({fullName:this.row.fullName, user_name:this.row.user_name})
}}
style={{ marginRight: '-5px' }}
circle
@@ -118,13 +117,16 @@ export default defineComponent({
<NButton
size='tiny'
type='info'
- onClick={() =>
+ onClick={() => {
this.handleRenameFile(
- this.row.id,
this.row.name,
- this.row.description
+ this.row.description,
+ this.row.fullName,
+ this.row.user_name
)
}
+
+ }
style={{ marginRight: '-5px' }}
circle
class='btn-rename'
@@ -147,7 +149,7 @@ export default defineComponent({
tag='div'
circle
style={{ marginRight: '-5px' }}
- onClick={() => downloadResource(this.row.id)}
+ onClick={() => downloadResource({fullName: this.row.fullName})}
class='btn-download'
>
<NIcon>
@@ -166,7 +168,7 @@ export default defineComponent({
positive-text={t('resource.file.confirm')}
negative-text={t('resource.file.cancel')}
onPositiveClick={() => {
- this.handleDeleteFile(this.row.id)
+ this.handleDeleteFile({fullName: this.row.fullName, tenantCode: this.row.user_name})
}}
>
{{
diff --git a/dolphinscheduler-ui/src/views/resource/file/table/use-table.ts b/dolphinscheduler-ui/src/views/resource/file/table/use-table.ts
index e236ecfa5f..1262a6f487 100644
--- a/dolphinscheduler-ui/src/views/resource/file/table/use-table.ts
+++ b/dolphinscheduler-ui/src/views/resource/file/table/use-table.ts
@@ -35,12 +35,11 @@ import type { TableColumns } from 'naive-ui/es/data-table/src/interface'
const goSubFolder = (router: Router, item: any) => {
const fileStore = useFileStore()
fileStore.setFileInfo(`${item.alias}|${item.size}`)
-
if (item.directory) {
fileStore.setCurrentDir(`${item.fullName}`)
- router.push({ name: 'resource-file-subdirectory', params: { id: item.id } })
+ router.push({ name: 'resource-file-subdirectory', query: { prefix: item.fullName, tenantCode: item.user_name}})
} else {
- router.push({ name: 'resource-file-list', params: { id: item.id } })
+ router.push({ name: 'resource-file-list', query: {prefix: item.fullName, tenantCode: item.user_name} })
}
}
@@ -72,7 +71,7 @@ export function useTable(renameResource: IRenameFile, updateList: () => void) {
)
},
{
- title: t('resource.file.user_name'),
+ title: t('resource.file.tenant_name'),
...COLUMN_WIDTH_CONFIG['userName'],
key: 'user_name'
},
@@ -86,12 +85,7 @@ export function useTable(renameResource: IRenameFile, updateList: () => void) {
{
title: t('resource.file.file_name'),
...COLUMN_WIDTH_CONFIG['name'],
- key: 'file_name'
- },
- {
- title: t('resource.file.description'),
- ...COLUMN_WIDTH_CONFIG['note'],
- key: 'description'
+ key: 'fullName'
},
{
title: t('resource.file.size'),
@@ -110,8 +104,9 @@ export function useTable(renameResource: IRenameFile, updateList: () => void) {
render: (row) =>
h(TableAction, {
row,
- onRenameResource: (id, name, description) =>
- renameResource(id, name, description),
+ onRenameResource: ( name, description, fullName, user_name ) => {
+ renameResource(name, description, fullName, user_name)
+ },
onUpdateList: () => updateList()
}),
...COLUMN_WIDTH_CONFIG['operation'](4)
diff --git a/dolphinscheduler-ui/src/views/resource/file/types.ts b/dolphinscheduler-ui/src/views/resource/file/types.ts
index 72f192a14d..e73ac607f8 100644
--- a/dolphinscheduler-ui/src/views/resource/file/types.ts
+++ b/dolphinscheduler-ui/src/views/resource/file/types.ts
@@ -16,8 +16,8 @@
*/
export interface ResourceFileTableData {
- id: number
name: string
+ fullName: string
user_name: string
directory: string
file_name: string
@@ -31,14 +31,14 @@ export interface IEmit {
}
export interface IRenameFile {
- (id: number, name: string, description: string): void
+ (name: string, description: string, fullName: string, user_name: string): void
}
export interface IRtDisb {
(name: string, size: number): boolean
}
export interface IResourceListState {
- (id?: number, searchVal?: string, pageNo?: number, pageSize?: number): any
+ (searchVal?: string, fullName?: string, tenantCode?: string, pageNo?: number, pageSize?: number): any
}
export interface BasicTableProps {
@@ -68,4 +68,5 @@ export interface ISetPagination {
export interface BreadcrumbItem {
id: number
fullName: string
+ userName: string
}
diff --git a/dolphinscheduler-ui/src/views/resource/file/upload/use-upload.ts b/dolphinscheduler-ui/src/views/resource/file/upload/use-upload.ts
index c8f3b05c9b..e4f4e6dfb8 100644
--- a/dolphinscheduler-ui/src/views/resource/file/upload/use-upload.ts
+++ b/dolphinscheduler-ui/src/views/resource/file/upload/use-upload.ts
@@ -17,14 +17,11 @@
import { useI18n } from 'vue-i18n'
import { IEmit } from '../types'
-import { useRouter } from 'vue-router'
-import type { Router } from 'vue-router'
import { useFileStore } from '@/store/file/file'
import { createResource } from '@/service/modules/resources'
export function useUpload(state: any) {
const { t } = useI18n()
- const router: Router = useRouter()
const fileStore = useFileStore()
const handleUploadFile = async (
@@ -37,13 +34,12 @@ export function useUpload(state: any) {
if (state.saving) return
state.saving = true
try {
- const pid = router.currentRoute.value.params.id || -1
+ // no more pid, as currentDir acts as the pid or parent path right now.
const currentDir = fileStore.getCurrentDir || '/'
const formData = new FormData()
formData.append('file', state.uploadForm.file)
formData.append('type', 'FILE')
formData.append('name', state.uploadForm.name)
- formData.append('pid', String(pid))
formData.append('currentDir', currentDir)
formData.append('description', state.uploadForm.description)
diff --git a/dolphinscheduler-ui/src/views/resource/file/use-file.ts b/dolphinscheduler-ui/src/views/resource/file/use-file.ts
index b16fb2b744..160a54057b 100644
--- a/dolphinscheduler-ui/src/views/resource/file/use-file.ts
+++ b/dolphinscheduler-ui/src/views/resource/file/use-file.ts
@@ -27,14 +27,16 @@ export function useFileState(
setPagination: ISetPagination = {} as ISetPagination
) {
const getResourceListState: IResourceListState = (
- id = -1,
+ fullName = '',
+ tenantCode = '',
searchVal = '',
pageNo = 1,
pageSize = 10
) => {
const { state } = useAsyncState(
queryResourceListPaging({
- id,
+ fullName,
+ tenantCode,
type: 'FILE',
searchVal,
pageNo,
@@ -66,12 +68,14 @@ export function useFileState(
return state
}
- const getResourceView = (id: number) => {
+ const getResourceView = (fullName: string, tenantCode: string) => {
const params = {
skipLineNum: 0,
- limit: 3000
+ limit: 3000,
+ fullName: fullName,
+ tenantCode: tenantCode
}
- const { state } = useAsyncState(viewResource(params, id), {})
+ const { state } = useAsyncState(viewResource(params), {})
return state
}
diff --git a/dolphinscheduler-ui/src/views/resource/udf/function/components/function-modal.tsx b/dolphinscheduler-ui/src/views/resource/udf/function/components/function-modal.tsx
index 8f05cbd0ac..d37a4e4c9b 100644
--- a/dolphinscheduler-ui/src/views/resource/udf/function/components/function-modal.tsx
+++ b/dolphinscheduler-ui/src/views/resource/udf/function/components/function-modal.tsx
@@ -106,6 +106,7 @@ export default defineComponent({
state.functionForm.funcName = props.row.funcName
state.functionForm.className = props.row.className
state.functionForm.resourceId = props.row.resourceId || -1
+ state.functionForm.fullName = props.row.resourceName || ""
state.functionForm.description = props.row.description
}
)
@@ -124,7 +125,6 @@ export default defineComponent({
},
render() {
const { t } = useI18n()
-
return (
<Modal
show={this.$props.show}
@@ -173,18 +173,19 @@ export default defineComponent({
</NFormItem>
<NFormItem
label={t('resource.function.udf_resources')}
- path='resourceId'
+ path='fullName'
>
<NInputGroup>
<NTreeSelect
options={this.udfResourceList}
- label-field='fullName'
- key-field='id'
- v-model={[this.functionForm.resourceId, 'value']}
+ label-field='name'
+ key-field='fullName'
+ check-strategy='child'
+ v-model={[this.functionForm.fullName, 'value']}
placeholder={t(
'resource.function.enter_select_udf_resources_tips'
)}
- defaultValue={this.functionForm.resourceId}
+ defaultValue={this.functionForm.fullName}
disabled={this.uploadShow}
showPath={false}
class='btn-udf-resource-dropdown'
diff --git a/dolphinscheduler-ui/src/views/resource/udf/function/components/use-form.ts b/dolphinscheduler-ui/src/views/resource/udf/function/components/use-form.ts
index 38e8f1ec3e..a1cedc2ab7 100644
--- a/dolphinscheduler-ui/src/views/resource/udf/function/components/use-form.ts
+++ b/dolphinscheduler-ui/src/views/resource/udf/function/components/use-form.ts
@@ -31,7 +31,8 @@ export const useForm = () => {
argTypes: '',
database: '',
description: '',
- resourceId: -1
+ resourceId: -1,
+ fullName: ''
},
saving: false,
rules: {
@@ -63,13 +64,22 @@ export const useForm = () => {
}
},
resourceId: {
- required: true,
+ required: false,
trigger: ['input', 'blur'],
validator() {
if (state.functionForm.resourceId === -1) {
return new Error(t('resource.function.enter_name_tips'))
}
}
+ },
+ fullName: {
+ required: true,
+ trigger: ['input', 'blur'],
+ validator() {
+ if (state.functionForm.fullName == "") {
+ return new Error(t('resource.function.enter_name_tips'))
+ }
+ }
}
} as FormRules
})
diff --git a/dolphinscheduler-ui/src/views/resource/udf/function/components/use-modal.ts b/dolphinscheduler-ui/src/views/resource/udf/function/components/use-modal.ts
index 3de23cd7e5..c3a52a6d8e 100644
--- a/dolphinscheduler-ui/src/views/resource/udf/function/components/use-modal.ts
+++ b/dolphinscheduler-ui/src/views/resource/udf/function/components/use-modal.ts
@@ -39,8 +39,7 @@ export function useModal(
await createUdfFunc(
{
...state.functionForm
- },
- state.functionForm.resourceId
+ }
)
)
}
@@ -52,7 +51,6 @@ export function useModal(
...state.functionForm,
id
},
- state.functionForm.resourceId,
id
)
})
@@ -84,9 +82,6 @@ export function useModal(
const filterEmptyDirectory = (list: any) => {
for (const item of list) {
if (item.children) {
- if (!/\.jar$/.test(item.name)) {
- item.disabled = true
- }
filterEmptyDirectory(item.children)
}
}
@@ -122,7 +117,7 @@ export function useModal(
const getUdfList = () => {
const { state } = useAsyncState(
- queryResourceList({ type: 'UDF' }).then((res: any) => {
+ queryResourceList({ type: 'UDF', fullName: "" }).then((res: any) => {
let item = res
let item1 = _.cloneDeep(res)
diff --git a/dolphinscheduler-ui/src/views/resource/udf/function/index.tsx b/dolphinscheduler-ui/src/views/resource/udf/function/index.tsx
index 1ec58e33a4..3df5275492 100644
--- a/dolphinscheduler-ui/src/views/resource/udf/function/index.tsx
+++ b/dolphinscheduler-ui/src/views/resource/udf/function/index.tsx
@@ -47,6 +47,7 @@ export default defineComponent({
const requestData = () => {
getTableData({
id: variables.id,
+ fullName: variables.fullName,
pageSize: variables.pageSize,
pageNo: variables.page,
searchVal: variables.searchVal
diff --git a/dolphinscheduler-ui/src/views/resource/udf/function/types.ts b/dolphinscheduler-ui/src/views/resource/udf/function/types.ts
index 2035418d89..0e507676d9 100644
--- a/dolphinscheduler-ui/src/views/resource/udf/function/types.ts
+++ b/dolphinscheduler-ui/src/views/resource/udf/function/types.ts
@@ -16,6 +16,7 @@
*/
export interface IUdfFunctionParam {
+ fullName: string
id: number
pageSize: number
pageNo: number
diff --git a/dolphinscheduler-ui/src/views/resource/udf/function/use-table.ts b/dolphinscheduler-ui/src/views/resource/udf/function/use-table.ts
index a7d737c6db..75a53b1507 100644
--- a/dolphinscheduler-ui/src/views/resource/udf/function/use-table.ts
+++ b/dolphinscheduler-ui/src/views/resource/udf/function/use-table.ts
@@ -43,7 +43,9 @@ export function useTable() {
tableWidth: DefaultTableWidth,
row: {},
tableData: [],
+ // here is id not prefix because udf function is still stored in db
id: ref(Number(router.currentRoute.value.params.id) || -1),
+ fullName: ref(String(router.currentRoute.value.query.prefix || "")),
page: ref(1),
pageSize: ref(10),
searchVal: ref(),
@@ -129,7 +131,7 @@ export function useTable() {
NPopconfirm,
{
onPositiveClick: () => {
- handleDelete(row.id)
+ handleDelete(row.id, row.fullName)
}
},
{
@@ -188,14 +190,15 @@ export function useTable() {
variables.row = row
}
- const handleDelete = (id: number) => {
+ const handleDelete = (id: number, fullName: string) => {
/* after deleting data from the current page, you need to jump forward when the page is empty. */
if (variables.tableData.length === 1 && variables.page > 1) {
variables.page -= 1
}
- deleteUdfFunc(id).then(() =>
+ deleteUdfFunc(id, {fullName: fullName}).then(() =>
getTableData({
+ fullName: variables.fullName,
id: variables.id,
pageSize: variables.pageSize,
pageNo: variables.page,
diff --git a/dolphinscheduler-ui/src/views/resource/udf/resource/components/folder-modal.tsx b/dolphinscheduler-ui/src/views/resource/udf/resource/components/folder-modal.tsx
index 89e253abea..a116ef970f 100644
--- a/dolphinscheduler-ui/src/views/resource/udf/resource/components/folder-modal.tsx
+++ b/dolphinscheduler-ui/src/views/resource/udf/resource/components/folder-modal.tsx
@@ -60,7 +60,7 @@ export default defineComponent({
}
const handleRename = () => {
- handleRenameResource(props.row.id)
+ handleRenameResource(props.row.fullName)
}
const trim = getCurrentInstance()?.appContext.config.globalProperties.trim
@@ -72,7 +72,7 @@ export default defineComponent({
state.folderForm.description = props.row.description
}
)
- const fileEdit = computed(() => props.row.id && !props.row.directory)
+ const fileEdit = computed(() => props.row.fullName && !props.row.directory)
return {
fileEdit,
@@ -90,10 +90,10 @@ export default defineComponent({
<Modal
show={this.$props.show}
title={
- this.row.id ? t('resource.udf.edit') : t('resource.udf.create_folder')
+ this.row.fullName ? t('resource.udf.edit') : t('resource.udf.create_folder')
}
onCancel={this.hideModal}
- onConfirm={this.row.id ? this.handleRename : this.handleCreate}
+ onConfirm={this.row.fullName ? this.handleRename : this.handleCreate}
confirmClassName='btn-submit'
cancelClassName='btn-cancel'
confirmLoading={this.saving}
diff --git a/dolphinscheduler-ui/src/views/resource/udf/resource/components/use-modal.ts b/dolphinscheduler-ui/src/views/resource/udf/resource/components/use-modal.ts
index d1565cbb3e..f8df1e474e 100644
--- a/dolphinscheduler-ui/src/views/resource/udf/resource/components/use-modal.ts
+++ b/dolphinscheduler-ui/src/views/resource/udf/resource/components/use-modal.ts
@@ -36,8 +36,8 @@ export function useModal(
const handleCreateResource = async () => {
const pid = router.currentRoute.value.params.id || -1
- const currentDir = pid === -1 ? '/' : fileStore.getCurrentDir || '/'
-
+ const currentFullName = String(router.currentRoute.value.query.prefix || "")
+ const currentDir = currentFullName == "" ? '/' : fileStore.getCurrentDir || '/'
submitRequest(
async () =>
await createDirectory({
@@ -47,21 +47,19 @@ export function useModal(
)
}
- const handleRenameResource = async (id: number) => {
+ const handleRenameResource = async (fullName: string) => {
submitRequest(async () => {
await updateResource(
{
+ fullName: fullName,
...state.folderForm,
- ...{ id }
- },
- id
+ }
)
})
}
const submitRequest = async (serviceHandle: any) => {
await state.folderFormRef.validate()
-
if (state.saving) return
state.saving = true
@@ -89,8 +87,9 @@ export function useModal(
state.saving = true
try {
- const pid = router.currentRoute.value.params.id || -1
- const currentDir = pid === -1 ? '/' : fileStore.getCurrentDir || '/'
+ const pid = Number(router.currentRoute.value.params.id) || "-1"
+ const currentFullName = String(router.currentRoute.value.query.prefix || "")
+ const currentDir = currentFullName == "" ? '/' : fileStore.getCurrentDir || '/'
const formData = new FormData()
formData.append('file', state.uploadForm.file)
diff --git a/dolphinscheduler-ui/src/views/resource/udf/resource/index.tsx b/dolphinscheduler-ui/src/views/resource/udf/resource/index.tsx
index 603bc0300c..6290f60dc4 100644
--- a/dolphinscheduler-ui/src/views/resource/udf/resource/index.tsx
+++ b/dolphinscheduler-ui/src/views/resource/udf/resource/index.tsx
@@ -50,7 +50,9 @@ export default defineComponent({
const requestData = () => {
getTableData({
- id: variables.id,
+ id: -1,
+ fullName: variables.fullName,
+ tenantCode: variables.tenantCode,
pageSize: variables.pageSize,
pageNo: variables.page,
searchVal: variables.searchVal
@@ -88,7 +90,7 @@ export default defineComponent({
let breadName = ''
variables.breadList.forEach((item, i) => {
if (i <= index) {
- breadName = breadName + '/' + item
+ breadName = breadName === "" ? item.toString() : breadName + '/' + item.toString();
}
})
goBread(breadName)
diff --git a/dolphinscheduler-ui/src/views/resource/udf/resource/types.ts b/dolphinscheduler-ui/src/views/resource/udf/resource/types.ts
index b2cfbe57c5..325814b0c2 100644
--- a/dolphinscheduler-ui/src/views/resource/udf/resource/types.ts
+++ b/dolphinscheduler-ui/src/views/resource/udf/resource/types.ts
@@ -16,6 +16,8 @@
*/
export interface IUdfResourceParam {
+ fullName: string
+ tenantCode: string
id: number
pageSize: number
pageNo: number
diff --git a/dolphinscheduler-ui/src/views/resource/udf/resource/use-table.ts b/dolphinscheduler-ui/src/views/resource/udf/resource/use-table.ts
index 4001a1f303..e4a20e74e0 100644
--- a/dolphinscheduler-ui/src/views/resource/udf/resource/use-table.ts
+++ b/dolphinscheduler-ui/src/views/resource/udf/resource/use-table.ts
@@ -30,7 +30,8 @@ import {
queryResourceListPaging,
downloadResource,
deleteResource,
- queryResourceById
+ queryCurrentResourceByFileName,
+ queryCurrentResourceByFullName
} from '@/service/modules/resources'
import ButtonLink from '@/components/button-link'
import {
@@ -39,14 +40,15 @@ import {
DefaultTableWidth
} from '@/common/column-width-config'
import type { IUdfResourceParam } from './types'
+import { ResourceFile } from '@/service/modules/resources/types'
const goSubFolder = (router: Router, item: any) => {
const fileStore = useFileStore()
fileStore.setFileInfo(`${item.alias}|${item.size}`)
-
if (item.directory) {
fileStore.setCurrentDir(`${item.fullName}`)
- router.push({ name: 'resource-sub-manage', params: { id: item.id } })
+ router.push({ name: 'resource-sub-manage',
+ query: {prefix: item.fullName, tenantCode: item.userName} })
}
}
@@ -60,8 +62,9 @@ export function useTable() {
tableWidth: DefaultTableWidth,
row: {},
tableData: [],
- breadList: [],
- id: ref(Number(router.currentRoute.value.params.id) || -1),
+ breadList: [] as String[],
+ fullName: ref(String(router.currentRoute.value.query.prefix || "")),
+ tenantCode: ref(String(router.currentRoute.value.query.tenantCode || "")),
page: ref(1),
pageSize: ref(10),
searchVal: ref(),
@@ -103,7 +106,7 @@ export function useTable() {
}
},
{
- title: t('resource.udf.user_name'),
+ title: t('resource.udf.tenant_name'),
...COLUMN_WIDTH_CONFIG['userName'],
key: 'userName'
},
@@ -117,7 +120,7 @@ export function useTable() {
{
title: t('resource.udf.file_name'),
...COLUMN_WIDTH_CONFIG['name'],
- key: 'fileName'
+ key: 'fullName'
},
{
title: t('resource.udf.file_size'),
@@ -125,11 +128,6 @@ export function useTable() {
...COLUMN_WIDTH_CONFIG['size'],
render: (row) => bytesToSize(row.size)
},
- {
- title: t('resource.udf.description'),
- key: 'description',
- ...COLUMN_WIDTH_CONFIG['note']
- },
{
title: t('resource.udf.create_time'),
key: 'createTime',
@@ -185,7 +183,7 @@ export function useTable() {
size: 'tiny',
class: 'btn-download',
disabled: row?.directory ? true : false,
- onClick: () => downloadResource(row.id)
+ onClick: () => downloadResource({fullName: row.fullName})
},
{
icon: () => h(DownloadOutlined)
@@ -198,7 +196,7 @@ export function useTable() {
NPopconfirm,
{
onPositiveClick: () => {
- handleDelete(row.id)
+ handleDelete({fullName: row.fullName, tenantCode: row.userName})
}
},
{
@@ -242,13 +240,25 @@ export function useTable() {
variables.loadingRef = true
const { state } = useAsyncState(
queryResourceListPaging({ ...params, type: 'UDF' }).then((res: any) => {
- const breadList =
- variables.id === -1
- ? []
- : (fileStore.getCurrentDir.split('/') as Array<never>)
- breadList.shift()
-
- variables.breadList = breadList
+ // use strict checking here
+ if (variables.fullName !== ""){
+ queryCurrentResourceByFullName(
+ {
+ type: 'UDF',
+ fullName: variables.fullName,
+ tenantCode: variables.tenantCode,
+ }
+ ).then((res: ResourceFile) => {
+ if (res.fileName) {
+ const breadList = res.fileName.split('/')
+ // pop the alias from the fullname path
+ breadList.pop()
+ variables.breadList = breadList
+ }
+ })
+ } else {
+ variables.breadList = []
+ }
variables.totalPage = res.totalPage
variables.tableData = res.totalList.map((item: any) => {
return { ...item }
@@ -265,15 +275,17 @@ export function useTable() {
variables.row = row
}
- const handleDelete = (id: number) => {
+ const handleDelete = (fullNameObj: {fullName: string, tenantCode: string}) => {
/* after deleting data from the current page, you need to jump forward when the page is empty. */
if (variables.tableData.length === 1 && variables.page > 1) {
variables.page -= 1
}
- deleteResource(id).then(() =>
+ deleteResource(fullNameObj).then(() =>
getTableData({
- id: variables.id,
+ id: -1,
+ fullName: variables.fullName,
+ tenantCode: variables.tenantCode,
pageSize: variables.pageSize,
pageNo: variables.page,
searchVal: variables.searchVal
@@ -285,18 +297,16 @@ export function useTable() {
router.push({ name: 'resource-manage' })
}
- const goBread = (fullName: string) => {
- const { id } = variables
- queryResourceById(
+ const goBread = (fileName: string) => {
+ queryCurrentResourceByFileName(
{
- id,
type: 'UDF',
- fullName
- },
- id
+ fileName: fileName + "/",
+ tenantCode: variables.tenantCode
+ }
).then((res: any) => {
fileStore.setCurrentDir(res.fullName)
- router.push({ name: 'resource-sub-manage', params: { id: res.id } })
+ router.push({ name: 'resource-sub-manage', query: {prefix: res.fullName, tenantCode: res.userName} })
})
}
diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java
index ff08b6d09b..7ec5279ce6 100644
--- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java
+++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/utils/TaskExecutionCheckerUtils.java
@@ -118,18 +118,20 @@ public class TaskExecutionCheckerUtils {
if (CollectionUtils.isNotEmpty(downloadFiles)) {
for (Pair<String, String> fileDownload : downloadFiles) {
try {
- // query the tenant code of the resource according to the name of the resource
String fullName = fileDownload.getLeft();
+ // we do not actually get & need tenantCode with this implementation right now.
String tenantCode = fileDownload.getRight();
- String resPath = storageOperate.getResourceFileName(tenantCode, fullName);
- logger.info("get resource file from path:{}", resPath);
+ // TODO: Need a better way to get fileName because this implementation is tricky.
+ String fileName = storageOperate.getResourceFileName(fullName);
+ logger.info("get resource file from path:{}", fullName);
+
long resourceDownloadStartTime = System.currentTimeMillis();
- storageOperate.download(tenantCode, resPath, execLocalPath + File.separator + fullName, false,
+ storageOperate.download(tenantCode, fullName, execLocalPath + File.separator + fileName, false,
true);
WorkerServerMetrics
.recordWorkerResourceDownloadTime(System.currentTimeMillis() - resourceDownloadStartTime);
WorkerServerMetrics.recordWorkerResourceDownloadSize(
- Files.size(Paths.get(execLocalPath, fullName)));
+ Files.size(Paths.get(execLocalPath, fileName)));
WorkerServerMetrics.incWorkerResourceDownloadSuccessCount();
} catch (Exception e) {
WorkerServerMetrics.incWorkerResourceDownloadFailureCount();