You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by jo...@apache.org on 2020/08/14 08:47:09 UTC
[incubator-dolphinscheduler] branch 1.3.2-release updated:
[Fix-3469][api]Should filter the resource by the different program type
(#3498)
This is an automated email from the ASF dual-hosted git repository.
journey pushed a commit to branch 1.3.2-release
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/1.3.2-release by this push:
new 5f5c084 [Fix-3469][api]Should filter the resource by the different program type (#3498)
5f5c084 is described below
commit 5f5c08402fdcecca8c35f4dc3021cc089949ef13
Author: lgcareer <18...@163.com>
AuthorDate: Fri Aug 14 16:47:01 2020 +0800
[Fix-3469][api]Should filter the resource by the different program type (#3498)
* [Fix-3463][api]Fixed that run the sql task will be failure after rename the udf resource
* [Fix-3469][api]Should list python file and jar file
* [Fix-3469][api]Should filter the resource by the different program type
* [Fix-3469][api]fix the code smell
---
.../api/controller/ResourcesController.java | 13 +++++++----
.../api/dto/resources/filter/ResourceFilter.java | 25 ++++++++++++++++------
.../api/service/ResourcesService.java | 19 +++++++++++++---
.../api/controller/ResourcesControllerTest.java | 25 ++++++++++++++++++++++
.../dto/resources/filter/ResourceFilterTest.java | 8 +++++--
.../dolphinscheduler/dao/upgrade/UpgradeDao.java | 8 +++++++
6 files changed, 82 insertions(+), 16 deletions(-)
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
index 2f2cb9c..697d364 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/ResourcesController.java
@@ -27,6 +27,7 @@ import org.apache.dolphinscheduler.api.service.ResourcesService;
import org.apache.dolphinscheduler.api.service.UdfFuncService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.ParameterUtils;
@@ -275,7 +276,7 @@ public class ResourcesController extends BaseController {
* @param type resource type
* @return resource list
*/
- @ApiOperation(value = "queryResourceJarList", notes = "QUERY_RESOURCE_LIST_NOTES")
+ @ApiOperation(value = "queryResourceByProgramType", notes = "QUERY_RESOURCE_LIST_NOTES")
@ApiImplicitParams({
@ApiImplicitParam(name = "type", value = "RESOURCE_TYPE", required = true, dataType = "ResourceType")
})
@@ -283,10 +284,14 @@ public class ResourcesController extends BaseController {
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_RESOURCES_LIST_ERROR)
public Result queryResourceJarList(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
- @RequestParam(value = "type") ResourceType type
+ @RequestParam(value = "type") ResourceType type,
+ @RequestParam(value = "programType",required = false) ProgramType programType
) {
- logger.info("query resource list, login user:{}, resource type:{}", loginUser.getUserName(), type.toString());
- Map<String, Object> result = resourceService.queryResourceJarList(loginUser, type);
+ String programTypeName = programType == null ? "" : programType.name();
+ String userName = loginUser.getUserName();
+ userName = userName.replaceAll("[\n|\r|\t]", "_");
+ logger.info("query resource list, login user:{}, resource type:{}, program type:{}", userName,programTypeName);
+ Map<String, Object> result = resourceService.queryResourceByProgramType(loginUser, type,programType);
return returnDataList(result);
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java
index c918a16..9c3a84c 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java
@@ -31,24 +31,29 @@ public class ResourceFilter implements IFilter {
/**
* resource suffix
*/
- private String suffix;
+ private Set<String> suffixs = new HashSet<>();
/**
* resource list
*/
private List<Resource> resourceList;
/**
- * parent list
+ * constructor
+ * @param suffix resource suffix
+ * @param resourceList resource list
*/
- //Set<Resource> parentList = new HashSet<>();
+ public ResourceFilter(String suffix, List<Resource> resourceList) {
+ this.suffixs.add(suffix);
+ this.resourceList = resourceList;
+ }
/**
* constructor
- * @param suffix resource suffix
+ * @param suffixs resource suffixs
* @param resourceList resource list
*/
- public ResourceFilter(String suffix, List<Resource> resourceList) {
- this.suffix = suffix;
+ public ResourceFilter(Set<String> suffixs, List<Resource> resourceList) {
+ this.suffixs = suffixs;
this.resourceList = resourceList;
}
@@ -59,7 +64,13 @@ public class ResourceFilter implements IFilter {
public Set<Resource> fileFilter(){
Set<Resource> resources = resourceList.stream().filter(t -> {
String alias = t.getAlias();
- return alias.endsWith(suffix);
+ boolean result = false;
+ for (String suffix : suffixs) {
+ if (alias.endsWith(suffix)) {
+ result = true;
+ }
+ }
+ return result;
}).collect(Collectors.toSet());
return resources;
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
index e1a9c85..f66e035 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
@@ -30,6 +30,7 @@ import org.apache.dolphinscheduler.api.exceptions.ServiceException;
import org.apache.dolphinscheduler.api.utils.PageInfo;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.utils.*;
import org.apache.dolphinscheduler.dao.entity.*;
@@ -654,21 +655,33 @@ public class ResourcesService extends BaseService {
}
/**
- * query resource list
+ * query resource list by program type
*
* @param loginUser login user
* @param type resource type
* @return resource list
*/
- public Map<String, Object> queryResourceJarList(User loginUser, ResourceType type) {
+ public Map<String, Object> queryResourceByProgramType(User loginUser, ResourceType type, ProgramType programType) {
Map<String, Object> result = new HashMap<>(5);
+ String suffix = ".jar";
int userId = loginUser.getId();
if(isAdmin(loginUser)){
userId = 0;
}
+ if (programType != null) {
+ switch (programType) {
+ case JAVA:
+ break;
+ case SCALA:
+ break;
+ case PYTHON:
+ suffix = ".py";
+ break;
+ }
+ }
List<Resource> allResourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal(),0);
- List<Resource> resources = new ResourceFilter(".jar",new ArrayList<>(allResourceList)).filter();
+ List<Resource> resources = new ResourceFilter(suffix,new ArrayList<>(allResourceList)).filter();
Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources);
result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren());
putMsg(result,Status.SUCCESS);
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
index a56e3f8..a2ebd27 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/controller/ResourcesControllerTest.java
@@ -19,6 +19,7 @@ package org.apache.dolphinscheduler.api.controller;
import com.alibaba.fastjson.JSON;
import org.apache.dolphinscheduler.api.enums.Status;
import org.apache.dolphinscheduler.api.utils.Result;
+import org.apache.dolphinscheduler.common.enums.ProgramType;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.enums.UdfType;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
@@ -452,4 +453,28 @@ public class ResourcesControllerTest extends AbstractControllerTest{
Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
logger.info(mvcResult.getResponse().getContentAsString());
}
+
+ @Test
+ public void testqueryResourceJarList() throws Exception {
+
+ MultiValueMap<String, String> paramsMap = new LinkedMultiValueMap<>();
+ paramsMap.add("type", ResourceType.FILE.name());
+ //paramsMap.add("programType", ProgramType.PYTHON.name());
+ paramsMap.add("programType", "JAVA");
+
+
+ MvcResult mvcResult = mockMvc.perform(get("/resources/list/jar")
+ .header(SESSION_ID, sessionId)
+ .params(paramsMap))
+ .andExpect(status().isOk())
+ .andExpect(content().contentType(MediaType.APPLICATION_JSON_UTF8))
+ .andReturn();
+
+ Result result = JSONUtils.parseObject(mvcResult.getResponse().getContentAsString(), Result.class);
+ result.getCode().equals(Status.SUCCESS.getCode());
+ JSONObject object = (JSONObject) JSON.parse(mvcResult.getResponse().getContentAsString());
+
+ Assert.assertEquals(Status.SUCCESS.getCode(),result.getCode().intValue());
+ logger.info(mvcResult.getResponse().getContentAsString());
+ }
}
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java
index 8a4a16c..8e3be6a 100644
--- a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java
@@ -23,7 +23,9 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
/**
* resource filter test
@@ -48,9 +50,11 @@ public class ResourceFilterTest {
allList.add(resource5);
allList.add(resource6);
allList.add(resource7);
+ Set<String> suffixSet = new HashSet<>();
+ suffixSet.add(".jar");
+ suffixSet.add(".txt");
-
- ResourceFilter resourceFilter = new ResourceFilter(".jar",allList);
+ ResourceFilter resourceFilter = new ResourceFilter(suffixSet,allList);
List<Resource> resourceList = resourceFilter.filter();
Assert.assertNotNull(resourceList);
resourceList.stream().forEach(t-> logger.info(t.toString()));
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
index 3d35b68..d111a8c 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/UpgradeDao.java
@@ -343,6 +343,14 @@ public abstract class UpgradeDao extends AbstractBaseDao {
if (param != null) {
List<ResourceInfo> resourceList = JSONUtils.toList(param.getString("resourceList"), ResourceInfo.class);
+ ResourceInfo mainJar = JSONUtils.parseObject(param.getString("mainJar"), ResourceInfo.class);
+ if (mainJar != null && mainJar.getId() == 0) {
+ String fullName = mainJar.getRes().startsWith("/") ? mainJar.getRes() : String.format("/%s",mainJar.getRes());
+ if (resourcesMap.containsKey(fullName)) {
+ mainJar.setId(resourcesMap.get(fullName));
+ param.put("mainJar",JSONUtils.parseObject(JSONObject.toJSONString(mainJar)));
+ }
+ }
if (CollectionUtils.isNotEmpty(resourceList)) {
List<ResourceInfo> newResourceList = resourceList.stream().map(resInfo -> {