You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by jo...@apache.org on 2020/03/18 10:38:54 UTC
[incubator-dolphinscheduler] branch dev-resource-tree updated:
choose main jar with resource tree (#2220)
This is an automated email from the ASF dual-hosted git repository.
journey pushed a commit to branch dev-resource-tree
in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
The following commit(s) were added to refs/heads/dev-resource-tree by this push:
new 1ed4656 choose main jar with resource tree (#2220)
1ed4656 is described below
commit 1ed4656e2d3f0b75e9a19c813e0a211dc1229dbd
Author: lgcareer <18...@163.com>
AuthorDate: Wed Mar 18 18:38:42 2020 +0800
choose main jar with resource tree (#2220)
* update resource service test
* Fix github action rerun failed
* add status of PARENT_RESOURCE_NOT_EXIST
* build resource process definition map
* update resource name also need update all the children full name
* need add queryResource
* add queryResourceJarList
* add queryResourceJarList
* add queryResourceJarList
* add resource filter in order to get filtered resource
* add comments of resource filter
* update list children by resource
---
.../api/dto/resources/filter/IFilter.java | 28 ++++++
.../api/dto/resources/filter/ResourceFilter.java | 99 ++++++++++++++++++++++
.../dto/resources/visitor/ResourceTreeVisitor.java | 27 +++---
.../api/service/ResourcesService.java | 24 ++----
.../dto/resources/filter/ResourceFilterTest.java | 56 ++++++++++++
.../dolphinscheduler/dao/entity/Resource.java | 8 ++
6 files changed, 215 insertions(+), 27 deletions(-)
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/IFilter.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/IFilter.java
new file mode 100644
index 0000000..ce6ce3a
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/IFilter.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.api.dto.resources.filter;
+
+import org.apache.dolphinscheduler.dao.entity.Resource;
+
+import java.util.List;
+
+/**
+ * interface filter
+ */
+public interface IFilter {
+ List<Resource> filter();
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java
new file mode 100644
index 0000000..53f3d78
--- /dev/null
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilter.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.api.dto.resources.filter;
+
+import org.apache.dolphinscheduler.dao.entity.Resource;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * resource filter
+ */
+public class ResourceFilter implements IFilter {
+ /**
+ * resource suffix
+ */
+ private String suffix;
+ /**
+ * resource list
+ */
+ private List<Resource> resourceList;
+
+ /**
+ * constructor
+ * @param suffix resource suffix
+ * @param resourceList resource list
+ */
+ public ResourceFilter(String suffix, List<Resource> resourceList) {
+ this.suffix = suffix;
+ this.resourceList = resourceList;
+ }
+
+ /**
+ * file filter
+ * @return file filtered by suffix
+ */
+ public Set<Resource> fileFilter(){
+ Set<Resource> resources = resourceList.stream().filter(t -> {
+ String alias = t.getAlias();
+ return alias.endsWith(suffix);
+ }).collect(Collectors.toSet());
+ return resources;
+ }
+
+ /**
+ * list all parent dir
+ * @return parent resource dir set
+ */
+ Set<Resource> listAllParent(){
+ Set<Resource> parentList = new HashSet<>();
+ Set<Resource> filterFileList = fileFilter();
+ for(Resource file:filterFileList){
+ parentList.add(file);
+ parentList.addAll(listAllParent(file));
+ //listAllParent(file);
+ }
+ return parentList;
+
+ }
+
+ /**
+ * list all parent dir
+ * @param resource resource
+ * @return parent resource dir set
+ */
+ Set<Resource> listAllParent(Resource resource){
+ Set<Resource> parentList = new HashSet<>();
+ for (Resource resourceTemp : resourceList) {
+ if (resourceTemp.getId() == resource.getPid()) {
+ parentList.add(resourceTemp);
+ listAllParent(resourceTemp);
+ }
+ }
+ return parentList;
+
+ }
+
+ @Override
+ public List<Resource> filter() {
+ return new ArrayList<>(listAllParent());
+ }
+}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java
index db17921..5cf1188 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/dto/resources/visitor/ResourceTreeVisitor.java
@@ -1,14 +1,3 @@
-package org.apache.dolphinscheduler.api.dto.resources.visitor;
-
-
-import org.apache.dolphinscheduler.api.dto.resources.Directory;
-import org.apache.dolphinscheduler.api.dto.resources.FileLeaf;
-import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
-import org.apache.dolphinscheduler.dao.entity.Resource;
-
-import java.util.ArrayList;
-import java.util.List;
-
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -25,6 +14,20 @@ import java.util.List;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+package org.apache.dolphinscheduler.api.dto.resources.visitor;
+
+
+import org.apache.dolphinscheduler.api.dto.resources.Directory;
+import org.apache.dolphinscheduler.api.dto.resources.FileLeaf;
+import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
+import org.apache.dolphinscheduler.dao.entity.Resource;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * resource tree visitor
+ */
public class ResourceTreeVisitor implements Visitor{
/**
@@ -115,7 +118,7 @@ public class ResourceTreeVisitor implements Visitor{
tempResourceComponent = new FileLeaf();
}
tempResourceComponent.setName(resource.getAlias());
- tempResourceComponent.setFullName(resource.getFullName());
+ tempResourceComponent.setFullName(resource.getFullName().replaceFirst("/",""));
tempResourceComponent.setId(resource.getId());
tempResourceComponent.setPid(resource.getPid());
tempResourceComponent.setIdValue(resource.getId(),resource.isDirectory());
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
index 49d12a2..3490061 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/ResourcesService.java
@@ -17,11 +17,11 @@
package org.apache.dolphinscheduler.api.service;
import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.serializer.SerializerFeature;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
import org.apache.commons.collections.BeanMap;
+import org.apache.dolphinscheduler.api.dto.resources.filter.ResourceFilter;
import org.apache.dolphinscheduler.api.dto.resources.visitor.ResourceTreeVisitor;
import org.apache.dolphinscheduler.api.dto.resources.visitor.Visitor;
import org.apache.dolphinscheduler.api.enums.Status;
@@ -30,7 +30,10 @@ import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.enums.ResourceType;
import org.apache.dolphinscheduler.common.utils.*;
-import org.apache.dolphinscheduler.dao.entity.*;
+import org.apache.dolphinscheduler.dao.entity.Resource;
+import org.apache.dolphinscheduler.dao.entity.Tenant;
+import org.apache.dolphinscheduler.dao.entity.UdfFunc;
+import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,7 +42,6 @@ import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
import org.springframework.web.multipart.MultipartFile;
-import java.io.IOException;
import java.text.MessageFormat;
import java.util.*;
import java.util.stream.Collectors;
@@ -532,10 +534,7 @@ public class ResourcesService extends BaseService {
userId = 0;
}
resourceList = resourcesMapper.queryResourceListAuthored(userId, type.ordinal());
- List<Resource> resources = resourceList.stream().filter(t -> {
- String alias = t.getAlias();
- return alias.endsWith(".jar");
- }).collect(Collectors.toList());
+ List<Resource> resources = new ResourceFilter(".jar",resourceList).filter();
Visitor resourceTreeVisitor = new ResourceTreeVisitor(resources);
result.put(Constants.DATA_LIST, resourceTreeVisitor.visit().getChildren());
putMsg(result,Status.SUCCESS);
@@ -1109,27 +1108,22 @@ public class ResourcesService extends BaseService {
List<Integer> childList = new ArrayList<>();
childList.add(resource.getId());
if(resource.isDirectory()){
- listAllChildren(resource.getId(),childList);
+ childList.addAll(listAllChildren(resource.getId()));
}
return childList;
}
/**
- * /**
* list all children id
* @param resourceId resource id
- * @param childIds child resource id list
* @return all children id
*/
- List<Integer> listAllChildren(int resourceId,List<Integer> childIds){
-
+ List<Integer> listAllChildren(int resourceId){
List<Integer> children = resourcesMapper.listChildren(resourceId);
for(int chlidId:children){
- children.add(chlidId);
- listAllChildren(chlidId,childIds);
+ listAllChildren(chlidId);
}
return children;
-
}
/**
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java
new file mode 100644
index 0000000..955fa23
--- /dev/null
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/dto/resources/filter/ResourceFilterTest.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.dolphinscheduler.api.dto.resources.filter;
+
+import org.apache.dolphinscheduler.dao.entity.Resource;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * resource filter test
+ */
+public class ResourceFilterTest {
+ private static Logger logger = LoggerFactory.getLogger(ResourceFilterTest.class);
+ @Test
+ public void filterTest(){
+ List<Resource> allList = new ArrayList<>();
+
+ Resource resource1 = new Resource(3,-1,"b","/b",true);
+ Resource resource2 = new Resource(4,2,"a1.txt","/a/a1.txt",false);
+ Resource resource3 = new Resource(5,3,"b1.txt","/b/b1.txt",false);
+ Resource resource4 = new Resource(6,3,"b2.jar","/b/b2.jar",false);
+ Resource resource5 = new Resource(7,-1,"b2","/b2",true);
+ Resource resource6 = new Resource(8,3,"c2.jar","/b/c2.jar",true);
+ allList.add(resource1);
+ allList.add(resource2);
+ allList.add(resource3);
+ allList.add(resource4);
+ allList.add(resource5);
+ allList.add(resource6);
+
+
+ ResourceFilter resourceFilter = new ResourceFilter(".jar",allList);
+ List<Resource> resourceList = resourceFilter.filter();
+ Assert.assertNotNull(resourceList);
+ resourceList.stream().forEach(t-> logger.info(t.toString()));
+ }
+}
\ No newline at end of file
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java
index a4de757..16d9491 100644
--- a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/Resource.java
@@ -104,6 +104,14 @@ public class Resource {
this.updateTime = updateTime;
}
+ public Resource(int id, int pid, String alias, String fullName, boolean isDirectory) {
+ this.id = id;
+ this.pid = pid;
+ this.alias = alias;
+ this.fullName = fullName;
+ this.isDirectory = isDirectory;
+ }
+
/*public Resource(String alias, String fileName, String description, int userId, ResourceType type, long size, Date createTime, Date updateTime) {
this.alias = alias;
this.fileName = fileName;