You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by GitBox <gi...@apache.org> on 2021/02/18 09:06:52 UTC
[GitHub] [incubator-dolphinscheduler] CalvinKirs commented on a change in pull request #4768: [Improvement-3369][api] Introduce taskrecord, udffunc, workflowlineage and workergroup service interface for clear code
CalvinKirs commented on a change in pull request #4768:
URL: https://github.com/apache/incubator-dolphinscheduler/pull/4768#discussion_r578239307
##########
File path: dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java
##########
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.service.impl;
+
+import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP;
+
+import org.apache.dolphinscheduler.api.enums.Status;
+import org.apache.dolphinscheduler.api.service.BaseService;
+import org.apache.dolphinscheduler.api.service.WorkerGroupService;
+import org.apache.dolphinscheduler.api.utils.PageInfo;
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.utils.CollectionUtils;
+import org.apache.dolphinscheduler.common.utils.DateUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
+import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
+import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+
+/**
+ * work group service impl
+ */
+@Service
+public class WorkerGroupServiceImpl extends BaseService implements WorkerGroupService {
+
+ private static final String NO_NODE_EXCEPTION_REGEX = "KeeperException$NoNodeException";
+
+ @Autowired
+ protected ZookeeperCachedOperator zookeeperCachedOperator;
+
+ @Autowired
+ ProcessInstanceMapper processInstanceMapper;
+
+ /**
+ * query worker group paging
+ *
+ * @param loginUser login user
+ * @param pageNo page number
+ * @param searchVal search value
+ * @param pageSize page size
+ * @return worker group list page
+ */
+ public Map<String, Object> queryAllGroupPaging(User loginUser, Integer pageNo, Integer pageSize, String searchVal) {
+ // list from index
+ Integer fromIndex = (pageNo - 1) * pageSize;
+ // list to index
+ Integer toIndex = (pageNo - 1) * pageSize + pageSize;
+
+ Map<String, Object> result = new HashMap<>();
+ if (isNotAdmin(loginUser, result)) {
+ return result;
+ }
+
+ List<WorkerGroup> workerGroups = getWorkerGroups(true);
+
+ List<WorkerGroup> resultDataList = new ArrayList<>();
+
+ if (CollectionUtils.isNotEmpty(workerGroups)) {
+ List<WorkerGroup> searchValDataList = new ArrayList<>();
+
+ if (StringUtils.isNotEmpty(searchVal)) {
+ for (WorkerGroup workerGroup : workerGroups) {
+ if (workerGroup.getName().contains(searchVal)) {
+ searchValDataList.add(workerGroup);
+ }
+ }
+ } else {
+ searchValDataList = workerGroups;
+ }
+
+ if (searchValDataList.size() < pageSize) {
+ toIndex = (pageNo - 1) * pageSize + searchValDataList.size();
+ }
+ resultDataList = searchValDataList.subList(fromIndex, toIndex);
+ }
+
+ PageInfo<WorkerGroup> pageInfo = new PageInfo<>(pageNo, pageSize);
+ pageInfo.setTotalCount(resultDataList.size());
+ pageInfo.setLists(resultDataList);
+
+ result.put(Constants.DATA_LIST, pageInfo);
+ putMsg(result, Status.SUCCESS);
+ return result;
+ }
+
+ /**
+ * query all worker group
+ *
+ * @return all worker group list
+ */
+ public Map<String, Object> queryAllGroup() {
+ Map<String, Object> result = new HashMap<>();
+
+ List<WorkerGroup> workerGroups = getWorkerGroups(false);
+
+ Set<String> availableWorkerGroupSet = workerGroups.stream()
+ .map(WorkerGroup::getName)
+ .collect(Collectors.toSet());
+ result.put(Constants.DATA_LIST, availableWorkerGroupSet);
+ putMsg(result, Status.SUCCESS);
+ return result;
+ }
+
+ /**
+ * get worker groups
+ *
+ * @param isPaging whether paging
+ * @return WorkerGroup list
+ */
+ private List<WorkerGroup> getWorkerGroups(boolean isPaging) {
+ String workerPath = zookeeperCachedOperator.getZookeeperConfig().getDsRoot() + Constants.ZOOKEEPER_DOLPHINSCHEDULER_WORKERS;
+ List<WorkerGroup> workerGroups = new ArrayList<>();
+ List<String> workerGroupList;
+ try {
+ workerGroupList = zookeeperCachedOperator.getChildrenKeys(workerPath);
+ } catch (Exception e) {
+ if (e.getMessage().contains(NO_NODE_EXCEPTION_REGEX)) {
+ if (isPaging) {
+ return workerGroups;
+ } else {
+ //ignore noNodeException return Default
+ WorkerGroup wg = new WorkerGroup();
+ wg.setName(DEFAULT_WORKER_GROUP);
+ workerGroups.add(wg);
+ return workerGroups;
+ }
+ } else {
+ throw e;
Review comment:
This approach doesn't look very good
##########
File path: dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java
##########
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.service.impl;
+
+import org.apache.dolphinscheduler.api.enums.Status;
+import org.apache.dolphinscheduler.api.service.BaseService;
+import org.apache.dolphinscheduler.api.service.WorkFlowLineageService;
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage;
+import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation;
+import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+/**
+ * work flow lineage service impl
+ */
+@Service
+public class WorkFlowLineageServiceImpl extends BaseService implements WorkFlowLineageService {
+
+ @Autowired
+ private WorkFlowLineageMapper workFlowLineageMapper;
+
+ public Map<String, Object> queryWorkFlowLineageByName(String workFlowName, int projectId) {
+ Map<String, Object> result = new HashMap<>(5);
+ List<WorkFlowLineage> workFlowLineageList = workFlowLineageMapper.queryByName(workFlowName, projectId);
+ result.put(Constants.DATA_LIST, workFlowLineageList);
+ putMsg(result, Status.SUCCESS);
+ return result;
+ }
+
+ private List<WorkFlowRelation> getWorkFlowRelationRecursion(Set<Integer> ids, List<WorkFlowRelation> workFlowRelations,Set<Integer> sourceIds) {
+ for (int id : ids) {
+ sourceIds.addAll(ids);
+ List<WorkFlowRelation> workFlowRelationsTmp = workFlowLineageMapper.querySourceTarget(id);
+ if (workFlowRelationsTmp != null && !workFlowRelationsTmp.isEmpty()) {
+ Set<Integer> idsTmp = new HashSet<>();
+ for (WorkFlowRelation workFlowRelation:workFlowRelationsTmp) {
+ if (!sourceIds.contains(workFlowRelation.getTargetWorkFlowId())) {
+ idsTmp.add(workFlowRelation.getTargetWorkFlowId());
+ }
+ }
+ workFlowRelations.addAll(workFlowRelationsTmp);
+ getWorkFlowRelationRecursion(idsTmp, workFlowRelations,sourceIds);
+ }
+ }
+ return workFlowRelations;
+ }
+
+ public Map<String, Object> queryWorkFlowLineageByIds(Set<Integer> ids,int projectId) {
+ Map<String, Object> result = new HashMap<>(5);
+ List<WorkFlowLineage> workFlowLineageList = workFlowLineageMapper.queryByIds(ids, projectId);
+ Map<String, Object> workFlowLists = new HashMap<>(5);
+ Set<Integer> idsV = new HashSet<>();
+ if (ids == null || ids.isEmpty()) {
+ for (WorkFlowLineage workFlowLineage:workFlowLineageList) {
+ idsV.add(workFlowLineage.getWorkFlowId());
+ }
+ } else {
+ idsV = ids;
+ }
+ List<WorkFlowRelation> workFlowRelations = new ArrayList<>();
+ Set<Integer> sourceIds = new HashSet<>();
+ getWorkFlowRelationRecursion(idsV, workFlowRelations, sourceIds);
+
+ Set<Integer> idSet = new HashSet<>();
+ //If the incoming parameter is not empty, you need to add downstream workflow detail attributes
+ if (ids != null && !ids.isEmpty()) {
+ for (WorkFlowRelation workFlowRelation : workFlowRelations) {
+ idSet.add(workFlowRelation.getTargetWorkFlowId());
+ }
+ for (int id : ids) {
+ idSet.remove(id);
+ }
+ if (!idSet.isEmpty()) {
+ workFlowLineageList.addAll(workFlowLineageMapper.queryByIds(idSet, projectId));
+ }
+ }
+
+ workFlowLists.put("workFlowList",workFlowLineageList);
Review comment:
It's best not to show magic numbers
##########
File path: dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkerGroupServiceImpl.java
##########
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.service.impl;
+
+import static org.apache.dolphinscheduler.common.Constants.DEFAULT_WORKER_GROUP;
+
+import org.apache.dolphinscheduler.api.enums.Status;
+import org.apache.dolphinscheduler.api.service.BaseService;
+import org.apache.dolphinscheduler.api.service.WorkerGroupService;
+import org.apache.dolphinscheduler.api.utils.PageInfo;
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.common.utils.CollectionUtils;
+import org.apache.dolphinscheduler.common.utils.DateUtils;
+import org.apache.dolphinscheduler.common.utils.StringUtils;
+import org.apache.dolphinscheduler.dao.entity.User;
+import org.apache.dolphinscheduler.dao.entity.WorkerGroup;
+import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
+import org.apache.dolphinscheduler.service.zk.ZookeeperCachedOperator;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+
+/**
+ * work group service impl
+ */
+@Service
+public class WorkerGroupServiceImpl extends BaseService implements WorkerGroupService {
+
+ private static final String NO_NODE_EXCEPTION_REGEX = "KeeperException$NoNodeException";
+
+ @Autowired
+ protected ZookeeperCachedOperator zookeeperCachedOperator;
+
+ @Autowired
+ ProcessInstanceMapper processInstanceMapper;
+
+ /**
+ * query worker group paging
+ *
+ * @param loginUser login user
+ * @param pageNo page number
+ * @param searchVal search value
+ * @param pageSize page size
+ * @return worker group list page
+ */
+ public Map<String, Object> queryAllGroupPaging(User loginUser, Integer pageNo, Integer pageSize, String searchVal) {
+ // list from index
+ Integer fromIndex = (pageNo - 1) * pageSize;
Review comment:
Modified to basic type
##########
File path: dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java
##########
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.service.impl;
+
+import org.apache.dolphinscheduler.api.enums.Status;
+import org.apache.dolphinscheduler.api.service.BaseService;
+import org.apache.dolphinscheduler.api.service.WorkFlowLineageService;
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage;
+import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation;
+import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+/**
+ * work flow lineage service impl
+ */
+@Service
+public class WorkFlowLineageServiceImpl extends BaseService implements WorkFlowLineageService {
+
+ @Autowired
+ private WorkFlowLineageMapper workFlowLineageMapper;
+
+ public Map<String, Object> queryWorkFlowLineageByName(String workFlowName, int projectId) {
+ Map<String, Object> result = new HashMap<>(5);
Review comment:
It doesn't make much sense to declare the initial size as 5
##########
File path: dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/WorkFlowLineageServiceImpl.java
##########
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.service.impl;
+
+import org.apache.dolphinscheduler.api.enums.Status;
+import org.apache.dolphinscheduler.api.service.BaseService;
+import org.apache.dolphinscheduler.api.service.WorkFlowLineageService;
+import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.dao.entity.WorkFlowLineage;
+import org.apache.dolphinscheduler.dao.entity.WorkFlowRelation;
+import org.apache.dolphinscheduler.dao.mapper.WorkFlowLineageMapper;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+
+/**
+ * work flow lineage service impl
+ */
+@Service
+public class WorkFlowLineageServiceImpl extends BaseService implements WorkFlowLineageService {
+
+ @Autowired
+ private WorkFlowLineageMapper workFlowLineageMapper;
+
+ public Map<String, Object> queryWorkFlowLineageByName(String workFlowName, int projectId) {
+ Map<String, Object> result = new HashMap<>(5);
+ List<WorkFlowLineage> workFlowLineageList = workFlowLineageMapper.queryByName(workFlowName, projectId);
+ result.put(Constants.DATA_LIST, workFlowLineageList);
+ putMsg(result, Status.SUCCESS);
+ return result;
+ }
+
+ private List<WorkFlowRelation> getWorkFlowRelationRecursion(Set<Integer> ids, List<WorkFlowRelation> workFlowRelations,Set<Integer> sourceIds) {
Review comment:
This method may be better declared as void
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org