You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by le...@apache.org on 2022/01/13 14:36:07 UTC
[dolphinscheduler] branch 2.0.3-prepare updated: [cherry-pick-2.0.3] cherry-pick to 2.0.3 (#8008)
This is an automated email from the ASF dual-hosted git repository.
leonbao pushed a commit to branch 2.0.3-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/2.0.3-prepare by this push:
new 4d5b5b4 [cherry-pick-2.0.3] cherry-pick to 2.0.3 (#8008)
4d5b5b4 is described below
commit 4d5b5b42a42d8d484ebdf81f781fcc6ce317547a
Author: wind <ca...@users.noreply.github.com>
AuthorDate: Thu Jan 13 22:36:00 2022 +0800
[cherry-pick-2.0.3] cherry-pick to 2.0.3 (#8008)
* finished issue#7527 (#7561)
Co-authored-by: 时光 <ca...@cai-inc.com>
* [Fix-7713] Handling the sensitive data in the log (#7728)
* add a feature to handle sensitive data
* [cherry-pick-2.0.3]fix-#7740-upgrade_sql #7761
* [cherry-pick-2.0.3][Fix-7203] Remedy the issue about importing a process json file
* [cherry-pick-2.0.3][Fix-7857] The user won't be allowed to disable their own account
Co-authored-by: 时光 <41...@users.noreply.github.com>
Co-authored-by: 时光 <ca...@cai-inc.com>
Co-authored-by: calvin <ji...@163.com>
Co-authored-by: uh001 <96...@users.noreply.github.com>
---
.../api/aspect/AccessLogAspect.java | 31 ++++++++++++++++
.../apache/dolphinscheduler/api/enums/Status.java | 3 +-
.../service/impl/ProcessDefinitionServiceImpl.java | 16 +++++++++
.../api/service/impl/UsersServiceImpl.java | 6 ++++
.../api/aspect/AccessLogAspectTest.java | 42 ++++++++++++++++++++++
.../dao/mapper/TaskInstanceMapper.xml | 7 ++--
.../2.0.0_schema/mysql/dolphinscheduler_ddl.sql | 10 ++++--
.../postgresql/dolphinscheduler_ddl.sql | 6 +++-
.../security/pages/users/_source/createUser.vue | 11 ++++--
9 files changed, 124 insertions(+), 8 deletions(-)
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java
index 14d75c7..b20b6d8 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspect.java
@@ -26,7 +26,10 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.Set;
import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import java.util.stream.Collectors;
+import java.util.stream.IntStream;
import javax.servlet.http.HttpServletRequest;
@@ -48,6 +51,10 @@ public class AccessLogAspect {
private static final String TRACE_ID = "traceId";
+ public static final String sensitiveDataRegEx = "(password=[\'\"]+)(\\S+)([\'\"]+)";
+
+ private static final Pattern sensitiveDataPattern = Pattern.compile(sensitiveDataRegEx, Pattern.CASE_INSENSITIVE);
+
@Pointcut("@annotation(org.apache.dolphinscheduler.api.aspect.AccessLogAnnotation)")
public void logPointCut(){
// Do nothing because of it's a pointcut
@@ -78,6 +85,8 @@ public class AccessLogAspect {
// handle args
String argsString = parseArgs(proceedingJoinPoint, annotation);
+ // handle sensitive data in the string
+ argsString = handleSensitiveData(argsString);
logger.info("REQUEST TRACE_ID:{}, LOGIN_USER:{}, URI:{}, METHOD:{}, HANDLER:{}, ARGS:{}",
traceId,
userName,
@@ -119,6 +128,28 @@ public class AccessLogAspect {
return argsString;
}
+ protected String handleSensitiveData(String originalData) {
+ Matcher matcher = sensitiveDataPattern.matcher(originalData.toLowerCase());
+ IntStream stream = IntStream.builder().build();
+ boolean exists = false;
+ while (matcher.find()) {
+ if (matcher.groupCount() == 3) {
+ stream = IntStream.concat(stream, IntStream.range(matcher.end(1),matcher.end(2)));
+ exists = true;
+ }
+ }
+
+ if (exists) {
+ char[] chars = originalData.toCharArray();
+ stream.forEach(idx -> {
+ chars[idx] = '*';
+ });
+ return new String(chars);
+ }
+
+ return originalData;
+ }
+
private String parseLoginInfo(HttpServletRequest request) {
String userName = "NOT LOGIN";
User loginUser = (User) (request.getAttribute(Constants.SESSION_USER));
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
index 824d0d8..ac01d65 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java
@@ -348,7 +348,8 @@ public enum Status {
VERIFY_ENVIRONMENT_ERROR(1200011, "verify environment error", "验证环境信息错误"),
ENVIRONMENT_WORKER_GROUPS_IS_INVALID(1200012, "environment worker groups is invalid format", "环境关联的工作组参数解析错误"),
UPDATE_ENVIRONMENT_WORKER_GROUP_RELATION_ERROR(1200013,"You can't modify the worker group, because the worker group [{0}] and this environment [{1}] already be used in the task [{2}]",
- "您不能修改工作组选项，因为该工作组 [{0}] 和 该环境 [{1}] 已经被用在任务 [{2}] 中");
+ "您不能修改工作组选项，因为该工作组 [{0}] 和 该环境 [{1}] 已经被用在任务 [{2}] 中"),
+ NOT_ALLOW_TO_DISABLE_OWN_ACCOUNT(130020, "Not allow to disable your own account", "不能停用自己的账号");
private final int code;
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
index 9459d0d..fb9c503 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessDefinitionServiceImpl.java
@@ -106,6 +106,7 @@ import org.springframework.web.multipart.MultipartFile;
import com.baomidou.mybatisplus.core.metadata.IPage;
import com.baomidou.mybatisplus.extension.plugins.pagination.Page;
+import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import com.google.common.collect.Lists;
@@ -935,6 +936,21 @@ public class ProcessDefinitionServiceImpl extends BaseServiceImpl implements Pro
processTaskRelationLog.setPostTaskVersion(Constants.VERSION_FIRST);
taskRelationLogList.add(processTaskRelationLog);
}
+ if (StringUtils.isNotEmpty(processDefinition.getLocations()) && JSONUtils.checkJsonValid(processDefinition.getLocations())) {
+ ArrayNode arrayNode = JSONUtils.parseArray(processDefinition.getLocations());
+ ArrayNode newArrayNode = JSONUtils.createArrayNode();
+ for (int i = 0; i < arrayNode.size(); i++) {
+ ObjectNode newObjectNode = newArrayNode.addObject();
+ JsonNode jsonNode = arrayNode.get(i);
+ Long taskCode = taskCodeMap.get(jsonNode.get("taskCode").asLong());
+ if (Objects.nonNull(taskCode)) {
+ newObjectNode.put("taskCode", taskCode);
+ newObjectNode.set("x", jsonNode.get("x"));
+ newObjectNode.set("y", jsonNode.get("y"));
+ }
+ }
+ processDefinition.setLocations(newArrayNode.toString());
+ }
Map<String, Object> createDagResult = createDagDefine(loginUser, taskRelationLogList, processDefinition, Lists.newArrayList());
if (Status.SUCCESS.equals(createDagResult.get(Constants.STATUS))) {
putMsg(createDagResult, Status.SUCCESS);
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
index 3b5e0d6..4e72082 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/UsersServiceImpl.java
@@ -407,6 +407,12 @@ public class UsersServiceImpl extends BaseServiceImpl implements UsersService {
putMsg(result, Status.REQUEST_PARAMS_NOT_VALID_ERROR, phone);
return result;
}
+
+ if (state == 0 && user.getState() != state && loginUser.getId() == user.getId()) {
+ putMsg(result, Status.NOT_ALLOW_TO_DISABLE_OWN_ACCOUNT);
+ return result;
+ }
+
user.setPhone(phone);
user.setQueue(queue);
user.setState(state);
diff --git a/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspectTest.java b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspectTest.java
new file mode 100644
index 0000000..e272e42
--- /dev/null
+++ b/dolphinscheduler-api/src/test/java/org/apache/dolphinscheduler/api/aspect/AccessLogAspectTest.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.api.aspect;
+
+import org.junit.Assert;
+import org.junit.jupiter.api.Test;
+
+/**
+ * @author Hua Jiang
+ */
+
+public class AccessLogAspectTest {
+
+ private AccessLogAspect accessLogAspect = new AccessLogAspect();
+
+ @Test
+ public void testHandleSensitiveData() {
+ String data = "userPassword='7ad2410b2f4c074479a8937a28a22b8f', email='xxx@qq.com', database='null', userName='root', password='root', other='null'";
+ String expected = "userPassword='********************************', email='xxx@qq.com', database='null', userName='root', password='****', other='null'";
+
+ String actual = accessLogAspect.handleSensitiveData(data);
+
+ Assert.assertEquals(expected, actual);
+
+ }
+
+}
diff --git a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml
index bdbc538..102b7a4 100644
--- a/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml
+++ b/dolphinscheduler-dao/src/main/resources/org/apache/dolphinscheduler/dao/mapper/TaskInstanceMapper.xml
@@ -81,8 +81,11 @@
#{i}
</foreach>
</if>
- <if test="startTime != null and endTime != null">
- and t.start_time > #{startTime} and t.start_time <![CDATA[ <= ]]> #{endTime}
+ <if test="startTime != null">
+ and t.start_time <![CDATA[ > ]]> #{startTime}
+ </if>
+ <if test="endTime != null">
+ and t.start_time <![CDATA[ <= ]]> #{endTime}
</if>
group by t.state
</select>
diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_ddl.sql
index 7e1e605..aeaeb4b 100644
--- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_ddl.sql
+++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/mysql/dolphinscheduler_ddl.sql
@@ -147,7 +147,10 @@ BEGIN
AND TABLE_SCHEMA=(SELECT DATABASE())
AND COLUMN_NAME ='code')
THEN
- alter table t_ds_project add `code` bigint(20) NOT NULL COMMENT 'encoding' AFTER `name`;
+ alter table t_ds_project add `code` bigint(20) COMMENT 'encoding' AFTER `name`;
+ -- update default value for not null
+ UPDATE t_ds_project SET code = id;
+ alter table t_ds_project modify `code` bigint(20) NOT NULL;
END IF;
END;
@@ -399,7 +402,10 @@ alter table t_ds_schedules add timezone_id varchar(40) DEFAULT NULL COMMENT 'tim
alter table t_ds_schedules add environment_code bigint(20) DEFAULT '-1' COMMENT 'environment code' AFTER worker_group;
-- t_ds_process_definition
-alter table t_ds_process_definition add `code` bigint(20) NOT NULL COMMENT 'encoding' AFTER `id`;
+alter table t_ds_process_definition add `code` bigint(20) COMMENT 'encoding' AFTER `id`;
+-- update default value for not null
+UPDATE t_ds_process_definition SET code = id;
+alter table t_ds_process_definition modify `code` bigint(20) NOT NULL;
alter table t_ds_process_definition change project_id project_code bigint(20) NOT NULL COMMENT 'project code' AFTER `description`;
alter table t_ds_process_definition add `warning_group_id` int(11) DEFAULT NULL COMMENT 'alert group id' AFTER `locations`;
alter table t_ds_process_definition add UNIQUE KEY `process_unique` (`name`,`project_code`) USING BTREE;
diff --git a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_ddl.sql b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_ddl.sql
index e5024d9..deca2d4 100644
--- a/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_ddl.sql
+++ b/dolphinscheduler-dao/src/main/resources/sql/upgrade/2.0.0_schema/postgresql/dolphinscheduler_ddl.sql
@@ -74,6 +74,10 @@ BEGIN
EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'.t_ds_process_definition ADD COLUMN IF NOT EXISTS "code" bigint';
EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'.t_ds_process_definition ADD COLUMN IF NOT EXISTS "warning_group_id" int';
+ --update default value for not null
+ EXECUTE 'UPDATE ' || quote_ident(v_schema) ||'.t_ds_process_definition SET code = id';
+ EXECUTE 'UPDATE ' || quote_ident(v_schema) ||'.t_ds_project SET code = id';
+
---drop columns
EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'.t_ds_tenant DROP COLUMN IF EXISTS "tenant_name"';
EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'.t_ds_process_instance DROP COLUMN IF EXISTS "process_instance_json"';
@@ -92,7 +96,7 @@ BEGIN
EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'."t_ds_process_definition" ALTER COLUMN "code" SET NOT NULL';
EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'."t_ds_process_definition" ALTER COLUMN "project_code" SET NOT NULL';
EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'."t_ds_process_definition" ADD CONSTRAINT "process_unique" UNIQUE ("name","project_code")';
- EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'."t_ds_process_definition" ALTER COLUMN "description" SET NOT NULL';
+ EXECUTE 'ALTER TABLE ' || quote_ident(v_schema) ||'."t_ds_project" ALTER COLUMN "code" SET NOT NULL';
--- drop index
EXECUTE 'DROP INDEX IF EXISTS "process_instance_index"';
diff --git a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue
index bb956e9..25575b6 100644
--- a/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue
+++ b/dolphinscheduler-ui/src/js/conf/home/pages/security/pages/users/_source/createUser.vue
@@ -97,7 +97,7 @@
</el-input>
</template>
</m-list-box-f>
- <m-list-box-f style="line-height: 38px;">
+ <m-list-box-f v-if="showState" style="line-height: 38px;">
<template slot="name">{{$t('State')}}</template>
<template slot="content">
<el-radio-group v-model="userState" size="small">
@@ -117,6 +117,7 @@
import router from '@/conf/home/router'
import mPopover from '@/module/components/popup/popover'
import mListBoxF from '@/module/components/listBoxF/listBoxF'
+ import { mapState } from 'vuex'
export default {
name: 'create-user',
@@ -133,6 +134,7 @@
phone: '',
userState: '1',
tenantList: [],
+ showState: true,
// Source admin user information
isADMIN: store.state.user.userInfo.userType === 'ADMIN_USER' && router.history.current.name !== 'account'
}
@@ -292,6 +294,7 @@
watch: {},
created () {
// Administrator gets tenant list
+ this.showState = true
if (this.isADMIN) {
Promise.all([this._getQueueList(), this._getTenantList()]).then(() => {
if (this.item) {
@@ -301,6 +304,7 @@
this.phone = this.item.phone
this.state = this.item.state
this.userState = this.item.state + '' || '1'
+ this.showState = this.item.id !== this.userInfo.id
if (this.fromUserInfo || this.item.tenantId) {
this.tenantId = this.item.tenantId
}
@@ -320,6 +324,7 @@
this.phone = this.item.phone
this.state = this.item.state
this.userState = this.state + '' || '1'
+ this.showState = this.item.id !== this.userInfo.id
if (this.fromUserInfo || this.item.tenantId) {
this.tenantId = this.item.tenantId
}
@@ -335,7 +340,9 @@
}
},
mounted () {
-
+ },
+ computed: {
+ ...mapState('user', ['userInfo'])
},
components: { mPopover, mListBoxF }
}