You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by li...@apache.org on 2021/06/18 11:11:08 UTC
[dolphinscheduler] branch dev updated: [bug fix] Some password
attributes can be viewd directly in the log (#5638)
This is an automated email from the ASF dual-hosted git repository.
lidongdai pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 75813e7 [bug fix] Some password attributes can be viewd directly in the log (#5638)
75813e7 is described below
commit 75813e7866af2cd4f11940c416fdeea3e3698f62
Author: kyoty <ec...@gmail.com>
AuthorDate: Fri Jun 18 19:10:57 2021 +0800
[bug fix] Some password attributes can be viewd directly in the log (#5638)
* fix the password display incorrectly
* update
* update
* fic code style
fic code style
* Update SensitiveDataConverterTest.java
* Update SensitiveDataConverter.java
* Update SensitiveDataConverter.java
* Update SensitiveDataConverterTest.java
* Update pom.xml
---
.../dolphinscheduler/logback/logback-alert.xml | 7 +-
.../conf/dolphinscheduler/logback/logback-api.xml | 7 +-
.../dolphinscheduler/logback/logback-master.xml | 11 +-
.../dolphinscheduler/logback/logback-worker.xml | 11 +-
.../src/main/resources/logback-alert.xml | 7 +-
.../src/main/resources/logback-api.xml | 7 +-
.../apache/dolphinscheduler/common/Constants.java | 2 +-
.../src/main/resources/logback-master.xml | 13 +-
.../src/main/resources/logback-worker.xml | 11 +-
.../server/log/SensitiveDataConverterTest.java | 179 ---------------------
.../service}/log/SensitiveDataConverter.java | 13 +-
.../service/log/SensitiveDataConverterTest.java | 137 ++++++++++++++++
pom.xml | 2 +-
13 files changed, 182 insertions(+), 225 deletions(-)
diff --git a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml
index 1718947..5f18729 100644
--- a/docker/build/conf/dolphinscheduler/logback/logback-alert.xml
+++ b/docker/build/conf/dolphinscheduler/logback/logback-alert.xml
@@ -18,12 +18,13 @@
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
-
+ <conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter">
+ </conversionRule>
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
@@ -38,7 +39,7 @@
</rollingPolicy>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
diff --git a/docker/build/conf/dolphinscheduler/logback/logback-api.xml b/docker/build/conf/dolphinscheduler/logback/logback-api.xml
index bec3d9a..ead5403 100644
--- a/docker/build/conf/dolphinscheduler/logback/logback-api.xml
+++ b/docker/build/conf/dolphinscheduler/logback/logback-api.xml
@@ -18,12 +18,13 @@
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
-
+ <conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter">
+ </conversionRule>
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
@@ -42,7 +43,7 @@
</rollingPolicy>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
diff --git a/docker/build/conf/dolphinscheduler/logback/logback-master.xml b/docker/build/conf/dolphinscheduler/logback/logback-master.xml
index f0d2c81..c8692b8 100644
--- a/docker/build/conf/dolphinscheduler/logback/logback-master.xml
+++ b/docker/build/conf/dolphinscheduler/logback/logback-master.xml
@@ -18,19 +18,18 @@
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
-
+ <conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter">
+ </conversionRule>
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
- <conversionRule conversionWord="messsage"
- converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<!-- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
@@ -45,7 +44,7 @@
<file>${log.base}/${taskAppId}.log</file>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
@@ -66,7 +65,7 @@
</rollingPolicy>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
diff --git a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml
index 7127219..fe03841 100644
--- a/docker/build/conf/dolphinscheduler/logback/logback-worker.xml
+++ b/docker/build/conf/dolphinscheduler/logback/logback-worker.xml
@@ -18,20 +18,19 @@
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
-
+ <conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter">
+ </conversionRule>
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<!-- worker server logback config start -->
- <conversionRule conversionWord="messsage"
- converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<!-- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
@@ -46,7 +45,7 @@
<file>${log.base}/${taskAppId}.log</file>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
@@ -66,7 +65,7 @@
</rollingPolicy>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
diff --git a/dolphinscheduler-alert/src/main/resources/logback-alert.xml b/dolphinscheduler-alert/src/main/resources/logback-alert.xml
index 1718947..5f18729 100644
--- a/dolphinscheduler-alert/src/main/resources/logback-alert.xml
+++ b/dolphinscheduler-alert/src/main/resources/logback-alert.xml
@@ -18,12 +18,13 @@
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
-
+ <conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter">
+ </conversionRule>
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
@@ -38,7 +39,7 @@
</rollingPolicy>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
diff --git a/dolphinscheduler-api/src/main/resources/logback-api.xml b/dolphinscheduler-api/src/main/resources/logback-api.xml
index ed655a0..77d3226 100644
--- a/dolphinscheduler-api/src/main/resources/logback-api.xml
+++ b/dolphinscheduler-api/src/main/resources/logback-api.xml
@@ -18,12 +18,13 @@
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
-
+ <conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter">
+ </conversionRule>
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
@@ -42,7 +43,7 @@
</rollingPolicy>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
diff --git a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
index d9e44c5..898e39c 100644
--- a/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
+++ b/dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/Constants.java
@@ -999,7 +999,7 @@ public final class Constants {
/**
* dataSource sensitive param
*/
- public static final String DATASOURCE_PASSWORD_REGEX = "(?<=(\"password\":\")).*?(?=(\"))";
+ public static final String DATASOURCE_PASSWORD_REGEX = "(?<=((?i)password((\\\\\":\\\\\")|(=')))).*?(?=((\\\\\")|(')))";
/**
* default worker group
diff --git a/dolphinscheduler-server/src/main/resources/logback-master.xml b/dolphinscheduler-server/src/main/resources/logback-master.xml
index a61d891..24d8147 100644
--- a/dolphinscheduler-server/src/main/resources/logback-master.xml
+++ b/dolphinscheduler-server/src/main/resources/logback-master.xml
@@ -18,19 +18,20 @@
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
-
+ <conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter">
+ </conversionRule>
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
- <conversionRule conversionWord="messsage"
- converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
+
+
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<!-- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
@@ -45,7 +46,7 @@
<file>${log.base}/${taskAppId}.log</file>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
@@ -66,7 +67,7 @@
</rollingPolicy>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
diff --git a/dolphinscheduler-server/src/main/resources/logback-worker.xml b/dolphinscheduler-server/src/main/resources/logback-worker.xml
index 31719d5..3549ff9 100644
--- a/dolphinscheduler-server/src/main/resources/logback-worker.xml
+++ b/dolphinscheduler-server/src/main/resources/logback-worker.xml
@@ -18,20 +18,19 @@
<!-- Logback configuration. See http://logback.qos.ch/manual/index.html -->
<configuration scan="true" scanPeriod="120 seconds"> <!--debug="true" -->
-
+ <conversionRule conversionWord="message" converterClass="org.apache.dolphinscheduler.service.log.SensitiveDataConverter">
+ </conversionRule>
<property name="log.base" value="logs"/>
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %msg%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
<!-- worker server logback config start -->
- <conversionRule conversionWord="messsage"
- converterClass="org.apache.dolphinscheduler.server.log.SensitiveDataConverter"/>
<appender name="TASKLOGFILE" class="ch.qos.logback.classic.sift.SiftingAppender">
<!-- <filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>INFO</level>
@@ -46,7 +45,7 @@
<file>${log.base}/${taskAppId}.log</file>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
@@ -66,7 +65,7 @@
</rollingPolicy>
<encoder>
<pattern>
- [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %messsage%n
+ [%level] %date{yyyy-MM-dd HH:mm:ss.SSS} %logger{96}:[%line] - %message%n
</pattern>
<charset>UTF-8</charset>
</encoder>
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverterTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverterTest.java
deleted file mode 100644
index 6319bf1..0000000
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverterTest.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.dolphinscheduler.server.log;
-
-
-import ch.qos.logback.classic.Level;
-import ch.qos.logback.classic.spi.ILoggingEvent;
-import ch.qos.logback.classic.spi.IThrowableProxy;
-import ch.qos.logback.classic.spi.LoggerContextVO;
-import org.apache.dolphinscheduler.common.Constants;
-import org.apache.dolphinscheduler.common.utils.SensitiveLogUtils;
-import org.junit.Assert;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.slf4j.Marker;
-
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-public class SensitiveDataConverterTest {
-
- private final Logger logger = LoggerFactory.getLogger(SensitiveDataConverterTest.class);
-
- /**
- * password pattern
- */
- private final Pattern pwdPattern = Pattern.compile(Constants.DATASOURCE_PASSWORD_REGEX);
-
- private final String logMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," +
- "\"database\":\"carbond\"," +
- "\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," +
- "\"user\":\"view\"," +
- "\"password\":\"view1\"}";
-
- private final String maskLogMsg = "{\"address\":\"jdbc:mysql://192.168.xx.xx:3306\"," +
- "\"database\":\"carbond\"," +
- "\"jdbcUrl\":\"jdbc:mysql://192.168.xx.xx:3306/ods\"," +
- "\"user\":\"view\"," +
- "\"password\":\"******\"}";
- @Test
- public void convert() {
- SensitiveDataConverter sensitiveDataConverter = new SensitiveDataConverter();
- String result = sensitiveDataConverter.convert(new ILoggingEvent() {
- @Override
- public String getThreadName() {
- return null;
- }
-
- @Override
- public Level getLevel() {
- return Level.INFO;
- }
-
- @Override
- public String getMessage() {
- return null;
- }
-
- @Override
- public Object[] getArgumentArray() {
- return new Object[0];
- }
-
- @Override
- public String getFormattedMessage() {
- return logMsg;
- }
-
- @Override
- public String getLoggerName() {
- return null;
- }
-
- @Override
- public LoggerContextVO getLoggerContextVO() {
- return null;
- }
-
- @Override
- public IThrowableProxy getThrowableProxy() {
- return null;
- }
-
- @Override
- public StackTraceElement[] getCallerData() {
- return new StackTraceElement[0];
- }
-
- @Override
- public boolean hasCallerData() {
- return false;
- }
-
- @Override
- public Marker getMarker() {
- return null;
- }
-
- @Override
- public Map<String, String> getMDCPropertyMap() {
- return null;
- }
-
- @Override
- public Map<String, String> getMdc() {
- return null;
- }
-
- @Override
- public long getTimeStamp() {
- return 0;
- }
-
- @Override
- public void prepareForDeferredProcessing() {
-
- }
- });
-
- Assert.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg));
-
- }
-
- /**
- * mask sensitive logMsg - sql task datasource password
- */
- @Test
- public void testPwdLogMsgConverter() {
- logger.info("parameter : {}", logMsg);
- logger.info("parameter : {}", passwordHandler(pwdPattern, logMsg));
-
- Assert.assertNotEquals(logMsg, passwordHandler(pwdPattern, logMsg));
- Assert.assertEquals(maskLogMsg, passwordHandler(pwdPattern, logMsg));
-
- }
-
- /**
- * password regex test
- *
- * @param logMsg original log
- */
- private static String passwordHandler(Pattern pattern, String logMsg) {
-
- Matcher matcher = pattern.matcher(logMsg);
-
- StringBuffer sb = new StringBuffer(logMsg.length());
-
- while (matcher.find()) {
-
- String password = matcher.group();
-
- String maskPassword = SensitiveLogUtils.maskDataSourcePwd(password);
-
- matcher.appendReplacement(sb, maskPassword);
- }
- matcher.appendTail(sb);
-
- return sb.toString();
- }
-
-
-
-}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverter.java
similarity index 93%
rename from dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java
rename to dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverter.java
index 16101c0..f6a6929 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/SensitiveDataConverter.java
+++ b/dolphinscheduler-service/src/main/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverter.java
@@ -14,11 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.dolphinscheduler.server.log;
+package org.apache.dolphinscheduler.service.log;
-import ch.qos.logback.classic.pattern.MessageConverter;
-import ch.qos.logback.classic.spi.ILoggingEvent;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.SensitiveLogUtils;
import org.apache.dolphinscheduler.common.utils.StringUtils;
@@ -26,17 +24,18 @@ import org.apache.dolphinscheduler.common.utils.StringUtils;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import ch.qos.logback.classic.pattern.ClassicConverter;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+
/**
* sensitive data log converter
*/
-public class SensitiveDataConverter extends MessageConverter {
-
+public class SensitiveDataConverter extends ClassicConverter {
/**
* password pattern
*/
private final Pattern pwdPattern = Pattern.compile(Constants.DATASOURCE_PASSWORD_REGEX);
-
@Override
public String convert(ILoggingEvent event) {
@@ -68,7 +67,6 @@ public class SensitiveDataConverter extends MessageConverter {
* @param logMsg original log
*/
private String passwordHandler(Pattern pwdPattern, String logMsg) {
-
Matcher matcher = pwdPattern.matcher(logMsg);
StringBuffer sb = new StringBuffer(logMsg.length());
@@ -86,5 +84,4 @@ public class SensitiveDataConverter extends MessageConverter {
return sb.toString();
}
-
}
diff --git a/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverterTest.java b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverterTest.java
new file mode 100644
index 0000000..133a504
--- /dev/null
+++ b/dolphinscheduler-service/src/test/java/org/apache/dolphinscheduler/service/log/SensitiveDataConverterTest.java
@@ -0,0 +1,137 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.service.log;
+
+import java.util.Map;
+import java.util.function.Function;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Marker;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.classic.spi.IThrowableProxy;
+import ch.qos.logback.classic.spi.LoggerContextVO;
+
+public class SensitiveDataConverterTest {
+
+ // Use the Function interface to make it easier to create an ILoggingEvent instance.
+ private static final Function<String, ILoggingEvent> LOGGING_EVENT_CREATOR = (logDetail) -> (new ILoggingEvent() {
+ @Override
+ public String getThreadName() {
+ return null;
+ }
+
+ @Override
+ public Level getLevel() {
+ return Level.INFO;
+ }
+
+ @Override
+ public String getMessage() {
+ return null;
+ }
+
+ @Override
+ public Object[] getArgumentArray() {
+ return new Object[0];
+ }
+
+ @Override
+ public String getFormattedMessage() {
+ return logDetail;
+ }
+
+ @Override
+ public String getLoggerName() {
+ return null;
+ }
+
+ @Override
+ public LoggerContextVO getLoggerContextVO() {
+ return null;
+ }
+
+ @Override
+ public IThrowableProxy getThrowableProxy() {
+ return null;
+ }
+
+ @Override
+ public StackTraceElement[] getCallerData() {
+ return new StackTraceElement[0];
+ }
+
+ @Override
+ public boolean hasCallerData() {
+ return false;
+ }
+
+ @Override
+ public Marker getMarker() {
+ return null;
+ }
+
+ @Override
+ public Map<String, String> getMDCPropertyMap() {
+ return null;
+ }
+
+ @Override
+ public Map<String, String> getMdc() {
+ return null;
+ }
+
+ @Override
+ public long getTimeStamp() {
+ return 0;
+ }
+
+ @Override
+ public void prepareForDeferredProcessing() {
+
+ }
+ });
+
+ @Test
+ public void convert() {
+ String[] initialLogs = new String[]{
+ "{\\\"user\\\":\\\"root\\\",\\\"password\\\":\\\"123456\\\","
+ + "\\\"address\\\":\\\"jdbc:mysql://localhost:3306\\\","
+ + "\\\"database\\\":\\\"dolphinscheduler\\\","
+ + "\\\"jdbcUrl\\\":\\\"jdbc:mysql://localhost/dolphinscheduler\\\"}",
+ "LOGIN_USER:admin, URI:/dolphinscheduler/users/verify-user-name, METHOD:GET, "
+ + "ARGS:[User{id=1, userName='admin', userPassword='Qazwsx.741', euserType=ADMIN_USER]}"
+ };
+ String[] encryptedLogs = new String[]{
+ "{\\\"user\\\":\\\"root\\\",\\\"password\\\":\\\"******\\\","
+ + "\\\"address\\\":\\\"jdbc:mysql://localhost:3306\\\","
+ + "\\\"database\\\":\\\"dolphinscheduler\\\","
+ + "\\\"jdbcUrl\\\":\\\"jdbc:mysql://localhost/dolphinscheduler\\\"}",
+ "LOGIN_USER:admin, URI:/dolphinscheduler/users/verify-user-name, METHOD:GET, "
+ + "ARGS:[User{id=1, userName='admin', userPassword='******', euserType=ADMIN_USER]}"
+ };
+
+ SensitiveDataConverter sensitiveDataConverter = new SensitiveDataConverter();
+ for (int i = 0; i < initialLogs.length; i++) {
+ Assert.assertEquals(encryptedLogs[i], sensitiveDataConverter.convert(LOGGING_EVENT_CREATOR.apply(initialLogs[i])));
+ }
+
+ }
+}
diff --git a/pom.xml b/pom.xml
index 4f9fa3e..3bfe355 100644
--- a/pom.xml
+++ b/pom.xml
@@ -961,7 +961,6 @@
<include>**/server/log/LoggerServerTest.java</include>
<include>**/server/entity/SQLTaskExecutionContextTest.java</include>
<include>**/server/log/MasterLogFilterTest.java</include>
- <include>**/server/log/SensitiveDataConverterTest.java</include>
<include>**/server/log/LoggerRequestProcessorTest.java</include>
<!--<include>**/server/log/TaskLogDiscriminatorTest.java</include>-->
<include>**/server/log/TaskLogFilterTest.java</include>
@@ -1023,6 +1022,7 @@
<include>**/service/queue/TaskUpdateQueueTest.java</include>
<include>**/service/queue/PeerTaskInstancePriorityQueueTest.java</include>
<include>**/service/log/LogClientServiceTest.java</include>
+ <include>**/service/log/SensitiveDataConverterTest.java</include>
<include>**/service/alert/AlertClientServiceTest.java</include>
<include>**/service/alert/ProcessAlertManagerTest.java</include>
<include>**/dao/mapper/DataSourceUserMapperTest.java</include>