You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by ga...@apache.org on 2022/03/18 00:16:13 UTC
[dolphinscheduler] branch 2.0.6-prepare updated: [Fix-Test][LoggerServer] fix LoggerServer UT (#8973)
This is an automated email from the ASF dual-hosted git repository.
gaojun2048 pushed a commit to branch 2.0.6-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/2.0.6-prepare by this push:
new d74c80e [Fix-Test][LoggerServer] fix LoggerServer UT (#8973)
d74c80e is described below
commit d74c80e2a1f1de96f0bb31385005d50e8516bd5f
Author: gaojun2048 <ga...@gmail.com>
AuthorDate: Fri Mar 18 08:16:01 2022 +0800
[Fix-Test][LoggerServer] fix LoggerServer UT (#8973)
* [Fix-Test][LoggerServer] fix LoggerServer UT
* fix checkstyle
* fix code style
* fix code style
* fix code style
* fix code style
---
.../server/log/LoggerRequestProcessor.java | 36 ++++++++++---------
.../server/log/MasterLogFilter.java | 8 +++--
.../server/log/LoggerRequestProcessorTest.java | 21 ++++++++---
.../server/log/LoggerServerTest.java | 41 ++++++++++++++--------
4 files changed, 67 insertions(+), 39 deletions(-)
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
index fb92849..78a4d32 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
@@ -17,6 +17,7 @@
package org.apache.dolphinscheduler.server.log;
+import io.netty.channel.Channel;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.remote.command.Command;
@@ -31,6 +32,9 @@ import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand;
import org.apache.dolphinscheduler.remote.command.log.ViewLogResponseCommand;
import org.apache.dolphinscheduler.remote.processor.NettyRequestProcessor;
import org.apache.dolphinscheduler.remote.utils.Constants;
+import org.apache.dolphinscheduler.spi.utils.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -46,16 +50,11 @@ import java.util.concurrent.Executors;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-import org.apache.dolphinscheduler.spi.utils.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import io.netty.channel.Channel;
-
/**
* logger request process logic
*/
-public class LoggerRequestProcessor implements NettyRequestProcessor {
+public class LoggerRequestProcessor
+ implements NettyRequestProcessor {
private final Logger logger = LoggerFactory.getLogger(LoggerRequestProcessor.class);
@@ -128,7 +127,8 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
if (taskLogFile.exists()) {
status = taskLogFile.delete();
}
- } catch (Exception e) {
+ }
+ catch (Exception e) {
status = false;
}
@@ -142,6 +142,7 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
private boolean checkPathSecurity(String path) {
String dsHome = System.getProperty("DOLPHINSCHEDULER_HOME");
+ // if we run server in IDE, user.dir is the DS Home.
if (StringUtils.isBlank(dsHome)) {
dsHome = System.getProperty("user.dir");
}
@@ -149,7 +150,8 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
if (path.startsWith(dsHome) && !path.contains("../") && path.endsWith(".log")) {
return true;
}
- } else {
+ }
+ else {
logger.warn("path is null");
}
return false;
@@ -168,14 +170,15 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
*/
private byte[] getFileContentBytes(String filePath) {
try (InputStream in = new FileInputStream(filePath);
- ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream()) {
byte[] buf = new byte[1024];
int len;
while ((len = in.read(buf)) != -1) {
bos.write(buf, 0, len);
}
return bos.toByteArray();
- } catch (IOException e) {
+ }
+ catch (IOException e) {
logger.error("get file bytes error", e);
}
return new byte[0];
@@ -190,19 +193,20 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
* @return part file content
*/
private List<String> readPartFileContent(String filePath,
- int skipLine,
- int limit) {
+ int skipLine,
+ int limit) {
File file = new File(filePath);
if (file.exists() && file.isFile()) {
try (Stream<String> stream = Files.lines(Paths.get(filePath))) {
return stream.skip(skipLine).limit(limit).collect(Collectors.toList());
- } catch (IOException e) {
+ }
+ catch (IOException e) {
logger.error("read file error", e);
}
- } else {
+ }
+ else {
logger.info("file path: {} not exists", filePath);
}
return Collections.emptyList();
}
-
}
diff --git a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/MasterLogFilter.java b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/MasterLogFilter.java
index 575571d..255b0c5 100644
--- a/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/MasterLogFilter.java
+++ b/dolphinscheduler-server/src/main/java/org/apache/dolphinscheduler/server/log/MasterLogFilter.java
@@ -24,7 +24,8 @@ import ch.qos.logback.core.spi.FilterReply;
/**
* master log filter
*/
-public class MasterLogFilter extends Filter<ILoggingEvent> {
+public class MasterLogFilter
+ extends Filter<ILoggingEvent> {
/**
* log level
*/
@@ -32,12 +33,13 @@ public class MasterLogFilter extends Filter<ILoggingEvent> {
/**
* Accept or reject based on thread name
+ *
* @param event event
* @return FilterReply
*/
@Override
public FilterReply decide(ILoggingEvent event) {
- if (event.getThreadName().startsWith("Master-") ){
+ if (event.getThreadName().startsWith("Master-")) {
return FilterReply.ACCEPT;
}
return FilterReply.DENY;
@@ -46,4 +48,4 @@ public class MasterLogFilter extends Filter<ILoggingEvent> {
public void setLevel(String level) {
this.level = Level.toLevel(level);
}
-}
\ No newline at end of file
+}
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java
index b10c60a..13eb520 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessorTest.java
@@ -17,26 +17,37 @@
package org.apache.dolphinscheduler.server.log;
+import io.netty.channel.Channel;
+import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.common.utils.LoggerUtils;
import org.apache.dolphinscheduler.remote.command.Command;
import org.apache.dolphinscheduler.remote.command.CommandType;
import org.apache.dolphinscheduler.remote.command.log.ViewLogRequestCommand;
-
+import org.junit.Before;
import org.junit.Test;
-import org.junit.Test.None;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.powermock.api.mockito.PowerMockito;
import org.powermock.core.classloader.annotations.PrepareForTest;
import org.powermock.modules.junit4.PowerMockRunner;
-import io.netty.channel.Channel;
-
@RunWith(PowerMockRunner.class)
@PrepareForTest({LoggerUtils.class})
public class LoggerRequestProcessorTest {
+ private String dsHome;
+
+ @Before
+ public void initDsHome() {
+ // DOLPHINSCHEDULER_HOME is be set in start.sh. if we run test in IDE user.dir is DS Home.
+ dsHome = System.getProperty("DOLPHINSCHEDULER_HOME");
+ if (StringUtils.isBlank(dsHome)) {
+ dsHome = System.getProperty("user.dir");
+ System.setProperty("DOLPHINSCHEDULER_HOME", dsHome);
+ }
+ }
+
@Test
public void testProcessViewWholeLogRequest() {
System.setProperty("DOLPHINSCHEDULER_HOME", System.getProperty("user.dir"));
@@ -107,4 +118,4 @@ public class LoggerRequestProcessorTest {
LoggerRequestProcessor loggerRequestProcessor = new LoggerRequestProcessor();
loggerRequestProcessor.process(channel, command);
}
-}
\ No newline at end of file
+}
diff --git a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java
index 74bdcea..0dc5fb8 100644
--- a/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java
+++ b/dolphinscheduler-server/src/test/java/org/apache/dolphinscheduler/server/log/LoggerServerTest.java
@@ -17,57 +17,68 @@
package org.apache.dolphinscheduler.server.log;
+import org.apache.commons.lang.StringUtils;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.common.utils.FileUtils;
import org.apache.dolphinscheduler.service.log.LogClientService;
-
-import org.apache.commons.lang.StringUtils;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.Charset;
-
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.Charset;
+
public class LoggerServerTest {
private LoggerServer loggerServer;
private LogClientService logClientService;
+ private String dsHome;
+
@Before
public void startServerAndClient() {
this.loggerServer = new LoggerServer();
this.loggerServer.start();
this.logClientService = new LogClientService();
+
+ // DOLPHINSCHEDULER_HOME is be set in start.sh. if we run test in IDE user.dir is DS Home.
+ dsHome = System.getProperty("DOLPHINSCHEDULER_HOME");
+ if (StringUtils.isBlank(dsHome)) {
+ dsHome = System.getProperty("user.dir");
+ System.setProperty("DOLPHINSCHEDULER_HOME", dsHome);
+ }
}
@Test
- public void testRollViewLog() throws IOException {
+ public void testRollViewLog()
+ throws IOException {
String expectedTmpDemoString = "testRolloViewLog";
- org.apache.commons.io.FileUtils.writeStringToFile(new File("/tmp/demo.txt"), expectedTmpDemoString, Charset.defaultCharset());
+ String testFile = dsHome + "/tmp/demo.log";
+ org.apache.commons.io.FileUtils.writeStringToFile(new File(testFile), expectedTmpDemoString, Charset.defaultCharset());
String resultTmpDemoString = this.logClientService.rollViewLog(
- "localhost", Constants.RPC_PORT,"/tmp/demo.txt", 0, 1000);
+ "localhost", Constants.RPC_PORT, testFile, 0, 1000);
Assert.assertEquals(expectedTmpDemoString, resultTmpDemoString.replaceAll("[\r|\n|\t]", StringUtils.EMPTY));
- FileUtils.deleteFile("/tmp/demo.txt");
+ FileUtils.deleteFile(testFile);
}
@Test
- public void testRemoveTaskLog() throws IOException {
+ public void testRemoveTaskLog()
+ throws IOException {
String expectedTmpRemoveString = "testRemoveTaskLog";
- org.apache.commons.io.FileUtils.writeStringToFile(new File("/tmp/remove.txt"), expectedTmpRemoveString, Charset.defaultCharset());
+ String testFile = dsHome + "/tmp/remove.log";
+ org.apache.commons.io.FileUtils.writeStringToFile(new File(testFile), expectedTmpRemoveString, Charset.defaultCharset());
- Boolean b = this.logClientService.removeTaskLog("localhost", Constants.RPC_PORT,"/tmp/remove.txt");
+ Boolean b = this.logClientService.removeTaskLog("localhost", Constants.RPC_PORT, testFile);
Assert.assertTrue(b);
- String result = this.logClientService.viewLog("localhost", Constants.RPC_PORT,"/tmp/demo.txt");
+ String result = this.logClientService.viewLog("localhost", Constants.RPC_PORT, testFile);
Assert.assertEquals(StringUtils.EMPTY, result);
}