You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by we...@apache.org on 2022/07/26 02:14:50 UTC
[dolphinscheduler] branch dev updated: [Bug] [API] If the single log length is long, it will causes the View Log page to hang (#10612)
This is an automated email from the ASF dual-hosted git repository.
wenjun pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 4d9db3451e [Bug] [API] If the single log length is long, it will causes the View Log page to hang (#10612)
4d9db3451e is described below
commit 4d9db3451ed7d038f5cf7815e61b669cdcf8164a
Author: zhuxt2015 <59...@qq.com>
AuthorDate: Tue Jul 26 10:14:45 2022 +0800
[Bug] [API] If the single log length is long, it will causes the View Log page to hang (#10612)
---
.../api/controller/LoggerController.java | 9 ++++---
.../api/service/LoggerService.java | 3 ++-
.../api/service/impl/LoggerServiceImpl.java | 10 ++++---
.../service/impl/ProcessInstanceServiceImpl.java | 5 ++--
.../dao/entity/ResponseTaskLog.java | 31 ++++++++++++++++++++++
.../server/log/LoggerRequestProcessor.java | 19 ++++++++++++-
.../src/views/projects/task/instance/index.tsx | 8 +++---
7 files changed, 69 insertions(+), 16 deletions(-)
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoggerController.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoggerController.java
index 88c715fab6..f24ee0273b 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoggerController.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/controller/LoggerController.java
@@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.api.exceptions.ApiException;
import org.apache.dolphinscheduler.api.service.LoggerService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
+import org.apache.dolphinscheduler.dao.entity.ResponseTaskLog;
import org.apache.dolphinscheduler.dao.entity.User;
import org.springframework.beans.factory.annotation.Autowired;
@@ -77,10 +78,10 @@ public class LoggerController extends BaseController {
@ResponseStatus(HttpStatus.OK)
@ApiException(QUERY_TASK_INSTANCE_LOG_ERROR)
@AccessLogAnnotation(ignoreRequestArgs = "loginUser")
- public Result<String> queryLog(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
- @RequestParam(value = "taskInstanceId") int taskInstanceId,
- @RequestParam(value = "skipLineNum") int skipNum,
- @RequestParam(value = "limit") int limit) {
+ public Result<ResponseTaskLog> queryLog(@ApiIgnore @RequestAttribute(value = Constants.SESSION_USER) User loginUser,
+ @RequestParam(value = "taskInstanceId") int taskInstanceId,
+ @RequestParam(value = "skipLineNum") int skipNum,
+ @RequestParam(value = "limit") int limit) {
return loggerService.queryLog(taskInstanceId, skipNum, limit);
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java
index b252522c7c..f7ba946e4c 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/LoggerService.java
@@ -18,6 +18,7 @@
package org.apache.dolphinscheduler.api.service;
import org.apache.dolphinscheduler.api.utils.Result;
+import org.apache.dolphinscheduler.dao.entity.ResponseTaskLog;
import org.apache.dolphinscheduler.dao.entity.User;
import java.util.Map;
@@ -35,7 +36,7 @@ public interface LoggerService {
* @param limit limit
* @return log string data
*/
- Result<String> queryLog(int taskInstId, int skipLineNum, int limit);
+ Result<ResponseTaskLog> queryLog(int taskInstId, int skipLineNum, int limit);
/**
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java
index 65aa7b3aee..39f9208b5f 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/LoggerServiceImpl.java
@@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.api.service.ProjectService;
import org.apache.dolphinscheduler.api.utils.Result;
import org.apache.dolphinscheduler.common.Constants;
import org.apache.dolphinscheduler.dao.entity.Project;
+import org.apache.dolphinscheduler.dao.entity.ResponseTaskLog;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.User;
@@ -100,7 +101,7 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService
*/
@Override
@SuppressWarnings("unchecked")
- public Result<String> queryLog(int taskInstId, int skipLineNum, int limit) {
+ public Result<ResponseTaskLog> queryLog(int taskInstId, int skipLineNum, int limit) {
TaskInstance taskInstance = processService.findTaskInstanceById(taskInstId);
@@ -110,9 +111,10 @@ public class LoggerServiceImpl extends BaseServiceImpl implements LoggerService
if (StringUtils.isBlank(taskInstance.getHost())) {
return Result.error(Status.TASK_INSTANCE_HOST_IS_NULL);
}
- Result<String> result = new Result<>(Status.SUCCESS.getCode(), Status.SUCCESS.getMsg());
- String log = queryLog(taskInstance,skipLineNum,limit);
- result.setData(log);
+ Result<ResponseTaskLog> result = new Result<>(Status.SUCCESS.getCode(), Status.SUCCESS.getMsg());
+ String log = queryLog(taskInstance, skipLineNum, limit);
+ int lineNum = log.split("\\r\\n").length;
+ result.setData(new ResponseTaskLog(lineNum, log));
return result;
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java
index 2335ff4c88..99427b954a 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/service/impl/ProcessInstanceServiceImpl.java
@@ -53,6 +53,7 @@ import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
import org.apache.dolphinscheduler.dao.entity.Project;
+import org.apache.dolphinscheduler.dao.entity.ResponseTaskLog;
import org.apache.dolphinscheduler.dao.entity.TaskDefinition;
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
@@ -352,10 +353,10 @@ public class ProcessInstanceServiceImpl extends BaseServiceImpl implements Proce
private void addDependResultForTaskList(List<TaskInstance> taskInstanceList) throws IOException {
for (TaskInstance taskInstance : taskInstanceList) {
if (TASK_TYPE_DEPENDENT.equalsIgnoreCase(taskInstance.getTaskType())) {
- Result<String> logResult = loggerService.queryLog(
+ Result<ResponseTaskLog> logResult = loggerService.queryLog(
taskInstance.getId(), Constants.LOG_QUERY_SKIP_LINE_NUMBER, Constants.LOG_QUERY_LIMIT);
if (logResult.getCode() == Status.SUCCESS.ordinal()) {
- String log = logResult.getData();
+ String log = logResult.getData().getMessage();
Map<String, DependResult> resultMap = parseLogForDependentResult(log);
taskInstance.setDependentResult(JSONUtils.toJsonString(resultMap));
}
diff --git a/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseTaskLog.java b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseTaskLog.java
new file mode 100644
index 0000000000..6876c060f8
--- /dev/null
+++ b/dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/entity/ResponseTaskLog.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.dao.entity;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+
+/**
+ * log of the logger service response
+ */
+@Data
+@AllArgsConstructor
+public class ResponseTaskLog {
+ private int lineNum;
+ private String message;
+}
diff --git a/dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java b/dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
index 1ab4aa13fe..b5cae3562d 100644
--- a/dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
+++ b/dolphinscheduler-log-server/src/main/java/org/apache/dolphinscheduler/server/log/LoggerRequestProcessor.java
@@ -40,6 +40,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Collections;
@@ -113,8 +114,24 @@ public class LoggerRequestProcessor implements NettyRequestProcessor {
List<String> lines = readPartFileContent(rollViewLogPath,
rollViewLogRequest.getSkipLineNum(), rollViewLogRequest.getLimit());
StringBuilder builder = new StringBuilder();
+ final int MaxResponseLogSize = 65535;
+ int totalLogByteSize = 0;
for (String line : lines) {
- builder.append(line).append("\r\n");
+ //If a single line of log is exceed max response size, cut off the line
+ final int lineByteSize = line.getBytes(StandardCharsets.UTF_8).length;
+ if (lineByteSize >= MaxResponseLogSize) {
+ builder.append(line, 0, MaxResponseLogSize)
+ .append(" [this line's size ").append(lineByteSize).append(" bytes is exceed ")
+ .append(MaxResponseLogSize).append(" bytes, so only ")
+ .append(MaxResponseLogSize).append(" characters are reserved for performance reasons.]")
+ .append("\r\n");
+ } else {
+ builder.append(line).append("\r\n");
+ }
+ totalLogByteSize += lineByteSize;
+ if (totalLogByteSize >= MaxResponseLogSize) {
+ break;
+ }
}
RollViewLogResponseCommand rollViewLogRequestResponse = new RollViewLogResponseCommand(builder.toString());
channel.writeAndFlush(rollViewLogRequestResponse.convert2Command(command.getOpaque()));
diff --git a/dolphinscheduler-ui/src/views/projects/task/instance/index.tsx b/dolphinscheduler-ui/src/views/projects/task/instance/index.tsx
index 9986f8844a..c742a20d20 100644
--- a/dolphinscheduler-ui/src/views/projects/task/instance/index.tsx
+++ b/dolphinscheduler-ui/src/views/projects/task/instance/index.tsx
@@ -76,11 +76,11 @@ const TaskInstance = defineComponent({
taskInstanceId: Number(row.id),
limit: variables.limit,
skipLineNum: variables.skipLineNum
- }).then((res: string) => {
- variables.logRef += res
- if (res) {
+ }).then((res: any) => {
+ if (res?.message) {
+ variables.logRef += res.message
variables.limit += 1000
- variables.skipLineNum += 1000
+ variables.skipLineNum += res.lineNum
getLogs(row)
} else {
variables.logLoadingRef = false