You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by pe...@apache.org on 2022/08/05 07:52:44 UTC
[incubator-linkis] branch dev-1.2.1 updated: Fix shell engine full log print (#2552)
This is an automated email from the ASF dual-hosted git repository.
peacewong pushed a commit to branch dev-1.2.1
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
The following commit(s) were added to refs/heads/dev-1.2.1 by this push:
new f1c0a22b9 Fix shell engine full log print (#2552)
f1c0a22b9 is described below
commit f1c0a22b9d4686e3cd990dfadeaabf8bbf75afe1
Author: huangKai-2323 <62...@users.noreply.github.com>
AuthorDate: Fri Aug 5 15:52:37 2022 +0800
Fix shell engine full log print (#2552)
---
.../resources/conf/linkis-engineconn.properties | 3 +-
.../shell/executor/ErrorStreamReaderThread.scala | 72 -----------------
.../engineplugin/shell/executor/ReaderThread.scala | 89 ++++++++++++++++++++++
.../shell/executor/ShellEngineConnExecutor.scala | 51 +++++--------
4 files changed, 109 insertions(+), 106 deletions(-)
diff --git a/linkis-engineconn-plugins/shell/src/main/resources/conf/linkis-engineconn.properties b/linkis-engineconn-plugins/shell/src/main/resources/conf/linkis-engineconn.properties
index 5350e7a3c..3233e537b 100644
--- a/linkis-engineconn-plugins/shell/src/main/resources/conf/linkis-engineconn.properties
+++ b/linkis-engineconn-plugins/shell/src/main/resources/conf/linkis-engineconn.properties
@@ -16,4 +16,5 @@
#
wds.linkis.server.version=v1
wds.linkis.engineconn.plugin.default.class=org.apache.linkis.manager.engineplugin.shell.ShellEngineConnPlugin
-wds.linkis.engineconn.max.free.time=30m
\ No newline at end of file
+wds.linkis.engineconn.max.free.time=30m
+wds.linkis.engineconn.log.list.count=50
\ No newline at end of file
diff --git a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ErrorStreamReaderThread.scala b/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ErrorStreamReaderThread.scala
deleted file mode 100644
index e04d07d59..000000000
--- a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ErrorStreamReaderThread.scala
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.manager.engineplugin.shell.executor
-
-import org.apache.linkis.common.utils.{Logging, Utils}
-import org.apache.linkis.engineconn.acessible.executor.log.LogHelper
-import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext
-
-import java.io.BufferedReader
-
-class ErrorStreamReaderThread extends Thread with Logging {
- private var engineExecutionContext: EngineExecutionContext = _
- private var errReader : BufferedReader = _
- private var extractor: YarnAppIdExtractor = _
-
- def this(engineExecutionContext: EngineExecutionContext, errReader: BufferedReader, extractor: YarnAppIdExtractor) {
- this()
- this.errReader = errReader
- this.engineExecutionContext = engineExecutionContext
- this.extractor = extractor
- }
-
- def onDestroy(): Unit = {
- Utils.tryCatch{
- errReader synchronized errReader.close()
- }{ t =>
- logger.warn("Error while closing the error stream", t)
- }
- }
-
-
- def startReaderThread(): Unit = {
- Utils.tryCatch {
- this.start()
- }{ t =>
- if (t.isInstanceOf[OutOfMemoryError]) {
- logger.warn("Caught " + t + ". One possible reason is that ulimit" + " setting of 'max user processes' is too low. If so, do" + " 'ulimit -u <largerNum>' and try again.")
- }
- logger.warn("Cannot start thread to read from error stream", t)
- }
- }
-
- override def run(): Unit = {
- Utils.tryCatch {
- var line = errReader.readLine
- while ( {
- (line != null) && !isInterrupted
- }) {
- LogHelper.logCache.cacheLog(line)
- extractor.appendLineToExtractor(line)
- line = errReader.readLine
- }
- }{ t =>
- logger.warn("Error reading the error stream", t)
- }
- }
-}
diff --git a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ReaderThread.scala b/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ReaderThread.scala
new file mode 100644
index 000000000..17cbaab3f
--- /dev/null
+++ b/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ReaderThread.scala
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.manager.engineplugin.shell.executor
+
+import org.apache.commons.lang3.StringUtils
+import org.apache.linkis.common.conf.CommonVars
+import org.apache.linkis.common.utils.{Logging, Utils}
+import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext
+import java.io.BufferedReader
+import java.util
+
+
+class ReaderThread extends Thread with Logging {
+ private var engineExecutionContext: EngineExecutionContext = _
+ private var inputReader: BufferedReader = _
+ private var extractor: YarnAppIdExtractor = _
+ private var isStdout: Boolean = false
+ private val logListCount = CommonVars[Int]("wds.linkis.engineconn.log.list.count", 50)
+
+ def this(engineExecutionContext: EngineExecutionContext, inputReader: BufferedReader, extractor: YarnAppIdExtractor, isStdout: Boolean) {
+ this()
+ this.inputReader = inputReader
+ this.engineExecutionContext = engineExecutionContext
+ this.extractor = extractor
+ this.isStdout = isStdout
+ }
+
+ def onDestroy(): Unit = {
+ Utils.tryCatch {
+ inputReader synchronized inputReader.close()
+ } { t =>
+ logger.warn("inputReader while closing the error stream", t)
+ }
+ }
+
+
+ def startReaderThread(): Unit = {
+ Utils.tryCatch {
+ this.start()
+ } { t =>
+ if (t.isInstanceOf[OutOfMemoryError]) {
+ logger.warn("Caught " + t + ". One possible reason is that ulimit" + " setting of 'max user processes' is too low. If so, do" + " 'ulimit -u <largerNum>' and try again.")
+ }
+ logger.warn("Cannot start thread to read from inputReader stream", t)
+ }
+ }
+
+ override def run(): Unit = {
+ Utils.tryCatch {
+ var line: String = null
+ val logArray: util.List[String] = new util.ArrayList[String]
+ while ({line = inputReader.readLine(); line != null}) {
+ logger.info("read logger line :{}", line)
+ logArray.add(line)
+ if (logArray.size > logListCount.getValue) {
+ val linelist = StringUtils.join(logArray, "\n")
+ extractor.appendLineToExtractor(linelist)
+ if (isStdout) engineExecutionContext.appendStdout(linelist)
+ engineExecutionContext.appendTextResultSet(linelist)
+ logArray.clear()
+ }
+ }
+ if (logArray.size > 0) {
+ val linelist = StringUtils.join(logArray, "\n")
+ extractor.appendLineToExtractor(linelist)
+ if (isStdout) engineExecutionContext.appendStdout(linelist)
+ engineExecutionContext.appendTextResultSet(linelist)
+ logArray.clear()
+ }
+ } { t =>
+ logger.warn("inputReader reading the input stream", t)
+ }
+ }
+}
diff --git a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnExecutor.scala b/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnExecutor.scala
index e13c4b138..5fa59360f 100644
--- a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnExecutor.scala
+++ b/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnExecutor.scala
@@ -14,16 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
package org.apache.linkis.manager.engineplugin.shell.executor
import org.apache.commons.io.IOUtils
-
import org.apache.commons.lang3.StringUtils
-import org.apache.hadoop.util.Shell
-
import org.apache.linkis.common.utils.{Logging, Utils}
-import org.apache.linkis.engineconn.acessible.executor.log.LogHelper
import org.apache.linkis.engineconn.computation.executor.execute.{ComputationExecutor, EngineExecutionContext}
import org.apache.linkis.engineconn.core.EngineConnObject
import org.apache.linkis.governance.common.utils.GovernanceUtils
@@ -36,7 +32,7 @@ import org.apache.linkis.protocol.engine.JobProgressInfo
import org.apache.linkis.rpc.Sender
import org.apache.linkis.scheduler.executer.{ErrorExecuteResponse, ExecuteResponse, SuccessExecuteResponse}
import scala.collection.JavaConverters._
-import java.io.{BufferedReader, File, InputStreamReader}
+import java.io.{BufferedReader, File, FileReader, IOException, InputStreamReader}
import java.util
import java.util.concurrent.atomic.AtomicBoolean
import scala.collection.mutable.ArrayBuffer
@@ -73,7 +69,8 @@ class ShellEngineConnExecutor(id: Int) extends ComputationExecutor with Logging
var errorsReader: BufferedReader = null
val completed = new AtomicBoolean(false)
- var errReaderThread: ErrorStreamReaderThread = null
+ var errReaderThread: ReaderThread = null
+ var inputReaderThread: ReaderThread = null
try {
engineExecutionContext.appendStdout(s"$getId >> ${code.trim}")
@@ -81,11 +78,11 @@ class ShellEngineConnExecutor(id: Int) extends ComputationExecutor with Logging
val argsArr = if (engineExecutionContext.getTotalParagraph == 1 &&
engineExecutionContext.getProperties != null &&
engineExecutionContext.getProperties.containsKey(ShellEngineConnPluginConst.RUNTIME_ARGS_KEY)) {
- Utils.tryCatch{
+ Utils.tryCatch {
val argsList = engineExecutionContext.getProperties.get(ShellEngineConnPluginConst.RUNTIME_ARGS_KEY).asInstanceOf[util.ArrayList[String]]
logger.info("Will execute shell task with user-specified arguments: \'" + argsList.toArray(new Array[String](argsList.size())).mkString("\' \'") + "\'")
argsList.toArray(new Array[String](argsList.size()))
- }{ t =>
+ } { t =>
logger.warn("Cannot read user-input shell arguments. Will execute shell task without them.", t)
null
}
@@ -125,35 +122,23 @@ class ShellEngineConnExecutor(id: Int) extends ComputationExecutor with Logging
}
processBuilder.redirectErrorStream(false)
+ extractor = new YarnAppIdExtractor
+ extractor.startExtraction()
process = processBuilder.start()
+
bufferedReader = new BufferedReader(new InputStreamReader(process.getInputStream))
errorsReader = new BufferedReader(new InputStreamReader(process.getErrorStream))
- /*
- Prepare to extract yarn application id
- */
- extractor = new YarnAppIdExtractor
- extractor.startExtraction()
- /*
- Read stderr with another thread
- */
- errReaderThread = new ErrorStreamReaderThread(engineExecutionContext, errorsReader, extractor)
- errReaderThread.start()
- /*
- Read stdout
- */
- var line: String = null
- while ( {
- line = bufferedReader.readLine(); line != null
- }) {
- logger.debug(s"$getId() >>> $line")
- LogHelper.logCache.cacheLog(line)
- engineExecutionContext.appendTextResultSet(line)
- extractor.appendLineToExtractor(line)
- }
+ inputReaderThread = new ReaderThread(engineExecutionContext, bufferedReader, extractor, true)
+ errReaderThread = new ReaderThread(engineExecutionContext, bufferedReader, extractor, false)
+
+ inputReaderThread.start()
+ errReaderThread.start()
val exitCode = process.waitFor()
+ joinThread(inputReaderThread)
joinThread(errReaderThread)
+
completed.set(true)
if (exitCode != 0) {
@@ -172,10 +157,10 @@ class ShellEngineConnExecutor(id: Int) extends ComputationExecutor with Logging
}
extractor.onDestroy()
+ inputReaderThread.onDestroy()
errReaderThread.onDestroy()
IOUtils.closeQuietly(bufferedReader)
-
IOUtils.closeQuietly(errorsReader)
}
}
@@ -194,7 +179,7 @@ class ShellEngineConnExecutor(id: Int) extends ComputationExecutor with Logging
}
private def joinThread(thread: Thread) = {
- while ( {
+ while ({
thread.isAlive
}) {
Utils.tryCatch{
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org