You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by pe...@apache.org on 2023/03/07 12:01:16 UTC
[linkis] branch dev-1.3.2 updated: Fix engine launch with -Xms (#4308)
This is an automated email from the ASF dual-hosted git repository.
peacewong pushed a commit to branch dev-1.3.2
in repository https://gitbox.apache.org/repos/asf/linkis.git
The following commit(s) were added to refs/heads/dev-1.3.2 by this push:
new dfc9ee72b Fix engine launch with -Xms (#4308)
dfc9ee72b is described below
commit dfc9ee72b4186d730dcc187636dea8f869a978a9
Author: Casion <ca...@gmail.com>
AuthorDate: Tue Mar 7 20:01:09 2023 +0800
Fix engine launch with -Xms (#4308)
---
.../JavaProcessEngineConnLaunchBuilder.scala | 21 ++++++++++++---
.../common/util/NodeResourceUtils.scala | 5 +++-
.../executor/OpenLooKengEngineConnExecutor.java | 20 +++------------
.../ShellEngineConnConcurrentExecutor.scala | 30 +++++-----------------
4 files changed, 33 insertions(+), 43 deletions(-)
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/launch/process/JavaProcessEngineConnLaunchBuilder.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/launch/process/JavaProcessEngineConnLaunchBuilder.scala
index bac742fae..89b1e6250 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/launch/process/JavaProcessEngineConnLaunchBuilder.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/launch/process/JavaProcessEngineConnLaunchBuilder.scala
@@ -64,9 +64,24 @@ abstract class JavaProcessEngineConnLaunchBuilder
val commandLine: ArrayBuffer[String] = ArrayBuffer[String]()
commandLine += (variable(JAVA_HOME) + "/bin/java")
commandLine += "-server"
- val engineConnMemory = EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.getValue.toString
- commandLine += ("-Xmx" + engineConnMemory)
- // commandLine += ("-Xms" + engineConnMemory)
+
+ val properties = engineConnBuildRequest.engineConnCreationDesc.properties
+ if (properties.containsKey(EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.key)) {
+ var settingClientMemory =
+ properties.get(EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.key)
+ if (
+ !settingClientMemory
+ .toLowerCase()
+ .endsWith("g") && !settingClientMemory.toLowerCase().endsWith("m")
+ ) {
+ settingClientMemory = settingClientMemory + "g"
+ }
+ commandLine += ("-Xmx" + settingClientMemory)
+ } else {
+ val engineConnMemory = EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.getValue.toString
+ commandLine += ("-Xmx" + engineConnMemory)
+ }
+
val javaOPTS = getExtractJavaOpts
if (StringUtils.isNotEmpty(EnvConfiguration.ENGINE_CONN_DEFAULT_JAVA_OPTS.getValue)) {
EnvConfiguration.ENGINE_CONN_DEFAULT_JAVA_OPTS.getValue
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/util/NodeResourceUtils.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/util/NodeResourceUtils.scala
index be4e56805..a25964ac6 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/util/NodeResourceUtils.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/scala/org/apache/linkis/manager/engineplugin/common/util/NodeResourceUtils.scala
@@ -35,7 +35,10 @@ object NodeResourceUtils {
properties.get(EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.key)
if (StringUtils.isBlank(settingClientMemory)) {
properties.remove(EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.key)
- } else if (!settingClientMemory.toLowerCase().endsWith("g")) {
+ } else if (
+ !settingClientMemory.toLowerCase().endsWith("g")
+ && !settingClientMemory.toLowerCase().endsWith("m")
+ ) {
properties.put(
EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.key,
settingClientMemory + "g"
diff --git a/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/executor/OpenLooKengEngineConnExecutor.java b/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/executor/OpenLooKengEngineConnExecutor.java
index bed705a33..def097b38 100644
--- a/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/executor/OpenLooKengEngineConnExecutor.java
+++ b/linkis-engineconn-plugins/openlookeng/src/main/java/org/apache/linkis/engineplugin/openlookeng/executor/OpenLooKengEngineConnExecutor.java
@@ -33,9 +33,8 @@ import org.apache.linkis.engineplugin.openlookeng.exception.OpenLooKengClientExc
import org.apache.linkis.engineplugin.openlookeng.exception.OpenLooKengStateInvalidException;
import org.apache.linkis.governance.common.paser.SQLCodeParser;
import org.apache.linkis.manager.common.entity.resource.CommonNodeResource;
-import org.apache.linkis.manager.common.entity.resource.LoadResource;
import org.apache.linkis.manager.common.entity.resource.NodeResource;
-import org.apache.linkis.manager.engineplugin.common.conf.EngineConnPluginConf;
+import org.apache.linkis.manager.engineplugin.common.util.NodeResourceUtils;
import org.apache.linkis.manager.label.entity.Label;
import org.apache.linkis.manager.label.entity.engine.EngineTypeLabel;
import org.apache.linkis.manager.label.entity.engine.UserCreatorLabel;
@@ -221,21 +220,10 @@ public class OpenLooKengEngineConnExecutor extends ConcurrentComputationExecutor
@Override
public NodeResource getCurrentNodeResource() {
- Map<String, String> properties = EngineConnObject.getEngineCreationContext().getOptions();
- if (properties.containsKey(EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY().key())) {
- String settingClientMemory =
- properties.get(EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY().key());
- if (!settingClientMemory.toLowerCase().endsWith("g")) {
- properties.put(
- EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY().key(), settingClientMemory + "g");
- }
- }
CommonNodeResource resource = new CommonNodeResource();
- LoadResource usedResource =
- new LoadResource(
- EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY().getValue(properties).toLong(),
- (Integer) EngineConnPluginConf.JAVA_ENGINE_REQUEST_CORES().getValue(properties));
- resource.setUsedResource(usedResource);
+ resource.setUsedResource(
+ NodeResourceUtils.applyAsLoadInstanceResource(
+ EngineConnObject.getEngineCreationContext().getOptions()));
return resource;
}
diff --git a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnConcurrentExecutor.scala b/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnConcurrentExecutor.scala
index e127f7abb..b66353ca0 100644
--- a/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnConcurrentExecutor.scala
+++ b/linkis-engineconn-plugins/shell/src/main/scala/org/apache/linkis/manager/engineplugin/shell/executor/ShellEngineConnConcurrentExecutor.scala
@@ -17,9 +17,8 @@
package org.apache.linkis.manager.engineplugin.shell.executor
-import org.apache.linkis.common.utils.{Logging, Utils}
+import org.apache.linkis.common.utils.{Logging, OverloadUtils, Utils}
import org.apache.linkis.engineconn.computation.executor.execute.{
- ComputationExecutor,
ConcurrentComputationExecutor,
EngineExecutionContext
}
@@ -27,10 +26,10 @@ import org.apache.linkis.engineconn.core.EngineConnObject
import org.apache.linkis.governance.common.utils.GovernanceUtils
import org.apache.linkis.manager.common.entity.resource.{
CommonNodeResource,
- LoadInstanceResource,
+ LoadResource,
NodeResource
}
-import org.apache.linkis.manager.engineplugin.common.conf.EngineConnPluginConf
+import org.apache.linkis.manager.engineplugin.common.util.NodeResourceUtils
import org.apache.linkis.manager.engineplugin.shell.common.ShellEngineConnPluginConst
import org.apache.linkis.manager.engineplugin.shell.conf.ShellEngineConnConf
import org.apache.linkis.manager.engineplugin.shell.exception.ShellCodeErrorException
@@ -43,7 +42,6 @@ import org.apache.linkis.scheduler.executer.{
SuccessExecuteResponse
}
-import org.apache.commons.io.IOUtils
import org.apache.commons.lang3.StringUtils
import java.io.{BufferedReader, File, InputStreamReader}
@@ -274,25 +272,11 @@ class ShellEngineConnConcurrentExecutor(id: Int, maxRunningNumber: Int)
}
override def getCurrentNodeResource(): NodeResource = {
- // todo refactor for duplicate
- val properties = EngineConnObject.getEngineCreationContext.getOptions
- if (properties.containsKey(EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.key)) {
- val settingClientMemory =
- properties.get(EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.key)
- if (!settingClientMemory.toLowerCase().endsWith("g")) {
- properties.put(
- EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.key,
- settingClientMemory + "g"
- )
- }
- }
- val actualUsedResource = new LoadInstanceResource(
- EngineConnPluginConf.JAVA_ENGINE_REQUEST_MEMORY.getValue(properties).toLong,
- EngineConnPluginConf.JAVA_ENGINE_REQUEST_CORES.getValue(properties),
- EngineConnPluginConf.JAVA_ENGINE_REQUEST_INSTANCE
- )
val resource = new CommonNodeResource
- resource.setUsedResource(actualUsedResource)
+ resource.setUsedResource(
+ NodeResourceUtils
+ .applyAsLoadInstanceResource(EngineConnObject.getEngineCreationContext.getOptions)
+ )
resource
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org