You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by pe...@apache.org on 2023/03/28 13:35:33 UTC
[linkis] branch dev-1.3.2 updated: appcon compatibility (#4420)
This is an automated email from the ASF dual-hosted git repository.
peacewong pushed a commit to branch dev-1.3.2
in repository https://gitbox.apache.org/repos/asf/linkis.git
The following commit(s) were added to refs/heads/dev-1.3.2 by this push:
new 8d2179123 appcon compatibility (#4420)
8d2179123 is described below
commit 8d2179123065204b8ba4d85a92821d22ed016ce0
Author: Casion <ca...@gmail.com>
AuthorDate: Tue Mar 28 21:35:24 2023 +0800
appcon compatibility (#4420)
* engine bml resource compatible prefix v format
---
.../linkis-engineconn-plugin-core-errorcode.md | 2 +-
.../linkis/server/security/SecurityFilter.scala | 6 +++-
.../errorcode/EngineconnCoreErrorCodeSummary.java | 2 +-
.../conf/EngineConnPluginConfiguration.scala | 4 +++
.../AbstractEngineConnBmlResourceGenerator.scala | 35 ++++++++++++++++++----
.../service/DefaultEngineConnResourceService.scala | 13 ++++++--
.../rm/service/impl/DefaultResourceManager.scala | 2 +-
linkis-web/src/apps/linkis/i18n/common/en.json | 2 +-
8 files changed, 53 insertions(+), 13 deletions(-)
diff --git a/docs/errorcode/linkis-engineconn-plugin-core-errorcode.md b/docs/errorcode/linkis-engineconn-plugin-core-errorcode.md
index a929d062d..0fc2521a7 100644
--- a/docs/errorcode/linkis-engineconn-plugin-core-errorcode.md
+++ b/docs/errorcode/linkis-engineconn-plugin-core-errorcode.md
@@ -3,7 +3,7 @@
| module name(模块名) | error code(错误码) | describe(描述) |enumeration name(枚举)| Exception Class(类名)|
| -------- | -------- | ----- |-----|-----|
|linkis-engineconn-plugin-core |10001|Failed to createEngineConnLaunchRequest(创建 EngineConnLaunchRequest失败)|FAILED_CREATE_ELR|EngineconnCoreErrorCodeSummary|
-|linkis-engineconn-plugin-core |10001|The engine plug-in material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)|EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION|EngineconnCoreErrorCodeSummary|
+|linkis-engineconn-plugin-core |10001|The engine plugin material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)|EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION|EngineconnCoreErrorCodeSummary|
|linkis-engineconn-plugin-core |10001|EngineTypeLabel are requested(需要参数 EngineTypeLabel)|ETL_REQUESTED|EngineconnCoreErrorCodeSummary|
|linkis-engineconn-plugin-core |20000|Cannot instance EngineConnExecution(无法实例化 EngineConnExecution)|CANNOT_INSTANCE_ECE|EngineconnCoreErrorCodeSummary|
|linkis-engineconn-plugin-core |20000|Cannot find default ExecutorFactory(找不到默认的 ExecutorFactory)|CANNOT_DEFAULT_EF|EngineconnCoreErrorCodeSummary|
diff --git a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala
index 411bda482..5cc796d23 100644
--- a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala
+++ b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala
@@ -199,7 +199,11 @@ object SecurityFilter {
def getLoginUsername(req: HttpServletRequest): String = {
if (Configuration.IS_TEST_MODE.getValue) {
- ServerConfiguration.BDP_TEST_USER.getValue;
+ val testUser = ServerConfiguration.BDP_TEST_USER.getValue
+ if (StringUtils.isBlank(testUser)) {
+ throw new IllegalUserTicketException("Need to set test user when enable test module")
+ }
+ testUser
} else {
getLoginUser(req).getOrElse(
throw new IllegalUserTicketException(ILLEGAL_USER_TOKEN.getErrorDesc)
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java
index 1685f4b65..f29b2a6ed 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-plugin-core/src/main/java/org/apache/linkis/manager/engineplugin/errorcode/EngineconnCoreErrorCodeSummary.java
@@ -23,7 +23,7 @@ public enum EngineconnCoreErrorCodeSummary implements LinkisErrorCode {
FAILED_CREATE_ELR(10001, "Failed to createEngineConnLaunchRequest(创建 EngineConnLaunchRequest失败)"),
EN_PLUGIN_MATERIAL_SOURCE_EXCEPTION(
10001,
- "The engine plug-in material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)"),
+ "The engine plugin material is abnormal, please check whether the material is uploaded successfully(引擎插件物料异常，请检查物料是否上传成功)"),
ETL_REQUESTED(10001, "EngineTypeLabel are requested(需要参数 EngineTypeLabel)"),
CANNOT_INSTANCE_ECE(20000, "Cannot instance EngineConnExecution(无法实例化 EngineConnExecution)"),
diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala
index de1add173..9a16d9b9e 100644
--- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala
+++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/conf/EngineConnPluginConfiguration.scala
@@ -35,4 +35,8 @@ object EngineConnPluginConfiguration {
val ENABLED_BML_UPLOAD_FAILED_EXIT: CommonVars[Boolean] =
CommonVars("wds.linkis.engineconn.bml.upload.failed.enable", true)
+ // for third party eg appconn/datax, if all update, can set to false then to remove
+ val EC_BML_VERSION_MAY_WITH_PREFIX_V: CommonVars[Boolean] =
+ CommonVars("linkis.engineconn.bml.version.may.with.prefix", true)
+
}
diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala
index fce3508c3..c21ee924e 100644
--- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala
+++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/localize/AbstractEngineConnBmlResourceGenerator.scala
@@ -17,6 +17,7 @@
package org.apache.linkis.engineplugin.server.localize
+import org.apache.linkis.engineplugin.server.conf.EngineConnPluginConfiguration
import org.apache.linkis.engineplugin.server.conf.EngineConnPluginConfiguration.ENGINE_CONN_HOME
import org.apache.linkis.engineplugin.server.localize.EngineConnBmlResourceGenerator.NO_VERSION_MARK
import org.apache.linkis.manager.engineplugin.common.exception.EngineConnPluginErrorException
@@ -53,13 +54,37 @@ abstract class AbstractEngineConnBmlResourceGenerator extends EngineConnBmlResou
val engineConnPackageHome = Paths.get(engineConnDistHome, version).toFile.getPath
logger.info("getEngineConnDistHome, engineConnPackageHome path:" + engineConnPackageHome)
val engineConnPackageHomeFile = new File(engineConnPackageHome)
+
if (!engineConnPackageHomeFile.exists()) {
- throw new EngineConnPluginErrorException(
- ENGINE_VERSION_NOT_FOUND.getErrorCode,
- MessageFormat.format(ENGINE_VERSION_NOT_FOUND.getErrorDesc, version, engineConnType)
- )
+ if (
+ !version.startsWith(
+ "v"
+ ) && EngineConnPluginConfiguration.EC_BML_VERSION_MAY_WITH_PREFIX_V.getValue
+ ) {
+ val versionOld = "v" + version
+ val engineConnPackageHomeOld = Paths.get(engineConnDistHome, versionOld).toFile.getPath
+ logger.info(
+ "try to getEngineConnDistHome with prefix v, engineConnPackageHome path:" + engineConnPackageHomeOld
+ )
+ val engineConnPackageHomeFileOld = new File(engineConnPackageHomeOld)
+ if (!engineConnPackageHomeFileOld.exists()) {
+ throw new EngineConnPluginErrorException(
+ ENGINE_VERSION_NOT_FOUND.getErrorCode,
+ MessageFormat.format(ENGINE_VERSION_NOT_FOUND.getErrorDesc, version, engineConnType)
+ )
+ } else {
+ engineConnPackageHomeOld
+ }
+ } else {
+ throw new EngineConnPluginErrorException(
+ ENGINE_VERSION_NOT_FOUND.getErrorCode,
+ MessageFormat.format(ENGINE_VERSION_NOT_FOUND.getErrorDesc, version, engineConnType)
+ )
+ }
+ } else {
+ engineConnPackageHome
}
- engineConnPackageHome
+
}
private def checkEngineConnDistHome(engineConnPackageHomePath: String): Unit = {
diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala
index 3ddfdc7bb..55f3b2568 100644
--- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala
+++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/engineplugin/server/service/DefaultEngineConnResourceService.scala
@@ -201,8 +201,6 @@ class DefaultEngineConnResourceService extends EngineConnResourceService with Lo
engineConnBmlResource.setFileName(localizeResource.fileName)
engineConnBmlResource.setFileSize(localizeResource.fileSize)
engineConnBmlResource.setLastModified(localizeResource.lastModified)
- if (version.startsWith("v")) engineConnBmlResource.setVersion(version.substring(1))
- else engineConnBmlResource.setVersion(version)
engineConnBmlResource.setVersion(version)
engineConnBmlResourceDao.save(engineConnBmlResource)
} else {
@@ -241,9 +239,18 @@ class DefaultEngineConnResourceService extends EngineConnResourceService with Lo
): EngineConnResource = {
val engineConnType = engineConnBMLResourceRequest.getEngineConnType
val version = engineConnBMLResourceRequest.getVersion
- val engineConnBmlResources = asScalaBufferConverter(
+ var engineConnBmlResources = asScalaBufferConverter(
engineConnBmlResourceDao.getAllEngineConnBmlResource(engineConnType, version)
)
+ if (
+ engineConnBmlResources.asScala.size == 0 && EngineConnPluginConfiguration.EC_BML_VERSION_MAY_WITH_PREFIX_V.getValue
+ ) {
+ logger.info("Try to get engine conn bml resource with prefex v")
+ engineConnBmlResources = asScalaBufferConverter(
+ engineConnBmlResourceDao.getAllEngineConnBmlResource(engineConnType, "v" + version)
+ )
+ }
+
val confBmlResourceMap = engineConnBmlResources.asScala
.find(_.getFileName == LaunchConstants.ENGINE_CONN_CONF_DIR_NAME + ".zip")
.map(parseToBmlResource)
diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala
index f1284268b..6145f8bc0 100644
--- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala
+++ b/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/rm/service/impl/DefaultResourceManager.scala
@@ -710,7 +710,7 @@ class DefaultResourceManager extends ResourceManager with Logging with Initializ
var heartbeatMsgMetrics = ""
Utils.tryAndWarn {
val oldMetrics = nodeMetricManagerPersistence.getNodeMetrics(ecNode)
- if (StringUtils.isNotBlank(oldMetrics.getHeartBeatMsg)) {
+ if (oldMetrics != null && StringUtils.isNotBlank(oldMetrics.getHeartBeatMsg)) {
heartbeatMsgMetrics = oldMetrics.getHeartBeatMsg
}
}
diff --git a/linkis-web/src/apps/linkis/i18n/common/en.json b/linkis-web/src/apps/linkis/i18n/common/en.json
index c6230555f..845b6fe03 100644
--- a/linkis-web/src/apps/linkis/i18n/common/en.json
+++ b/linkis-web/src/apps/linkis/i18n/common/en.json
@@ -629,7 +629,7 @@
"versionList": "Version List",
"rollback": "Rollback",
"checkEngineConnTypeAndVersion": "Please select the engine type and version",
- "upload": "Please click the button to upload the engine plug-in"
+ "upload": "Please click the button to upload the engine plugin"
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org