You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by al...@apache.org on 2022/02/14 03:39:13 UTC
[incubator-linkis] branch dev-udf updated: Added multiple features of Linkis-UDF version (#1447)
This is an automated email from the ASF dual-hosted git repository.
alexkun pushed a commit to branch dev-udf
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
The following commit(s) were added to refs/heads/dev-udf by this push:
new 7773a6f Added multiple features of Linkis-UDF version (#1447)
7773a6f is described below
commit 7773a6fca3ecba040bd01894ed89bb91d4a63a4c
Author: peacewong <wp...@gmail.com>
AuthorDate: Mon Feb 14 11:39:04 2022 +0800
Added multiple features of Linkis-UDF version (#1447)
* Adjust SimpleDateFormat to ThreadLocal variable close #1379
* Adjust log
* Implement the default retryHandler to facilitate java class inheritance
* provide a method of forced cleanup when HDFS fs cache is enabled close #1380
* Add retry to enable httpClient to support Gateway HA close #1381
* Access-Control-Allow-Methods Support configuration
* Access-Control-Allow-Methods Support configuration
* add method of getSessionTimeOut
* Adjust the default connection pool of the mybatis module to druid close #1384
* Added resource related constants
* optimize code
* Optimize the way the RPC module supports the @receiver annotation close #1385
* optimize code
* Fix the memory leak bug of FS in ResultSetReader close #1386
* HDFSFileSystem supports updating FS objects close #1387
* Add a method to get the row and col number of the file close #1388
* optimize code
* for client add getTableStatisticInfo method
* Add tool class for killing Linux process close #1399
* Add jsonCodeParser
* optimize code
* Add the resultLocation property to the JobRequest object close #1400
* Add bml copy resources restful_interface and rollback version restful_interface close #1401
* Add datasource-cient to provide client way for datasource to access the server close #1407
* The naming method of the jobhistory field needs to be converted to camelCase. close #1413
* The ps module adds a database-based distributed lock module. close #1416
* optimize code
* add RetryWaitTimeOutLabel
* Modify message-scheduler to the receiver of rpc
* Solve compilation problems
* Solve compilation problems
* optimize code
* Modify message-scheduler to the receiver of rpc
* Fix issue #1406
* Support Engine working time idle Maintain feature close #1402
* Record task yarn queue resource usage. close #1405
* Fix issue #1406
* merge the resource-manager-common module into the manager-common module $1420
* remove useless code
* RM supports record changes of resource usage #1423
* 1.The udf module adds version management function and stores udf resources to bml-server.
2.modify ps dependency linkis-message-scheduler to linkis-rpc. close #1422
* Solve compilation problems
* Add a Token authentication module for gateway. close #1430
* optimize code
* Modify message-scheduler to the receiver of rpc
* optimize code
* use spotless plugin to format java code
* fix Licenses issue
* use spotless plugin to format java code
* fix Licenses issue
* add gateway-authentication module
* add Licenses
* add gateway-authentication module
* add Licenses
* add Licenses
* use spotless plugin to format java code
* add OnceEngineConnHook
* Module optimization
* Cooperate with UDF to support version management optimization close #1422
* Module optimization
* Module optimization
* Support Engine working time idle Maintain feature close #1402
* computation engineConn supports asynchronous execution close #1443
* optimize code
* add init sql hook
* add reset label hook
* add use db hook
* Optimization task reporting error information
* Support reporting yarn resource information #1423
* error code optimize
* add get pid method
* ecm supports reporting actual resource usage to manager #1444
* optimize code
* Optimized to use GovernanceUtils.killProcess kill ec #1399
* Modify message-scheduler to the receiver of rpc
* Optimize ec startup directory format #1445
* Replace message-scheduler with rpc module
* Module optimization
* Module optimization
Co-authored-by: HmhWz <93...@qq.com>
---
.gitignore | 2 -
.../assembly-combined/sbin/kill-process-by-pid.sh | 81 ++
.../linkis/common/conf/BDPConfiguration.scala | 10 +-
.../apache/linkis/common/conf/Configuration.scala | 2 +-
.../apache/linkis/common/utils/RetryHandler.scala | 10 +-
.../apache/linkis/common/utils/VariableUtils.scala | 20 +-
linkis-commons/linkis-hadoop-common/pom.xml | 24 +
.../linkis/hadoop/common/utils/HDFSUtils.scala | 60 +-
.../linkis/httpclient/AbstractHttpClient.scala | 230 +++--
.../AbstractAuthenticationStrategy.scala | 31 +-
.../linkis/httpclient/config/ClientConfig.scala | 29 +-
.../httpclient/config/HttpClientConstant.scala | 14 +-
.../httpclient/discovery/AbstractDiscovery.scala | 110 ++-
.../exception/HttpClientRetryException.scala | 19 +-
.../loadbalancer/DefaultLoadbalancerStrategy.scala | 3 +
.../linkis/httpclient/request/DownloadAction.scala | 13 +-
.../linkis/server/conf/ServerConfiguration.scala | 2 +
.../apache/linkis/server/security/SSOUtils.scala | 2 +
.../linkis/server/security/SecurityFilter.scala | 4 +-
linkis-commons/linkis-mybatis/pom.xml | 7 +
.../apache/linkis/mybatis/DataSourceConfig.java | 5 +-
.../linkis/protocol/constants/TaskConstant.java | 6 +
.../apache/linkis/rpc/serializer/NoneDelegate.java | 28 -
.../linkis/rpc/serializer/annotation/Chain.java | 13 +-
.../linkis/rpc/serializer/annotation/Method.java | 14 +-
.../linkis/rpc/serializer/annotation/Order.java | 12 +-
.../linkis/rpc/serializer/annotation/Receiver.java | 12 +-
.../exception/MessageErrorException.java | 12 +-
.../serializer/exception/MessageWarnException.java | 11 +-
.../rpc/serializer/method/MessageExecutor.java | 95 ++
.../serializer/method/MethodExecuteWrapper.java | 69 ++
.../serializer/method/ReceiverMethodSearcher.java | 99 +++
.../serializer/parser/DefaultServiceParser.java | 109 +++
.../rpc/serializer/parser/ServiceMethod.java | 113 +++
.../rpc/serializer/parser/ServiceParser.java} | 15 +-
.../registry/AbstractServiceRegistry.java | 98 ++
.../rpc/serializer/registry/ServiceRegistry.java | 14 +-
.../serializer/registry/SpringServiceRegistry.java | 33 +-
.../linkis/rpc/serializer/utils/MessageUtils.java | 94 ++
.../org/apache/linkis/rpc/MessageReceiver.scala | 52 ++
.../apache/linkis/rpc/MessageReceiverChooser.scala | 21 +-
.../org/apache/linkis/rpc/RPCReceiveRestful.scala | 4 +-
.../apache/linkis/rpc/conf/RPCConfiguration.scala | 8 +-
.../linkis/scheduler/queue/LoopArrayQueue.scala | 7 +-
.../linkis/storage/fs/impl/HDFSFileSystem.java | 37 +-
.../apache/linkis/storage/domain/DataType.scala | 31 +-
.../linkis/storage/resultset/ResultSetReader.scala | 7 +-
.../storage/resultset/StorageResultSetReader.scala | 22 +-
.../script/reader/StorageScriptFsReader.scala | 6 +-
.../linkis/storage/source/AbstractFileSource.scala | 2 +
.../apache/linkis/storage/source/FileSource.scala | 2 +
.../apache/linkis/storage/source/FileSplit.scala | 19 +
.../storage/source/ResultsetFileSource.scala | 3 +-
.../linkis/storage/utils/FileSystemUtils.scala | 63 +-
.../storage/utils/StorageConfiguration.scala | 2 +
.../linkis-computation-client/pom.xml | 14 +
.../computation/client/LinkisJobBuilder.scala | 23 +-
.../computation/client/LinkisJobClient.scala | 2 +-
.../client/interactive/InteractiveJobBuilder.scala | 7 +-
.../client/once/LinkisManagerClient.scala | 3 +-
.../once/result/EngineConnOperateResult.scala | 11 +-
.../client/once/simple/SimpleOnceJobBuilder.scala | 15 +-
.../org/apache/linkis/ujes/client/UJESClient.scala | 4 +
.../apache/linkis/ujes/client/UJESClientImpl.scala | 2 +-
.../request/GetTableStatisticInfoAction.scala | 81 ++
.../ujes/client/request/JobExecuteAction.scala | 2 +-
.../response/GetTableStatisticInfoResult.scala} | 36 +-
.../ujes/client/response/JobInfoResult.scala | 11 +
.../governance/common/entity/job/JobRequest.java | 11 +
.../common/entity/job/OnceExecutorContent.scala | 6 +-
.../governance/common/paser/CodeParser.scala | 43 +
.../governance/common/utils/GovernanceUtils.scala | 50 ++
.../launch/LinuxProcessEngineConnLaunch.scala | 18 +
.../apache/linkis/ecm/core/conf/ECMErrorCode.scala | 8 +-
.../ecm/core/launch/EngineConnManagerEnv.scala | 4 +-
.../ecm/core/launch/ProcessEngineConnLaunch.scala | 8 +
.../linkis-engineconn-manager-server/pom.xml | 8 +-
.../linkis/ecm/server/conf/ECMConfiguration.scala | 8 +-
.../linkis/ecm/server/hook/JarUDFLoadECMHook.scala | 69 +-
.../server/operator/EngineConnLogOperator.scala | 10 +-
.../ecm/server/service/ECMHealthService.scala | 17 +-
.../server/service/LocalDirsHandleService.scala | 6 +-
.../impl/AbstractEngineConnLaunchService.scala | 29 +-
.../impl/BmlResourceLocalizationService.scala | 55 +-
.../service/impl/DefaultECMHealthService.scala | 136 ++-
.../service/impl/DefaultEngineConnKillService.java | 44 +-
.../impl/DefaultEngineConnPidCallbackService.scala | 2 +-
.../DefaultEngineConnStatusCallbackService.scala | 2 +-
.../impl/DefaultLocalDirsHandleService.scala | 19 +-
.../service/impl/DefaultOperateService.scala | 2 +-
.../service/impl/DefaultYarnCallbackService.scala | 2 +-
.../impl/LinuxProcessEngineConnLaunchService.scala | 10 +-
.../impl/ProcessEngineConnLaunchService.scala | 44 +-
.../ecm/server/spring/ECMSpringConfiguration.scala | 3 +-
.../apache/linkis/ecm/server/util/ECMUtils.scala | 17 +-
.../linkis/ecm/server/util/HardwareUtils.scala | 49 +-
.../linkis-once-engineconn/pom.xml | 2 +-
.../once/executor/hook/OnceEngineConnHook.scala | 40 +
.../linkis-computation-engineconn/pom.xml | 6 +-
.../async/AsyncConcurrentComputationExecutor.scala | 199 ++++-
.../executor/async/AsyncEngineConnJob.scala | 65 ++
.../executor/async/AsyncExecuteContext.scala | 58 ++
.../executor/async/AsyncExecuteExecutor.scala | 50 ++
...ionExecutor.scala => AsyncExecuteRequest.scala} | 9 +-
.../executor/async/AsyncExecutorManager.scala | 58 ++
.../executor/conf/ComputationExecutorConf.scala | 10 +
.../execute/ComputationEngineConnExecution.scala | 5 +-
.../executor/execute/ComputationExecutor.scala | 27 +-
.../execute/ConcurrentComputationExecutor.scala | 2 +-
.../executor/hook/ComputationExecutorHook.scala | 2 +-
.../executor/hook/ExecutorLabelsRestHook.scala | 58 ++
.../computation/executor/hook/InitSQLHook.scala | 104 +++
.../executor/hook/UDFLoadEngineConnHook.scala | 238 +++--
.../executor/hook/UseDatabaseEngineHook.scala | 79 ++
.../hook/executor/FunctionExecutorHook.scala | 74 ++
.../executor/listener/TaskStatusListener.scala | 4 +-
.../executor/service/TaskExecutionService.scala | 4 +-
.../service/TaskExecutionServiceImpl.scala | 116 ++-
.../executor/utlis/ComputationErrorCode.scala} | 10 +-
.../engineconn/core/executor/ExecutorManager.scala | 63 +-
.../engineconn/core/hook/ExecutorHook.scala} | 45 +-
.../engineconn/launch/EngineConnServer.scala | 41 +-
.../accessible-executor/pom.xml | 10 +-
.../conf/AccessibleExecutorConfiguration.scala | 9 +-
.../execution/AccessibleEngineConnExecution.scala | 66 +-
.../executor/listener/event/TaskEvent.scala | 4 +-
.../executor/service/AccessibleService.scala | 4 +-
.../service/DefaultAccessibleService.scala | 37 +-
.../service/DefaultExecutorHeartbeatService.scala | 2 +-
.../executor/service/DefaultManagerService.scala | 9 +-
.../executor/service/DefaultOperateService.scala | 4 +-
.../service/EngineConnTimedLockService.scala | 4 +-
.../callback-service/pom.xml | 3 +-
.../callback/service/EngineConnCallback.scala | 10 +-
.../executor/entity/ResourceFetchExecutor.scala} | 12 +-
.../executor/service/ManagerService.scala | 4 +-
.../linkis-engineconn-launch/pom.xml | 73 --
.../linkis-engineconn/pom.xml | 1 -
.../linkis-entrance-client/pom.xml | 77 --
.../ClientEngineBuilderBeanAnnotation.java | 50 --
.../ClientEngineRequesterBeanAnnotation.java | 50 --
.../ClientEngineSelectorBeanAnnotation.java | 50 --
.../ClientEntranceParserBeanAnnotation.java | 50 --
.../ClientGroupFactoryBeanAnnotation.java | 50 --
.../ClientInterceptorsBeanAnnotation.java | 50 --
.../DefaultEntranceClientBeanAnnotation.java | 50 --
.../conf/ClientSpringConfiguration.java | 98 --
.../src/main/resources/META-INF/spring.factories | 20 -
.../EngineApplicationNameFactory.scala | 36 -
.../linkis/entranceclient/EntranceClient.scala | 43 -
.../linkis/entranceclient/EntranceClientImpl.scala | 132 ---
.../entranceclient/conf/ClientConfiguration.scala | 33 -
.../ClientForEntranceSpringConfiguration.scala | 82 --
.../conf/MultiEntranceCondition.scala | 35 -
.../context/ClientEntranceParser.scala | 45 -
.../entranceclient/context/ClientLogManager.scala | 44 -
.../context/ClientPersistenceEngine.scala | 57 --
.../context/ClientPersistenceManager.scala | 61 --
.../linkis/entranceclient/execute/ClientJob.scala | 62 --
.../linkis-entrance/pom.xml | 2 +-
.../entrance/conf/EntranceSpringConfiguration.java | 56 +-
.../persistence/QueryPersistenceManager.java | 32 +-
.../entrance/restful/EntranceRestfulApi.java | 10 +-
.../entrance/vo/YarnResourceWithStatusVo.java | 109 +++
.../entrance/conf/EntranceConfiguration.scala | 2 +-
.../manager/label/constant/LabelKeyConstant.java | 2 +
.../manager/label/entity/SerializableLabel.java | 3 -
.../linkis/manager/label/entity/TenantLabel.java | 3 +-
.../manager/label/entity/engine/RunType.scala | 1 +
.../label/entity/engine/UserCreatorLabel.java | 4 +-
.../entity/entrance/RetryWaitTimeOutLabel.java | 46 +
.../manager/label/entity/route/RouteLabel.java | 3 +-
.../linkis-manager/label-manager/pom.xml | 3 +-
.../label/service/ResourceLabelService.scala | 3 +
.../service/impl/DefaultNodeLabelAddService.scala | 2 +-
.../impl/DefaultNodeLabelRemoveService.scala | 6 +-
.../service/impl/DefaultNodeLabelService.scala | 6 +-
.../service/impl/DefaultResourceLabelService.scala | 21 +-
.../linkis-application-manager/pom.xml | 2 +-
.../linkis/manager/am/restful/EMRestfulApi.java | 30 +-
.../manager/am/restful/EngineRestfulApi.java | 27 +-
.../linkis/manager/am/conf/AMConfiguration.scala | 8 +-
.../linkis/manager/am/label/AMLabelChecker.scala | 7 -
.../am/locker/DefaultEngineNodeLocker.scala | 2 +-
.../manager/am/manager/DefaultEMNodeManager.scala | 8 +-
.../am/manager/DefaultEngineNodeManager.scala | 23 +-
.../manager/am/pointer/AbstractNodePointer.scala | 3 +
.../manager/am/pointer/DefaultEMNodPointer.scala | 6 +-
.../manager/am/selector/ECAvailableRule.scala | 50 ++
.../am/selector/rule/AvailableNodeSelectRule.scala | 2 +-
.../linkis/manager/am/service/EngineService.scala | 2 +-
.../am/service/cache/DefaultConfCacheService.scala | 2 +-
.../am/service/em/DefaultEMEngineService.scala | 8 +-
.../am/service/em/DefaultEMInfoService.scala | 10 +-
.../am/service/em/DefaultEMRegisterService.scala | 18 +-
.../am/service/em/DefaultEMUnregisterService.scala | 9 +-
.../manager/am/service/em/EMRegisterService.scala | 4 +-
.../am/service/em/EMUnregisterService.scala | 3 +-
.../engine/DefaultEngineAskEngineService.scala | 18 +-
.../engine/DefaultEngineConnCanKillService.scala | 159 ++++
.../DefaultEngineConnStatusCallbackService.scala | 6 +-
.../engine/DefaultEngineCreateService.scala | 99 ++-
.../service/engine/DefaultEngineInfoService.scala | 2 +-
.../engine/DefaultEngineRecycleService.scala | 6 +-
.../service/engine/DefaultEngineReuseService.scala | 12 +-
.../service/engine/DefaultEngineStopService.scala | 22 +-
.../engine/DefaultEngineSwitchService.scala | 2 +-
.../am/service/engine/EngineAskEngineService.scala | 4 +-
...ervice.scala => EngineConnCanKillService.scala} | 13 +-
.../am/service/engine/EngineCreateService.scala | 4 +-
.../am/service/engine/EngineInfoService.scala | 3 +-
.../am/service/engine/EngineKillService.scala | 3 +-
.../am/service/engine/EngineReuseService.scala | 3 +-
.../am/service/engine/EngineStopService.scala | 9 +-
.../am/service/heartbeat/AMHeartbeatService.scala | 4 +-
.../am/service/monitor/NodeHeartbeatMonitor.scala | 18 +-
.../linkis-manager/linkis-manager-client/pom.xml | 60 --
.../annotation/resource/RMAnnotationParser.java | 82 --
.../client/resource/ResourceManagerClient.scala | 87 --
.../common/entity/enumeration/MaintainType.java} | 14 +-
.../PersistenceResourceActionRecord.java | 155 ++++
.../entity/resource/ResourceActionRecord.java | 133 +++
.../common}/exception/RMErrorException.java | 2 +-
.../common}/exception/RMFatalException.java | 2 +-
.../manager/common}/exception/RMWarnException.java | 2 +-
.../common/protocol/engine/ECCanKillRequest.java | 81 ++
.../common/protocol/engine/ECCanKillResponse.java} | 41 +-
.../engine/EngineReourceClearRequest.java} | 31 +-
.../common/protocol/node/NodeHealthyRequest.java} | 16 +-
.../protocol/node/NodeHeartbeatResourceMsg.java | 107 +++
.../protocol/yarn/YarnAuthorizationRequest.java | 15 +-
.../protocol/yarn/YarnAuthorizationResponse.java} | 36 +-
.../manager/common/conf}/RMConfiguration.scala | 10 +-
.../protocol/resource/ResourceProtocol.scala | 2 +
.../protocol/resource/ResourceWithStatus.scala} | 13 +-
.../resourcemanager/ResourceManagerParam.scala | 0
.../linkis/resourcemanager/ResultResource.scala | 0
.../service/common/pointer/NodePointer.scala | 4 +-
.../linkis-resource-manager-common/pom.xml | 63 --
.../linkis/resourcemanager/RMRPCFormats.scala | 30 -
.../linkis-manager-persistence/pom.xml | 2 +-
.../linkis/manager/dao/ResourceManagerMapper.java | 9 +
.../manager/dao/impl/ResourceManagerMapper.xml | 50 +-
.../persistence/ResourceManagerPersistence.java | 7 +
.../impl/DefaultResourceManagerPersistence.java | 16 +
.../linkis-manager/linkis-resource-manager/pom.xml | 8 +-
.../resourcemanager/conf/ResourceStatus.java | 53 ++
.../resourcemanager/domain/RMLabelContainer.java | 2 +-
.../external/service/ExternalResourceService.java | 2 +-
.../service/impl/ExternalResourceServiceImpl.java | 4 +-
.../restful/vo/UserCreatorEngineType.java} | 38 +-
.../resourcemanager/restful/vo/UserResourceVo.java | 178 ++++
.../linkis/resourcemanager/utils/AlertUtils.java | 77 +-
.../external/yarn/YarnResourceRequester.scala | 4 +-
.../resourcemanager/message/RMMessageService.scala | 45 +-
.../resourcemanager/restful/RMMonitorRest.scala | 204 ++++-
.../service/LabelResourceService.scala | 2 +
.../service/RequestResourceService.scala | 3 +-
.../service/impl/DefaultResourceManager.scala | 163 ++--
.../impl/DriverAndYarnReqResourceService.scala | 3 +-
.../service/impl/LabelResourceServiceImpl.scala | 4 +
.../service/impl/ResourceLogService.scala | 86 +-
.../service/impl/UserResourceService.scala | 100 +++
.../linkis/resourcemanager/utils/RMUtils.scala | 36 +-
.../resourcemanager/utils/UserConfiguration.scala | 4 +-
.../linkis-manager/pom.xml | 1 -
.../engineconn-plugins/spark/pom.xml | 21 +
.../org/apache/linkis/bml/client/BmlClient.java | 8 +
.../linkis/bml/client/impl/HttpBmlClient.scala | 90 +-
.../org/apache/linkis/bml/http/HttpConf.scala | 4 +
.../apache/linkis/bml/request/BmlPOSTAction.scala | 31 +-
.../org/apache/linkis/bml/response/BmlResult.scala | 10 +
.../apache/linkis/bml/protocol/BmlProtocol.scala | 8 +-
.../src/main/assembly/distribution.xml | 26 +-
.../org/apache/linkis/bml/Entity/ResourceTask.java | 37 +
.../apache/linkis/bml/Entity/ResourceVersion.java | 20 +
.../linkis/bml/common/HdfsResourceHelper.java | 6 +-
.../apache/linkis/bml/common/OperationEnum.java | 5 +-
.../apache/linkis/bml/common/ScheduledTask.java | 3 -
.../org/apache/linkis/bml/dao/ResourceDao.java | 8 +
.../java/org/apache/linkis/bml/dao/VersionDao.java | 4 +
.../apache/linkis/bml/restful/BmlRestfulApi.java | 77 ++
.../apache/linkis/bml/service/ResourceService.java | 4 +
.../org/apache/linkis/bml/service/TaskService.java | 7 +
.../bml/service/impl/ResourceServiceImpl.java | 8 +
.../linkis/bml/service/impl/TaskServiceImpl.java | 125 +++
.../linkis-cs-highavailable/pom.xml | 2 +-
.../linkis-cs-search/pom.xml | 2 +-
.../linkis-cs-server/pom.xml | 28 +-
.../datasource/client/DataSourceRemoteClient.scala | 38 +
...moteClient.scala => MetaDataRemoteClient.scala} | 17 +-
.../linkis/datasource/client/RemoteClient.scala | 2 +-
.../client/impl/LinkisDataSourceRemoteClient.scala | 58 ++
.../client/impl/LinkisMetaDataRemoteClient.scala | 37 +
.../client/request/CreateDataSourceAction.scala | 68 ++
.../client/request/DataSourceAction.scala | 14 +-
.../request/DataSourceTestConnectAction.scala | 77 ++
.../client/request/DeleteDataSourceAction.scala | 60 ++
.../client/request/ExpireDataSourceAction.scala | 61 ++
.../request/GetAllDataSourceTypesAction.scala | 50 ++
.../GetConnectParamsByDataSourceIdAction.scala | 73 ++
.../GetConnectParamsByDataSourceNameAction.scala | 83 ++
.../request/GetDataSourceVersionsAction.scala | 61 ++
.../request/GetInfoByDataSourceIdAction.scala | 71 ++
.../request/GetKeyTypeDatasourceAction.scala | 61 ++
.../client/request/MetadataGetColumnsAction.scala | 91 ++
.../request/MetadataGetDatabasesAction.scala | 71 ++
.../request/MetadataGetPartitionsAction.scala | 89 ++
.../request/MetadataGetTablePropsAction.scala | 90 ++
.../client/request/MetadataGetTablesAction.scala | 79 ++
.../request/PublishDataSourceVersionAction.scala | 72 ++
.../client/request/QueryDataSourceAction.scala | 101 +++
.../client/request/QueryDataSourceEnvAction.scala | 87 ++
.../client/request/UpdateDataSourceAction.scala | 81 ++
.../request/UpdateDataSourceParameterAction.scala | 80 ++
.../client/response/CreateDataSourceResult.scala | 12 +-
.../response/DataSourceTestConnectResult.scala | 18 +-
.../client/response/DeleteDataSourceResult.scala | 12 +-
.../client/response/ExpireDataSourceResult.scala | 12 +-
.../response/GetAllDataSourceTypesResult.scala | 37 +-
.../GetConnectParamsByDataSourceIdResult.scala | 17 +-
.../GetConnectParamsByDataSourceNameResult.scala | 17 +-
.../response/GetDataSourceVersionsResult.scala | 35 +-
.../response/GetInfoByDataSourceIdResult.scala | 31 +-
.../response/GetKeyTypeDatasourceResult.scala | 38 +
.../client/response/MetadataGetColumnsResult.scala | 36 +-
.../response/MetadataGetDatabasesResult.scala | 13 +-
.../response/MetadataGetPartitionsResult.scala | 18 +-
.../response/MetadataGetTablePropsResult.scala | 18 +-
.../client/response/MetadataGetTablesResult.scala | 18 +-
.../response/PublishDataSourceVersionResult.scala | 10 +-
.../client/response/QueryDataSourceEnvResult.scala | 36 +-
.../client/response/QueryDataSourceResult.scala | 36 +-
.../response/UpdateDataSourceParameterResult.scala | 12 +-
.../client/response/UpdateDataSourceResult.scala | 12 +-
.../datasource/client/TestDataSourceClient.scala | 78 ++
.../datasourcemanager/common/DsmConfiguration.java | 10 +-
.../common/util/PatternInjectUtils.java | 209 -----
.../server/receiver/BaseMetaReceiver.scala | 67 ++
.../linkis-metadata-manager/service/hive/pom.xml | 2 +-
.../linkis-datasource/linkis-metadata/pom.xml | 8 +-
.../org/apache/linkis/metadata/dao/impl/MdqDao.xml | 2 +-
.../metadata/domain/mdq/DomainCoversionUtils.java | 14 +
.../domain/mdq/bo/MdqTableFieldsInfoBO.java | 12 +
.../linkis/metadata/domain/mdq/po/MdqField.java | 10 +
.../linkis/metadata/ddl/ScalaDDLCreator.scala | 3 +
.../linkis-configuration/pom.xml | 2 +-
.../linkis-instance-label-client/pom.xml | 2 +-
.../linkis-instance-label-server/pom.xml | 2 +-
.../label/service/InsLabelRpcService.scala | 12 +-
.../service/rpc/DefaultInsLabelRpcService.scala | 14 +-
.../linkis-publicservice/linkis-jobhistory/pom.xml | 7 +-
.../linkis/jobhistory/dao/impl/JobDetailMapper.xml | 28 +-
.../jobhistory/dao/impl/JobHistoryMapper.xml | 70 +-
.../apache/linkis/jobhistory/entity/JobDetail.java | 72 +-
.../linkis/jobhistory/entity/JobHistory.java | 133 +--
.../jobhistory/conversions/TaskConversions.scala | 108 +--
.../service/JobHistoryDetailQueryService.java | 5 +
.../impl/JobHistoryDetailQueryServiceImpl.scala | 13 +-
.../service/impl/JobHistoryQueryServiceImpl.scala | 27 +-
.../apache/linkis/jobhistory/util/QueryUtils.scala | 8 +-
.../linkis-ps-common-lock}/pom.xml | 31 +-
.../common/lock/CommonLockSpringConfiguration.java | 38 +
.../common/lock/dao/CommonLockMapper.java | 42 +
.../common/lock/entity/CommonLock.java | 87 ++
.../common/lock/service/CommonLockService.java | 14 +-
.../service/impl/DefaultCommonLockService.java | 79 ++
.../linkis/filesystem/WorkspaceClientImpl.scala | 18 +
.../filesystem/request/WorkspaceClient.scala | 2 +
.../linkis-storage-script-dev-server/pom.xml | 2 +-
.../linkis-udf/linkis-udf-client/pom.xml | 2 +-
.../scala/org/apache/linkis/udf/UDFClient.scala | 72 +-
.../linkis-udf/linkis-udf-common/pom.xml | 8 +-
.../java/org/apache/linkis/udf/entity/UDFInfo.java | 59 +-
.../java/org/apache/linkis/udf/entity/UDFTree.java | 8 +-
.../org/apache/linkis/udf/entity/UDFVersion.java | 133 +++
.../org/apache/linkis/udf/utils/ConstantVar.java | 16 +-
.../java/org/apache/linkis/udf/vo/DownloadVo.java | 31 +-
.../udf/{entity/UDFInfo.java => vo/UDFAddVo.java} | 146 +--
.../udf/{entity/UDFInfo.java => vo/UDFInfoVo.java} | 146 +--
.../{entity/UDFInfo.java => vo/UDFUpdateVo.java} | 112 +--
.../{entity/UDFInfo.java => vo/UDFVersionVo.java} | 144 ++-
.../linkis-udf/linkis-udf-service/pom.xml | 7 +-
.../java/org/apache/linkis/udf/api/UDFApi.java | 424 ++++++---
.../java/org/apache/linkis/udf/dao/UDFDao.java | 48 +-
.../java/org/apache/linkis/udf/dao/UDFTreeDao.java | 14 +-
.../org/apache/linkis/udf/dao/UDFVersionDao.java | 60 ++
.../java/org/apache/linkis/udf/dao/impl/UDFDao.xml | 226 +++--
.../org/apache/linkis/udf/dao/impl/UDFTreeDao.xml | 20 +-
.../apache/linkis/udf/dao/impl/UDFVersionDao.xml | 107 +++
.../org/apache/linkis/udf/service/UDFService.java | 61 +-
.../linkis/udf/service/impl/UDFServiceImpl.java | 989 +++++++++++++++------
.../udf/service/impl/UDFTreeServiceImpl.java | 30 +-
.../linkis-publicservice/linkis-variable/pom.xml | 2 +-
.../linkis-publicservice/pom.xml | 2 +
.../src/main/assembly/distribution.xml | 26 +-
.../linkis-gateway-authentication}/pom.xml | 75 +-
.../gateway/authentication/dao/TokenDao.java | 18 +-
.../authentication/dao/impl/TokenMapper.xml | 53 ++
.../gateway/authentication/entity/TokenEntity.java | 104 +++
.../exception/TokenAuthException.java | 18 +-
.../exception/TokenNotExistException.java | 18 +-
.../linkis/gateway/authentication/bo/Token.scala | 70 ++
.../linkis/gateway/authentication/bo/User.scala | 14 +-
.../gateway/authentication/bo/impl/TokenImpl.scala | 158 ++++
.../gateway/authentication/bo/impl/UserImpl.scala | 12 +-
.../authentication/conf/TokenConfiguration.scala | 24 +-
.../service/CachedTokenService.scala | 157 ++++
.../authentication/service/TokenService.scala | 49 +
.../gateway/authentication/dao/TokenDaoTest.java | 51 ++
.../authentication/dao/utils/MyBatisUtil.java | 53 ++
.../service/CachedTokenServiceTest.java | 110 +++
.../src/test/resources/TokenMapper.xml | 53 ++
.../src/test/resources/conf/mybatis-config.xml} | 48 +-
.../src/test/resources/conf/run_first_for_test.sql | 30 +-
.../linkis-gateway-core/pom.xml | 7 +
.../gateway/config/GatewayConfiguration.scala | 12 +-
.../config/GatewaySpringConfiguration.scala | 7 +
.../linkis/gateway/route/GatewayRouter.scala | 27 +-
.../linkis/gateway/security/GatewaySSOUtils.scala | 37 +-
.../linkis/gateway/security/SecurityFilter.scala | 8 +-
.../linkis/gateway/security/UserRestful.scala | 178 +---
.../security/token/TokenAuthentication.scala | 81 +-
.../linkis/httpclient/dws/DWSHttpClient.scala | 13 +-
.../StaticAuthenticationStrategy.scala | 41 +-
.../dws/config/GatewayHttpClientConf.scala | 11 +-
.../dws/discovery/DefaultConfigDiscovery.scala | 53 ++
.../linkis/httpclient/dws/response/DWSResult.scala | 46 +-
.../linkis-gateway-server-support/pom.xml | 30 +
.../src/main/assembly/distribution.xml | 29 +-
.../ujes/parser/ResultSetCacheGatewayParser.scala | 110 +++
.../ujes/route/AbstractLabelGatewayRouter.scala | 2 +-
.../ujes/route/GatewayRouterConfiguration.scala | 4 +-
.../ujes/route/label/RouteLabelParser.scala | 12 +-
.../linkis-service-gateway/pom.xml | 1 +
pom.xml | 1 +
436 files changed, 12042 insertions(+), 5061 deletions(-)
diff --git a/.gitignore b/.gitignore
index 9e68f71..73a643c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -32,7 +32,6 @@ linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manage
linkis-computation-governance/linkis-engineconn/target/
linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/target/
linkis-computation-governance/linkis-engineconn/linkis-engineconn-common/target/
-linkis-computation-governance/linkis-engineconn/linkis-engineconn-launch/target/
linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/target/
linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/target/
linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/target/
@@ -49,7 +48,6 @@ linkis-computation-governance/linkis-manager/label-manager/target/
linkis-computation-governance/linkis-manager/linkis-application-manager/target/
linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/target/
linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-service-common/target/
-linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-resource-manager-common/target/
linkis-computation-governance/linkis-manager/linkis-manager-monitor/target/
linkis-computation-governance/linkis-manager/linkis-manager-persistence/target/
linkis-computation-governance/linkis-manager/linkis-resource-manager/target/
diff --git a/assembly-combined-package/assembly-combined/sbin/kill-process-by-pid.sh b/assembly-combined-package/assembly-combined/sbin/kill-process-by-pid.sh
new file mode 100644
index 0000000..aff0efe
--- /dev/null
+++ b/assembly-combined-package/assembly-combined/sbin/kill-process-by-pid.sh
@@ -0,0 +1,81 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+# http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+function list_descendants ()
+{
+ ppid=$1
+ local children=$(pgrep -P ${ppid})
+
+ for spid in $children
+ do
+ list_descendants "$spid"
+ done
+
+ sub_pid_list=(${sub_pid_list[@]} ${children[@]})
+}
+function kill_proc_by_pid() {
+ pid=$1
+ sub_pid_list=()
+
+ echo "`date '+%Y-%m-%d %H:%M:%S'` Starting to kill parent process ${pid}"
+ #KILL PARENT FIRST
+ cnt=0
+ MAX_RETRY=6
+ while kill -0 ${pid} > /dev/null 2>&1 && [ $cnt -lt $MAX_RETRY ]
+ do
+ list_descendants ${pid}
+ kill -15 ${pid}
+ let cnt++
+ sleep 5s
+ done
+ if kill -0 ${pid} > /dev/null 2>&1; then
+ list_descendants ${pid}
+ kill -9 ${pid}
+ echo "`date '+%Y-%m-%d %H:%M:%S'` Killed parent process ${pid} with \'-9\' signal. May not be able to kill gracefully(yarn application may still be alive)" 1>&2
+ else
+ echo "`date '+%Y-%m-%d %H:%M:%S'` Finished to kill parent process ${pid}"
+ fi
+
+ #REMOVE DUPLICATE
+ sub_pid_list=($(echo ${sub_pid_list[*]} | sed 's/ /\n/g' | sort | uniq))
+ echo "sub_pid_list:${sub_pid_list[*]}"
+ #THEN KILL CHILDREN
+ for sub_pid in ${sub_pid_list[@]}; do
+ cnt=0
+ MAX_RETRY=6
+ while kill -0 ${sub_pid} > /dev/null 2>&1 && [ $cnt -lt $MAX_RETRY ]
+ do
+ kill -15 ${sub_pid}
+ echo "to Killed sub-process ${sub_pid}"
+ let cnt++
+ sleep 5s
+ if kill -0 ${sub_pid} > /dev/null 2>&1; then
+ let 1+1
+ else
+ echo "`date '+%Y-%m-%d %H:%M:%S'` Killed sub-process ${sub_pid}"
+ fi
+ done
+ if kill -0 ${sub_pid} > /dev/null 2>&1; then
+ kill -9 ${sub_pid}
+ echo "`date '+%Y-%m-%d %H:%M:%S'` Killed sub-process ${sub_pid} with \'-9\' signal. May not be able to kill gracefully(yarn application may still be alive)" 1>&2
+ fi
+ done
+
+ echo "`date '+%Y-%m-%d %H:%M:%S'` Finished killing process and its sub-process ${pid}"
+}
+
+
+kill_proc_by_pid $@
\ No newline at end of file
diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala
index 12be538..917e065 100644
--- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala
+++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/BDPConfiguration.scala
@@ -45,8 +45,8 @@ private[conf] object BDPConfiguration extends Logging {
val propertyFile = sysProps.getOrElse("wds.linkis.configuration", DEFAULT_PROPERTY_FILE_NAME)
val configFileURL = getClass.getClassLoader.getResource(propertyFile)
if (configFileURL != null && new File(configFileURL.getPath).exists) {
+ warn(s"******************************** Notice: The Linkis configuration file is $propertyFile ! ***************************")
initConfig(config, configFileURL.getPath)
- info(s"******************************** Info: The Linkis read $propertyFile file from $configFileURL ！***************************")
}
else warn(s"******************************** Notice: The Linkis configuration file $propertyFile is not exists! ***************************")
@@ -54,10 +54,10 @@ private[conf] object BDPConfiguration extends Logging {
val serverConf = sysProps.getOrElse("wds.linkis.server.conf", DEFAULT_SERVER_CONF_FILE_NAME)
val serverConfFileURL = getClass.getClassLoader.getResource(serverConf)
if (serverConfFileURL != null && new File(serverConfFileURL.getPath).exists) {
+ warn(s"******************************** Notice: The Linkis serverConf file is $propertyFile ! ***************************")
initConfig(config, serverConfFileURL.getPath)
- info(s"******************************** Info: The Linkis read $serverConf file from $serverConfFileURL ！***************************")
}
- else warn(s"******************************** Notice: The Linkis serverConf file $serverConf is not exists! ***************************")
+ else warn(s"******************************** Notice: The Linkis serverConf file $serverConfFileURL is not exists! ***************************")
// load server confs
val propertyFileOptions = sysProps.get("wds.linkis.server.confs")
@@ -66,10 +66,10 @@ private[conf] object BDPConfiguration extends Logging {
propertyFiles.foreach { propertyF =>
val configFileURL = getClass.getClassLoader.getResource(propertyF)
if (configFileURL != null && new File(configFileURL.getPath).exists) {
+ warn(s"******************************** Notice: The Linkis server.confs is file $propertyF ***************************")
initConfig(config, configFileURL.getPath)
- info(s"******************************** Info: The Linkis read $propertyF file from $configFileURL ！***************************")
}
- else warn(s"******************************** Notice: The Linkis configuration file $propertyF is not exists! ***************************")
+ else warn(s"******************************** Notice: The Linkis server.confs file $propertyF is not exists! ***************************")
}
}
diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala
index 1131a64..b649dfe 100644
--- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala
+++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/conf/Configuration.scala
@@ -38,7 +38,7 @@ object Configuration extends Logging {
val REFLECT_SCAN_PACKAGE = CommonVars.apply("wds.linkis.reflect.scan.package", "org.apache.linkis,com.webank.wedatasphere").getValue.split(",")
- val CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME = CommonVars("wds.linkis.console.configuration.application.name", "linkis-ps-publicservice")
+ val CLOUD_CONSOLE_CONFIGURATION_SPRING_APPLICATION_NAME = CommonVars("wds.linkis.console.configuration.application.name", "linkis-ps-configuration")
val CLOUD_CONSOLE_VARIABLE_SPRING_APPLICATION_NAME = CommonVars("wds.linkis.console.variable.application.name", "linkis-ps-publicservice")
diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/RetryHandler.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/RetryHandler.scala
index 9de19b0..3ada866 100644
--- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/RetryHandler.scala
+++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/RetryHandler.scala
@@ -37,7 +37,7 @@ trait RetryHandler extends Logging {
def setRetryMaxPeriod(retryMaxPeriod: Long): Unit = this.maxPeriod = retryMaxPeriod
def getRetryMaxPeriod: Long = maxPeriod
def addRetryException(t: Class[_ <: Throwable]): Unit = retryExceptions += t
- def getRetryExceptions: Array[Class[_ <: Throwable]] = retryExceptions.toArray
+ def getRetryExceptions: Array[Class[_ <: Throwable]] = retryExceptions.toArray
protected def exceptionCanRetry(t: Throwable): Boolean = !t.isInstanceOf[FatalException] &&
retryExceptions.exists(c => CommonClassUtils.isAssignable(t.getClass, c))
@@ -49,13 +49,13 @@ trait RetryHandler extends Logging {
}
def retry[T](op: => T, retryName: String): T = {
- if(retryExceptions.isEmpty || retryNum <= 1) return op
+ if (retryExceptions.isEmpty || retryNum <= 1) return op
var retry = 0
var result = null.asInstanceOf[T]
while(retry < retryNum && result == null) result = Utils.tryCatch(op) { t =>
retry += 1
if(retry >= retryNum) throw t
- else if(exceptionCanRetry(t)) {
+ else if (exceptionCanRetry(t)) {
val retryInterval = nextInterval(retry)
info(retryName + s" failed with ${t.getClass.getName}, wait ${ByteTimeUtils.msDurationToString(retryInterval)} for next retry. Retried $retry++ ...")
Utils.tryQuietly(Thread.sleep(retryInterval))
@@ -65,3 +65,7 @@ trait RetryHandler extends Logging {
result
}
}
+
+class DefaultRetryHandler extends RetryHandler {
+}
+
diff --git a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala
index 8b259ee..1ce3586 100644
--- a/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala
+++ b/linkis-commons/linkis-common/src/main/scala/org/apache/linkis/common/utils/VariableUtils.scala
@@ -36,8 +36,17 @@ object VariableUtils extends Logging {
/**
* date Format
*/
- val dateFormat = new SimpleDateFormat("yyyyMMdd")
- val dateFormat_std = new SimpleDateFormat("yyyy-MM-dd")
+ //val dateFormat = new SimpleDateFormat("yyyyMMdd")
+ //val dateFormat_std = new SimpleDateFormat("yyyy-MM-dd")
+
+ val dateFormatLocal = new ThreadLocal[SimpleDateFormat]() {
+ override protected def initialValue = new SimpleDateFormat("yyyyMMdd")
+ }
+
+ val dateFormatStdLocal = new ThreadLocal[SimpleDateFormat]() {
+ override protected def initialValue = new SimpleDateFormat("yyyy-MM-dd")
+ }
+
val codeReg: Regex = "\\$\\{\\s*[A-Za-z][A-Za-z0-9_\\.]*\\s*[\\+\\-\\*/]?\\s*[A-Za-z0-9_\\.]*\\s*\\}".r
val calReg: Regex = "(\\s*[A-Za-z][A-Za-z0-9_\\.]*\\s*)([\\+\\-\\*/]?)(\\s*[A-Za-z0-9_\\.]*\\s*)".r
@@ -124,6 +133,8 @@ object VariableUtils extends Logging {
* @return
*/
private def getYesterday(std: Boolean = true): String = {
+ val dateFormat = dateFormatLocal.get()
+ val dateFormat_std = dateFormatStdLocal.get()
val cal: Calendar = Calendar.getInstance()
cal.add(Calendar.DATE, -1)
if (std) {
@@ -141,6 +152,8 @@ object VariableUtils extends Logging {
* @return
*/
private[utils] def getMonth(date: Date, std: Boolean = true, isEnd: Boolean = false): String = {
+ val dateFormat = dateFormatLocal.get()
+ val dateFormat_std = dateFormatStdLocal.get()
val cal = Calendar.getInstance()
cal.setTime(date)
cal.set(Calendar.DATE, 1)
@@ -232,7 +245,8 @@ case class StringType(value: String) extends VariableType {
import VariableUtils._
class CustomDateType(date: String, std: Boolean = true) {
-
+ protected val dateFormat = dateFormatLocal.get()
+ protected val dateFormat_std = dateFormatStdLocal.get()
def -(days: Int): String = {
if (std) {
dateFormat_std.format(DateUtils.addDays(dateFormat_std.parse(date), -days))
diff --git a/linkis-commons/linkis-hadoop-common/pom.xml b/linkis-commons/linkis-hadoop-common/pom.xml
index b56ce78..eba9677 100644
--- a/linkis-commons/linkis-hadoop-common/pom.xml
+++ b/linkis-commons/linkis-hadoop-common/pom.xml
@@ -129,8 +129,20 @@
<artifactId>*</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+
</exclusions>
</dependency>
+
+ <dependency>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ <version>${protobuf.version}</version>
+ </dependency>
+
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
@@ -184,6 +196,18 @@
<groupId>org.eclipse.jetty</groupId>
<artifactId>*</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git a/linkis-commons/linkis-hadoop-common/src/main/scala/org/apache/linkis/hadoop/common/utils/HDFSUtils.scala b/linkis-commons/linkis-hadoop-common/src/main/scala/org/apache/linkis/hadoop/common/utils/HDFSUtils.scala
index d73e739..0fa2f2f 100644
--- a/linkis-commons/linkis-hadoop-common/src/main/scala/org/apache/linkis/hadoop/common/utils/HDFSUtils.scala
+++ b/linkis-commons/linkis-hadoop-common/src/main/scala/org/apache/linkis/hadoop/common/utils/HDFSUtils.scala
@@ -99,40 +99,50 @@ object HDFSUtils extends Logging {
def getHDFSUserFileSystem(userName: String, conf: org.apache.hadoop.conf.Configuration): FileSystem = if (HadoopConf.HDFS_ENABLE_CACHE) {
val locker = userName + LOCKER_SUFFIX
- locker.intern().synchronized {
- val hdfsFileSystemContainer = if (fileSystemCache.containsKey(userName)) {
- fileSystemCache.get(userName)
- } else {
- val newHDFSFileSystemContainer = new HDFSFileSystemContainer(createFileSystem(userName, conf), userName)
- fileSystemCache.put(userName, newHDFSFileSystemContainer)
- newHDFSFileSystemContainer
- }
- hdfsFileSystemContainer.addAccessCount()
- hdfsFileSystemContainer.updateLastAccessTime
- hdfsFileSystemContainer.getFileSystem
- }
+ locker.intern().synchronized {
+ val hdfsFileSystemContainer = if (fileSystemCache.containsKey(userName)) {
+ fileSystemCache.get(userName)
} else {
- createFileSystem(userName, conf)
+ val newHDFSFileSystemContainer = new HDFSFileSystemContainer(createFileSystem(userName, conf), userName)
+ fileSystemCache.put(userName, newHDFSFileSystemContainer)
+ newHDFSFileSystemContainer
}
+ hdfsFileSystemContainer.addAccessCount()
+ hdfsFileSystemContainer.updateLastAccessTime
+ hdfsFileSystemContainer.getFileSystem
+ }
+ } else {
+ createFileSystem(userName, conf)
+ }
- def createFileSystem(userName: String, conf: org.apache.hadoop.conf.Configuration): FileSystem =
- getUserGroupInformation(userName)
+ def createFileSystem(userName: String, conf: org.apache.hadoop.conf.Configuration): FileSystem =
+ getUserGroupInformation(userName)
.doAs(new PrivilegedExceptionAction[FileSystem] {
def run = FileSystem.get(conf)
})
- def closeHDFSFIleSystem(fileSystem: FileSystem, userName: String): Unit = if (null != fileSystem && StringUtils.isNotBlank(userName)) {
- if (HadoopConf.HDFS_ENABLE_CACHE) {
- val hdfsFileSystemContainer = fileSystemCache.get(userName)
- if (null != hdfsFileSystemContainer) {
- val locker = userName + LOCKER_SUFFIX
- locker synchronized hdfsFileSystemContainer.minusAccessCount()
- }
- } else {
- fileSystem.close()
- }
+ def closeHDFSFIleSystem(fileSystem: FileSystem, userName: String): Unit = if (null != fileSystem && StringUtils.isNotBlank(userName)) {
+ closeHDFSFIleSystem(fileSystem, userName, false)
+ }
+
+ def closeHDFSFIleSystem(fileSystem: FileSystem, userName: String, isForce: Boolean): Unit = if (null != fileSystem && StringUtils.isNotBlank(userName)) {
+ if (HadoopConf.HDFS_ENABLE_CACHE) {
+ val hdfsFileSystemContainer = fileSystemCache.get(userName)
+ if (null != hdfsFileSystemContainer) {
+ val locker = userName + LOCKER_SUFFIX
+ if (isForce) {
+ locker synchronized fileSystemCache.remove(hdfsFileSystemContainer.getUser)
+ IOUtils.closeQuietly(hdfsFileSystemContainer.getFileSystem)
+ info(s"user${hdfsFileSystemContainer.getUser} to Force remove hdfsFileSystemContainer")
+ } else {
+ locker synchronized hdfsFileSystemContainer.minusAccessCount()
}
+ }
+ } else {
+ IOUtils.closeQuietly(fileSystem)
+ }
+ }
def getUserGroupInformation(userName: String): UserGroupInformation = {
if (KERBEROS_ENABLE.getValue) {
diff --git a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/AbstractHttpClient.scala b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/AbstractHttpClient.scala
index a40fe4c..622053c 100644
--- a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/AbstractHttpClient.scala
+++ b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/AbstractHttpClient.scala
@@ -14,44 +14,51 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
package org.apache.linkis.httpclient
+import java.net.URI
import java.util
-import org.apache.linkis.common.conf.Configuration
+import org.apache.linkis.common.conf.{CommonVars, Configuration}
import org.apache.linkis.common.io.{Fs, FsPath}
import org.apache.linkis.httpclient.authentication.{AbstractAuthenticationStrategy, AuthenticationAction, HttpAuthentication}
-import org.apache.linkis.httpclient.config.ClientConfig
+import org.apache.linkis.httpclient.config.{ClientConfig, HttpClientConstant}
import org.apache.linkis.httpclient.discovery.{AbstractDiscovery, Discovery, HeartbeatAction}
-import org.apache.linkis.httpclient.exception.{HttpClientResultException, HttpMessageParseException, HttpMethodNotSupportException}
+import org.apache.linkis.httpclient.exception.{HttpClientResultException, HttpClientRetryException, HttpMessageParseException, HttpMethodNotSupportException}
import org.apache.linkis.httpclient.loadbalancer.{AbstractLoadBalancer, DefaultLoadbalancerStrategy, LoadBalancer}
import org.apache.linkis.httpclient.request._
import org.apache.linkis.httpclient.response._
import org.apache.commons.io.IOUtils
import org.apache.commons.lang.StringUtils
-import org.apache.http.client.CookieStore
+import org.apache.http.client.{CookieStore, ResponseHandler}
import org.apache.http.client.config.RequestConfig
-import org.apache.http.client.entity.{DeflateDecompressingEntity, GzipDecompressingEntity, UrlEncodedFormEntity}
+import org.apache.http.client.entity.{DeflateDecompressingEntity, EntityBuilder, GzipDecompressingEntity, UrlEncodedFormEntity}
import org.apache.http.client.methods._
import org.apache.http.client.utils.URIBuilder
+import org.apache.http.conn.{ConnectTimeoutException, ConnectionPoolTimeoutException, HttpHostConnectException}
import org.apache.http.entity.mime.MultipartEntityBuilder
import org.apache.http.entity.{ContentType, StringEntity}
import org.apache.http.impl.client.{BasicCookieStore, CloseableHttpClient, HttpClients}
import org.apache.http.message.BasicNameValuePair
import org.apache.http.util.EntityUtils
import org.apache.http.{HttpResponse, _}
+import org.apache.linkis.common.utils.Logging
import scala.collection.JavaConversions._
+abstract class AbstractHttpClient(clientConfig: ClientConfig, clientName: String) extends Client with Logging {
-abstract class AbstractHttpClient(clientConfig: ClientConfig, clientName: String) extends Client {
-
- protected val CONNECT_TIME_OUT = 50000
+ protected val CONNECT_TIME_OUT = CommonVars("wds.linkis.httpclient.default.connect.timeOut", 50000).getValue
protected val cookieStore = new BasicCookieStore
- protected val httpClient: CloseableHttpClient = HttpClients.custom().setDefaultCookieStore(cookieStore).build
+ protected val httpClient: CloseableHttpClient = HttpClients
+ .custom()
+ .setDefaultCookieStore(cookieStore)
+ .setMaxConnTotal(clientConfig.getMaxConnection)
+ .setMaxConnPerRoute(clientConfig.getMaxConnection / 2)
+ .build
if (clientConfig.getAuthenticationStrategy != null) clientConfig.getAuthenticationStrategy match {
case auth: AbstractAuthenticationStrategy => auth.setClient(this)
@@ -67,10 +74,9 @@ abstract class AbstractHttpClient(clientConfig: ClientConfig, clientName: String
d.setSchedule(clientConfig.getDiscoveryPeriod, clientConfig.getDiscoveryTimeUnit)
case d => d.setServerUrl(clientConfig.getServerUrl)
}
- //如果discovery没有启用，那么启用loadBalancer是没有意义的
- val loadBalancer = if (clientConfig.isLoadbalancerEnabled && this.clientConfig.getLoadbalancerStrategy != null)
+ val loadBalancer = if (clientConfig.isLoadbalancerEnabled && this.clientConfig.getLoadbalancerStrategy != null) {
Some(this.clientConfig.getLoadbalancerStrategy.createLoadBalancer())
- else if (clientConfig.isLoadbalancerEnabled) Some(DefaultLoadbalancerStrategy.createLoadBalancer())
+ } else if (clientConfig.isLoadbalancerEnabled) Some(DefaultLoadbalancerStrategy.createLoadBalancer())
else None
loadBalancer match {
case Some(lb: AbstractLoadBalancer) =>
@@ -87,26 +93,36 @@ abstract class AbstractHttpClient(clientConfig: ClientConfig, clientName: String
override def execute(requestAction: Action): Result = execute(requestAction, -1)
override def execute(requestAction: Action, waitTime: Long): Result = {
- if (!requestAction.isInstanceOf[HttpAction])
+ if (!requestAction.isInstanceOf[HttpAction]) {
throw new UnsupportedOperationException("only HttpAction supported, but the fact is " + requestAction.getClass)
+ }
val action = prepareAction(requestAction.asInstanceOf[HttpAction])
val startTime = System.currentTimeMillis
- val req = prepareReq(action)
val prepareReqTime = System.currentTimeMillis - startTime
prepareCookie(action)
val attempts = new util.ArrayList[Long]()
+
def addAttempt(): CloseableHttpResponse = {
+ val req = prepareReq(action)
val startTime = System.currentTimeMillis
val response = executeRequest(req, Some(waitTime).filter(_ > 0))
- attempts.add(System.currentTimeMillis - startTime)
+ if (response.getStatusLine.getStatusCode == 401) {
+ tryLogin(action, getRequestUrl(action), true)
+ logger.info("The user is not logged in, client retry Login")
+ throw new HttpClientRetryException("The user is not logged in,Client had reTry Login, you can set a retry")
+ }
+ val taken = System.currentTimeMillis - startTime
+ attempts.add(taken)
+ logger.info(s"invoke ${req.getURI} taken: ${taken}")
response
}
+
val response = if (!clientConfig.isRetryEnabled) addAttempt()
else clientConfig.getRetryHandler.retry(addAttempt(), action.getClass.getSimpleName + "HttpRequest")
val beforeDeserializeTime = System.currentTimeMillis
responseToResult(response, action) match {
case metricResult: MetricResult =>
- if(metricResult.getMetric == null) metricResult.setMetric(new HttpMetric)
+ if (metricResult.getMetric == null) metricResult.setMetric(new HttpMetric)
metricResult.getMetric.setPrepareReqTime(prepareReqTime)
metricResult.getMetric.addRetries(attempts)
metricResult.getMetric.setDeserializeTime(System.currentTimeMillis - beforeDeserializeTime)
@@ -122,54 +138,78 @@ abstract class AbstractHttpClient(clientConfig: ClientConfig, clientName: String
}
protected def getRequestUrl(suffixUrl: String, requestBody: String): String = {
- val urlPrefix = loadBalancer.map(_.chooseServerUrl(requestBody)).getOrElse(clientConfig.getServerUrl)
- if(suffixUrl.contains(urlPrefix)) suffixUrl else connectUrl(urlPrefix, suffixUrl)
+ val chooseUrlPrefix = loadBalancer.map(_.chooseServerUrl(requestBody)).orNull
+ val urlPrefix = if (null == chooseUrlPrefix) clientConfig.getDefaultServerUrl else chooseUrlPrefix
+ if (suffixUrl.contains(urlPrefix)) suffixUrl else connectUrl(urlPrefix, suffixUrl)
}
protected def connectUrl(prefix: String, suffix: String): String = {
- val prefixEnd = prefix.endsWith("/")
- val suffixStart = suffix.startsWith("/")
+ val prefixEnd = prefix.endsWith(HttpClientConstant.PATH_SPLIT_TOKEN)
+ val suffixStart = suffix.startsWith(HttpClientConstant.PATH_SPLIT_TOKEN)
if (prefixEnd && suffixStart) prefix.substring(0, prefix.length - 1) + suffix
- else if (!prefixEnd && !suffixStart) prefix + "/" + suffix
+ else if (!prefixEnd && !suffixStart) prefix + HttpClientConstant.PATH_SPLIT_TOKEN + suffix
else prefix + suffix
}
+ protected def getServerUrl(uri: URI): String = {
+ s"${uri.getScheme}://${uri.getHost}:${uri.getPort}"
+ }
+
protected def prepareAction(requestAction: HttpAction): HttpAction = requestAction
- protected def prepareCookie(requestAction:HttpAction): Unit = if(requestAction.getCookies.nonEmpty) {
+ protected def prepareCookie(requestAction: HttpAction): Unit = if (requestAction.getCookies.nonEmpty) {
requestAction.getCookies.foreach(cookieStore.addCookie)
}
- protected def prepareReq(requestAction: HttpAction): HttpRequestBase = {
- var realURL = ""
+ private def tryLogin(requestAction: HttpAction, realURL: String, isForce: Boolean): Unit = {
+ if (clientConfig.getAuthenticationStrategy != null) {
+ val authentication = clientConfig.getAuthenticationStrategy match {
+ case abstractAuthenticationStrategy: AbstractAuthenticationStrategy if (isForce) =>
+ abstractAuthenticationStrategy.enforceLogin(requestAction, realURL.replaceAll(requestAction.getURL, ""))
+ case _ =>
+ clientConfig.getAuthenticationStrategy.login(requestAction, realURL.replaceAll(requestAction.getURL, ""))
+ }
+ authentication match {
+ case authAction: HttpAuthentication =>
+ val cookies = authAction.authToCookies
+ if (cookies != null && cookies.nonEmpty) cookies.foreach(requestAction.addCookie)
+ val headers = authAction.authToHeaders
+ if (headers != null && !headers.isEmpty) {
+ headers.foreach { case (k, v) => if (k != null && v != null) requestAction.addHeader(k.toString, v.toString) }
+ }
+ case _ =>
+ }
+ }
+ }
+
+ private def getRequestUrl(requestAction: HttpAction): String = {
requestAction match {
case serverUrlAction: ServerUrlAction =>
- realURL = connectUrl(serverUrlAction.serverUrl, requestAction.getURL)
+ connectUrl(serverUrlAction.serverUrl, requestAction.getURL)
case _ =>
- realURL = getRequestUrl(requestAction.getURL, requestAction.getRequestBody)
+ getRequestUrl(requestAction.getURL, requestAction.getRequestBody)
}
+ }
- if (clientConfig.getAuthenticationStrategy != null) clientConfig.getAuthenticationStrategy.login(requestAction, realURL.replaceAll(requestAction.getURL, "")) match {
- case authAction: HttpAuthentication =>
- val cookies = authAction.authToCookies
- if (cookies != null && cookies.nonEmpty) cookies.foreach(requestAction.addCookie)
- val headers = authAction.authToHeaders
- if (headers != null && !headers.isEmpty) {
- headers.foreach { case (k, v) => if(k != null && v != null) requestAction.addHeader(k.toString, v.toString) }
- }
- case _ =>
- }
+ protected def prepareReq(requestAction: HttpAction): HttpRequestBase = {
+
+ val realURL = getRequestUrl(requestAction)
+ tryLogin(requestAction, realURL, false)
val request = requestAction match {
case delete: DeleteAction =>
val builder = new URIBuilder(realURL)
if (!delete.getParameters.isEmpty) {
- delete.getParameters.foreach { case (k, v) => if(k != null && v != null) builder.addParameter(k.toString, v.toString) }
+ delete.getParameters.foreach {
+ case (k, v) => if (k != null && v != null) builder.addParameter(k.toString, v.toString)
+ }
}
val httpDelete = new HttpDelete(builder.build())
if (requestAction.getHeaders.nonEmpty) {
- requestAction.getHeaders.foreach { case (k, v) => if(k != null && v != null) httpDelete.addHeader(k.toString, v.toString) }
+ requestAction.getHeaders.foreach {
+ case (k, v) => if (k != null && v != null) httpDelete.addHeader(k.toString, v.toString)
+ }
}
httpDelete
case put: PutAction =>
@@ -177,10 +217,14 @@ abstract class AbstractHttpClient(clientConfig: ClientConfig, clientName: String
if (put.getParameters.nonEmpty || put.getFormParams.nonEmpty) {
val nameValuePairs = new util.ArrayList[NameValuePair]
if (put.getParameters.nonEmpty) {
- put.getParameters.foreach { case (k, v) => if(v != null) nameValuePairs.add(new BasicNameValuePair(k, v.toString)) }
+ put.getParameters.foreach {
+ case (k, v) => if (v != null) nameValuePairs.add(new BasicNameValuePair(k, v.toString))
+ }
}
if (put.getFormParams.nonEmpty) {
- put.getFormParams.foreach { case (k, v) => if(v != null) nameValuePairs.add(new BasicNameValuePair(k, v.toString)) }
+ put.getFormParams.foreach {
+ case (k, v) => if (v != null) nameValuePairs.add(new BasicNameValuePair(k, v.toString))
+ }
}
httpPut.setEntity(new UrlEncodedFormEntity(nameValuePairs))
}
@@ -193,27 +237,34 @@ abstract class AbstractHttpClient(clientConfig: ClientConfig, clientName: String
}
if (requestAction.getHeaders.nonEmpty) {
- requestAction.getHeaders.foreach { case (k, v) => if(k != null && v != null) httpPut.addHeader(k.toString, v.toString) }
+ requestAction.getHeaders.foreach {
+ case (k, v) => if (k != null && v != null) httpPut.addHeader(k.toString, v.toString)
+ }
}
httpPut
case upload: UploadAction =>
val httpPost = new HttpPost(realURL)
val builder = MultipartEntityBuilder.create()
if (upload.inputStreams != null)
- upload.inputStreams.foreach { case (k, v) =>
- builder.addBinaryBody(k, v, ContentType.create("multipart/form-data"), k)
+ upload.inputStreams.foreach {
+ case (k, v) =>
+ builder.addBinaryBody(k, v, ContentType.create("multipart/form-data"), k)
}
upload.binaryBodies.foreach(binaryBody => builder.addBinaryBody(binaryBody.parameterName, binaryBody.inputStream, binaryBody.contentType, binaryBody.fileName))
upload match {
case get: GetAction => get.getParameters.
retain((k, v) => v != null && k != null).
- foreach { case (k, v) => if(k != null && v != null) builder.addTextBody(k.toString, v.toString) }
+ foreach {
+ case (k, v) => if (k != null && v != null) builder.addTextBody(k.toString, v.toString)
+ }
case _ =>
}
upload match {
case get: GetAction => get.getHeaders.
retain((k, v) => v != null && k != null).
- foreach { case (k, v) => if(k != null && v != null) httpPost.addHeader(k.toString, v.toString) }
+ foreach {
+ case (k, v) => if (k != null && v != null) httpPost.addHeader(k.toString, v.toString)
+ }
case _ =>
}
val httpEntity = builder.build()
@@ -224,15 +275,21 @@ abstract class AbstractHttpClient(clientConfig: ClientConfig, clientName: String
if (post.getParameters.nonEmpty || post.getFormParams.nonEmpty) {
val nvps = new util.ArrayList[NameValuePair]
if (post.getParameters.nonEmpty) {
- post.getParameters.foreach { case (k, v) => if(v != null) nvps.add(new BasicNameValuePair(k, v.toString)) }
- }
- if (post.getFormParams.nonEmpty) {
- post.getFormParams.foreach { case (k, v) => if(v != null) nvps.add(new BasicNameValuePair(k, v.toString)) }
+ post.getParameters.foreach {
+ case (k, v) => if (v != null) nvps.add(new BasicNameValuePair(k, v.toString))
+ }
+ httpPost.setEntity(new UrlEncodedFormEntity(nvps))
+ } else if (post.getFormParams.nonEmpty) {
+ post.getFormParams.foreach {
+ case (k, v) => if (v != null) nvps.add(new BasicNameValuePair(k, v.toString))
+ }
+ val entity: HttpEntity = EntityBuilder.create(). /*setContentEncoding("UTF-8").*/
+ setContentType(ContentType.create("application/x-www-form-urlencoded", Consts.UTF_8))
+ .setParameters(nvps).build();
+ httpPost.setEntity(entity)
}
- httpPost.setEntity(new UrlEncodedFormEntity(nvps))
- }
- if (StringUtils.isNotBlank(post.getRequestPayload)) {
+ } else if (StringUtils.isNotBlank(post.getRequestPayload)) {
val stringEntity = new StringEntity(post.getRequestPayload, "UTF-8")
stringEntity.setContentEncoding(Configuration.BDP_ENCODING.getValue)
stringEntity.setContentType("application/json")
@@ -240,17 +297,23 @@ abstract class AbstractHttpClient(clientConfig: ClientConfig, clientName: String
}
if (requestAction.getHeaders.nonEmpty) {
- requestAction.getHeaders.foreach { case (k, v) => if(k != null && v != null) httpPost.addHeader(k.toString, v.toString) }
+ requestAction.getHeaders.foreach {
+ case (k, v) => if (k != null && v != null) httpPost.addHeader(k.toString, v.toString)
+ }
}
httpPost
case get: GetAction =>
val builder = new URIBuilder(realURL)
if (!get.getParameters.isEmpty) {
- get.getParameters.foreach { case (k, v) => if(k != null && v != null) builder.addParameter(k.toString, v.toString) }
+ get.getParameters.foreach {
+ case (k, v) => if (k != null && v != null) builder.addParameter(k.toString, v.toString)
+ }
}
val httpGet = new HttpGet(builder.build())
if (requestAction.getHeaders.nonEmpty) {
- requestAction.getHeaders.foreach { case (k, v) => if(k != null && v != null) httpGet.addHeader(k.toString, v.toString) }
+ requestAction.getHeaders.foreach {
+ case (k, v) => if (k != null && v != null) httpGet.addHeader(k.toString, v.toString)
+ }
}
httpGet
case _ =>
@@ -260,7 +323,9 @@ abstract class AbstractHttpClient(clientConfig: ClientConfig, clientName: String
stringEntity.setContentType("application/json")
httpost.setEntity(stringEntity)
if (requestAction.getHeaders.nonEmpty) {
- requestAction.getHeaders.foreach { case (k, v) => if(k != null && v != null) httpost.addHeader(k.toString, v.toString) }
+ requestAction.getHeaders.foreach {
+ case (k, v) => if (k != null && v != null) httpost.addHeader(k.toString, v.toString)
+ }
}
httpost
}
@@ -269,7 +334,6 @@ abstract class AbstractHttpClient(clientConfig: ClientConfig, clientName: String
protected def getFsByUser(user: String, path: FsPath): Fs
-
protected def executeRequest(req: HttpRequestBase, waitTime: Option[Long]): CloseableHttpResponse = {
val readTimeOut = waitTime.getOrElse(clientConfig.getReadTimeout)
val connectTimeOut = if (clientConfig.getConnectTimeout > 1000 || clientConfig.getConnectTimeout < 0) clientConfig.getConnectTimeout else CONNECT_TIME_OUT
@@ -278,10 +342,38 @@ abstract class AbstractHttpClient(clientConfig: ClientConfig, clientName: String
.setConnectionRequestTimeout(connectTimeOut.toInt)
.setSocketTimeout(readTimeOut.toInt).build
req.setConfig(requestConfig)
- val response = httpClient.execute(req)
+ val response = try {
+ httpClient.execute(req)
+ } catch {
+ case connectionPoolTimeOutException: ConnectionPoolTimeoutException =>
+ val serverUrl = getServerUrl(req.getURI)
+ addUnHealThyUrlToDiscovery(serverUrl)
+ logger.warn("will be server url add unhealthy for connectionPoolTimeOutException")
+ throw new HttpClientRetryException("connectionPoolTimeOutException", connectionPoolTimeOutException)
+ case connectionTimeOutException: ConnectTimeoutException =>
+ val serverUrl = getServerUrl(req.getURI)
+ addUnHealThyUrlToDiscovery(serverUrl)
+ logger.warn("will be server url add unhealthy for connectionTimeOutException")
+ throw new HttpClientRetryException("connectionTimeOutException", connectionTimeOutException)
+ case httpHostConnectException: HttpHostConnectException =>
+ val serverUrl = getServerUrl(req.getURI)
+ addUnHealThyUrlToDiscovery(serverUrl)
+ logger.warn("will be server url add unhealthy for httpHostConnectException")
+ throw new HttpClientRetryException("httpHostConnectException", httpHostConnectException)
+ case t: Throwable =>
+ throw t
+ }
response
}
+ private def addUnHealThyUrlToDiscovery(serverUrl: String): Unit = {
+ discovery.foreach {
+ case d: AbstractDiscovery =>
+ d.addUnhealthyServerInstances(serverUrl)
+ case _ =>
+ }
+ }
+
protected def executeRequest(req: HttpRequestBase, waitTime: Option[Long], cookieStore: CookieStore): CloseableHttpResponse = {
val readTimeOut = waitTime.getOrElse(clientConfig.getReadTimeout)
val connectTimeOut = if (clientConfig.getConnectTimeout > 1000 || clientConfig.getConnectTimeout < 0) clientConfig.getConnectTimeout else CONNECT_TIME_OUT
@@ -306,13 +398,12 @@ abstract class AbstractHttpClient(clientConfig: ClientConfig, clientName: String
}
throw new HttpClientResultException(s"request failed! ResponseBody is $responseBody.")
}
- val inputStream = if(entity.getContentEncoding != null && StringUtils.isNotBlank(entity.getContentEncoding.getValue))
- entity.getContentEncoding.getValue.toLowerCase match {
- case "gzip" => new GzipDecompressingEntity(entity).getContent
- case "deflate" => new DeflateDecompressingEntity(entity).getContent
- case str => throw new HttpClientResultException(s"request failed! Reason: not support decompress type $str.")
- } else entity.getContent
- download.write(inputStream)
+ val inputStream = if (entity.getContentEncoding != null && StringUtils.isNotBlank(entity.getContentEncoding.getValue)) entity.getContentEncoding.getValue.toLowerCase match {
+ case "gzip" => new GzipDecompressingEntity(entity).getContent
+ case "deflate" => new DeflateDecompressingEntity(entity).getContent
+ case str => throw new HttpClientResultException(s"request failed! Reason: not support decompress type $str.")
+ } else entity.getContent
+ download.write(inputStream, response)
Result()
case heartbeat: HeartbeatAction =>
discovery.map {
@@ -331,6 +422,13 @@ abstract class AbstractHttpClient(clientConfig: ClientConfig, clientName: String
httpResponseToResult(response, httpAction, responseBody)
.getOrElse(throw new HttpMessageParseException("cannot parse message: " + responseBody))
}
+ if (!requestAction.isInstanceOf[DownloadAction]) {
+ response match {
+ case r: CloseableHttpResponse =>
+ r.close()
+ case _ =>
+ }
+ }
result match {
case userAction: UserAction => requestAction match {
case _userAction: UserAction => userAction.setUser(_userAction.getUser)
diff --git a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/authentication/AbstractAuthenticationStrategy.scala b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/authentication/AbstractAuthenticationStrategy.scala
index 53e8aa9..b7e9082 100644
--- a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/authentication/AbstractAuthenticationStrategy.scala
+++ b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/authentication/AbstractAuthenticationStrategy.scala
@@ -18,21 +18,22 @@
package org.apache.linkis.httpclient.authentication
import java.util.concurrent.ConcurrentHashMap
-
import org.apache.linkis.httpclient.Client
import org.apache.linkis.httpclient.config.ClientConfig
import org.apache.linkis.httpclient.request.{Action, UserAction}
import org.apache.commons.lang.StringUtils
import org.apache.http.HttpResponse
+import org.apache.linkis.common.utils.Logging
-abstract class AbstractAuthenticationStrategy extends AuthenticationStrategy {
+abstract class AbstractAuthenticationStrategy extends AuthenticationStrategy with Logging{
private var client: Client = _
private val userNameToAuthentications = new ConcurrentHashMap[String, Authentication]()
private var clientConfig: ClientConfig = _
protected val sessionMaxAliveTime: Long
def setClient(client: Client): Unit = this.client = client
+
def getClient: Client = client
protected def getKeyByUserAndURL(user: String, serverUrl: String): String = user + "@" + serverUrl
@@ -46,25 +47,27 @@ abstract class AbstractAuthenticationStrategy extends AuthenticationStrategy {
protected def getKey(requestAction: Action, serverUrl: String): String = {
val user = getUser(requestAction)
- if(user == null) return null
+ if (user == null) return null
getKeyByUserAndURL(user, serverUrl)
}
def setClientConfig(clientConfig: ClientConfig): Unit = this.clientConfig = clientConfig
+
def getClientConfig: ClientConfig = clientConfig
def login(requestAction: Action, serverUrl: String): Authentication = {
val key = getKey(requestAction, serverUrl)
- if(key == null) return null
- if(userNameToAuthentications.containsKey(key) && !isTimeout(userNameToAuthentications.get(key))) {
+ if (key == null) return null
+ if (userNameToAuthentications.containsKey(key) && !isTimeout(userNameToAuthentications.get(key))) {
val authenticationAction = userNameToAuthentications.get(key)
authenticationAction.updateLastAccessTime()
authenticationAction
} else key.intern() synchronized {
var authentication = userNameToAuthentications.get(key)
- if(authentication == null || isTimeout(authentication)) {
+ if (authentication == null || isTimeout(authentication)) {
authentication = tryLogin(requestAction, serverUrl)
- userNameToAuthentications.put(key, authentication)
+ putSession(key, authentication)
+ logger.info(s"$key try reLogin")
}
authentication
}
@@ -77,9 +80,23 @@ abstract class AbstractAuthenticationStrategy extends AuthenticationStrategy {
}
}
+ /**
+ * The static account password method, when the return code is determined to be 401, a mandatory login is triggered
+ * @param requestAction
+ * @param serverUrl
+ * @return
+ */
+ def enforceLogin(requestAction: Action, serverUrl: String): Authentication = {
+ login(requestAction, serverUrl)
+ }
+
protected def getAuthenticationAction(requestAction: Action, serverUrl: String): AuthenticationAction
def getAuthenticationResult(response: HttpResponse, requestAction: AuthenticationAction): AuthenticationResult
def isTimeout(authentication: Authentication): Boolean = System.currentTimeMillis() - authentication.getLastAccessTime >= sessionMaxAliveTime
+
+ def putSession(key: String, authentication: Authentication): Unit = {
+ this.userNameToAuthentications.put(key, authentication)
+ }
}
\ No newline at end of file
diff --git a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfig.scala b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfig.scala
index 5f9ea81..f117d1b 100644
--- a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfig.scala
+++ b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/ClientConfig.scala
@@ -14,9 +14,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
package org.apache.linkis.httpclient.config
+import org.apache.commons.lang.StringUtils
import org.apache.linkis.common.utils.RetryHandler
import org.apache.linkis.httpclient.authentication.{AbstractAuthenticationStrategy, AuthenticationStrategy}
import org.apache.linkis.httpclient.loadbalancer.LoadBalancerStrategy
@@ -25,6 +26,7 @@ import scala.concurrent.duration.TimeUnit
class ClientConfig private() {
+
private var serverUrl: String = _
private var discoveryEnabled: Boolean = false
private var discoveryPeriod: Long = _
@@ -36,7 +38,7 @@ class ClientConfig private() {
private var authTokenValue: String = _
private var connectTimeout: Long = _
private var readTimeout: Long = _
- private var maxConnection: Int = _
+ private var maxConnection: Int = 20
private var retryEnabled: Boolean = _
private var retryHandler: RetryHandler = _
@@ -46,7 +48,9 @@ class ClientConfig private() {
authTokenKey: String, authTokenValue: String) = {
this()
this.serverUrl = serverUrl
+
this.discoveryEnabled = discoveryEnabled
+
this.discoveryPeriod = discoveryPeriod
this.discoveryTimeUnit = discoveryTimeUnit
this.loadbalancerEnabled = loadbalancerEnabled
@@ -66,18 +70,39 @@ class ClientConfig private() {
}
def getServerUrl = serverUrl
+
+ def getDefaultServerUrl: String = {
+ if (StringUtils.isNotBlank(serverUrl) && serverUrl.contains(HttpClientConstant.URL_SPLIT_TOKEN)) {
+ serverUrl.split(HttpClientConstant.URL_SPLIT_TOKEN)(0)
+ } else {
+ serverUrl
+ }
+ }
+
def isDiscoveryEnabled = discoveryEnabled
+
def getDiscoveryPeriod = discoveryPeriod
+
def getDiscoveryTimeUnit = discoveryTimeUnit
+
def isLoadbalancerEnabled = loadbalancerEnabled
+
def getLoadbalancerStrategy = loadbalancerStrategy
+
def getAuthenticationStrategy = authenticationStrategy
+
def getAuthTokenKey: String = authTokenKey
+
def getAuthTokenValue: String = authTokenValue
+
def getConnectTimeout = connectTimeout
+
def getReadTimeout = readTimeout
+
def getMaxConnection = maxConnection
+
def isRetryEnabled = retryEnabled
+
def getRetryHandler = retryHandler
}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/exception/ClientErrorException.scala b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/HttpClientConstant.scala
similarity index 78%
rename from linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/exception/ClientErrorException.scala
rename to linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/HttpClientConstant.scala
index a73d6c9..3181f81 100644
--- a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/exception/ClientErrorException.scala
+++ b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/config/HttpClientConstant.scala
@@ -5,18 +5,20 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.linkis.entranceclient.exception
-import org.apache.linkis.common.exception.ErrorException
+package org.apache.linkis.httpclient.config
-class ClientErrorException(errorCode: Int, desc: String) extends ErrorException(errorCode, desc)
\ No newline at end of file
+object HttpClientConstant {
+ val URL_SPLIT_TOKEN = ","
+
+ val PATH_SPLIT_TOKEN = "/"
+}
diff --git a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/discovery/AbstractDiscovery.scala b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/discovery/AbstractDiscovery.scala
index cc74114..19a3215 100644
--- a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/discovery/AbstractDiscovery.scala
+++ b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/discovery/AbstractDiscovery.scala
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
package org.apache.linkis.httpclient.discovery
import java.io.Closeable
@@ -22,10 +22,12 @@ import java.net.ConnectException
import java.util
import java.util.concurrent.ScheduledFuture
+import org.apache.commons.lang.StringUtils
import org.apache.linkis.common.utils.{Logging, Utils}
import org.apache.linkis.httpclient.Client
import org.apache.linkis.httpclient.exception.DiscoveryException
import org.apache.http.HttpResponse
+import org.apache.linkis.httpclient.config.HttpClientConstant
import scala.collection.JavaConversions._
import scala.concurrent.duration.TimeUnit
@@ -45,9 +47,11 @@ abstract class AbstractDiscovery extends Discovery with Closeable with Logging {
private var heartbeatFuture: ScheduledFuture[_] = _
private var unhealthyHeartbeatFuture: ScheduledFuture[_] = _
+
override def setServerUrl(serverUrl: String): Unit = {
this.serverUrl = serverUrl
}
+
def getServerUrl: String = serverUrl
def setSchedule(period: Long, timeUnit: TimeUnit): Unit = {
@@ -56,59 +60,67 @@ abstract class AbstractDiscovery extends Discovery with Closeable with Logging {
}
def setClient(client: Client): Unit = this.client = client
+
def getClient: Client = client
override def start(): Unit = {
- client.execute(getHeartbeatAction(serverUrl), 3000) match {
- case heartbeat: HeartbeatResult => if(!heartbeat.isHealthy)
- throw new DiscoveryException(s"connect to serverUrl $serverUrl failed! Reason: gateway server is unhealthy!")
- else discoveryListeners.foreach(_.onServerDiscovered(serverUrl))
- }
- val delayTime = timeUnit.convert(timeUnit.toMillis(period)/5, timeUnit)
- discoveryFuture = Utils.defaultScheduler.scheduleAtFixedRate(new Runnable {
- override def run(): Unit = Utils.tryAndWarn {
- val serverUrls = discovery()
- serverInstances synchronized serverUrls.foreach { serverUrl =>
- if(!serverInstances.contains(serverUrl)) {
- serverInstances.add(serverUrl)
- discoveryListeners.foreach(_.onServerDiscovered(serverUrl))
- }
- }
- }
- }, delayTime, period, timeUnit)
- heartbeatFuture = Utils.defaultScheduler.scheduleAtFixedRate(new Runnable {
+ val delayTime = timeUnit.convert(timeUnit.toMillis(period) / 5, timeUnit)
+ discoveryFuture = startDiscovery()
+ heartbeatFuture = startHealthyCheck(delayTime)
+ unhealthyHeartbeatFuture = startUnHealthyCheck(delayTime)
+ }
+
+ def startHealthyCheck(delayTime: Long): ScheduledFuture[_] = {
+ logger.info("start HealthyCheck thread")
+ Utils.defaultScheduler.scheduleAtFixedRate(new Runnable {
override def run(): Unit = {
serverInstances.toList.foreach { serverUrl =>
val action = getHeartbeatAction(serverUrl)
- info("heartbeat to healthy gateway " + serverUrl)
+ logger.info("heartbeat to healthy gateway " + serverUrl)
Utils.tryCatch(client.execute(action, 3000) match {
case heartbeat: HeartbeatResult =>
- if(!heartbeat.isHealthy) {
- unhealthyServerInstances synchronized unhealthyServerInstances.add(serverUrl)
- discoveryListeners.foreach(_.onServerUnhealthy(serverUrl))
- serverInstances synchronized serverInstances.remove(serverUrl)
+ if (!heartbeat.isHealthy) {
+ addUnhealthyServerInstances(serverUrl)
}
}) {
case _: ConnectException =>
- unhealthyServerInstances synchronized unhealthyServerInstances.remove(serverUrl)
- serverInstances synchronized serverInstances.remove(serverUrl)
- discoveryListeners.foreach(_.onServerUnconnected(serverUrl))
+ addUnhealthyServerInstances(serverUrl)
}
}
}
}, delayTime, period, timeUnit)
- unhealthyHeartbeatFuture = Utils.defaultScheduler.scheduleAtFixedRate(new Runnable {
+ }
+
+ def startDiscovery(): ScheduledFuture[_] = {
+ logger.info("start Discovery thread")
+ client.execute(getHeartbeatAction(serverUrl), 3000) match {
+ case heartbeat: HeartbeatResult => if (!heartbeat.isHealthy) throw new DiscoveryException(s"connect to serverUrl $serverUrl failed! Reason: gateway server is unhealthy!")
+ else discoveryListeners.foreach(_.onServerDiscovered(serverUrl))
+ }
+
+ Utils.defaultScheduler.scheduleAtFixedRate(new Runnable {
+ override def run(): Unit = Utils.tryAndWarn {
+ logger.info("to discovery gateway" + serverUrl)
+ val serverUrls = discovery()
+ addServerInstances(serverUrls)
+ }
+ }, 1, period + 30, timeUnit)
+ }
+
+ def startUnHealthyCheck(delayTime: Long): ScheduledFuture[_] = {
+ logger.info("start UnHealthyCheck thread")
+ Utils.defaultScheduler.scheduleAtFixedRate(new Runnable {
override def run(): Unit = {
unhealthyServerInstances.toList.foreach { serverUrl =>
val action = getHeartbeatAction(serverUrl)
- info("heartbeat to unhealthy gateway " + serverUrl)
+ logger.info("heartbeat to unhealthy gateway " + serverUrl)
Utils.tryCatch(client.execute(action, 3000) match {
case heartbeat: HeartbeatResult =>
- if(heartbeat.isHealthy) {
+ if (heartbeat.isHealthy) {
unhealthyServerInstances synchronized unhealthyServerInstances.remove(serverUrl)
discoveryListeners.foreach(_.onServerHealthy(serverUrl))
serverInstances synchronized serverInstances.add(serverUrl)
- } else if(serverInstances.contains(serverUrl)) serverInstances synchronized serverInstances.remove(serverUrl)
+ } else if (serverInstances.contains(serverUrl)) serverInstances synchronized serverInstances.remove(serverUrl)
}) {
case _: ConnectException =>
unhealthyServerInstances synchronized unhealthyServerInstances.remove(serverUrl)
@@ -132,8 +144,38 @@ abstract class AbstractDiscovery extends Discovery with Closeable with Logging {
override def getServerInstances: Array[String] = serverInstances.toArray(new Array[String](serverInstances.size()))
override def close(): Unit = {
- discoveryFuture.cancel(true)
- heartbeatFuture.cancel(true)
- unhealthyHeartbeatFuture.cancel(true)
+ if (null != discoveryFuture) discoveryFuture.cancel(true)
+ if (null != heartbeatFuture) heartbeatFuture.cancel(true)
+ if (null != unhealthyHeartbeatFuture) unhealthyHeartbeatFuture.cancel(true)
}
+
+ def addUnhealthyServerInstances(unhealthyUrl: String): Unit = {
+ logger.info(s"add ${unhealthyUrl} to unhealthy list ")
+ val updateUnhealthyUrl = if (serverInstances.contains(unhealthyUrl)) {
+ unhealthyUrl
+ } else if (serverInstances.contains(unhealthyUrl + HttpClientConstant.PATH_SPLIT_TOKEN)) {
+ unhealthyUrl + HttpClientConstant.PATH_SPLIT_TOKEN
+ } else {
+ ""
+ }
+ if (StringUtils.isBlank(updateUnhealthyUrl)) {
+ logger.info(s"${unhealthyUrl} unhealthy url not exists")
+ } else {
+ unhealthyServerInstances synchronized unhealthyServerInstances.add(updateUnhealthyUrl)
+ discoveryListeners.foreach(_.onServerUnhealthy(updateUnhealthyUrl))
+ serverInstances synchronized serverInstances.remove(updateUnhealthyUrl)
+ }
+ }
+
+ def addServerInstances(serverUrls: Array[String]): Unit = {
+ serverInstances synchronized serverUrls.foreach { url =>
+ if (!serverInstances.contains(url)) {
+ val parsedUrl = if (url.endsWith(HttpClientConstant.PATH_SPLIT_TOKEN)) url.substring(0, url.length - 1)
+ else url
+ serverInstances.add(parsedUrl)
+ discoveryListeners.foreach(_.onServerDiscovered(parsedUrl))
+ }
+ }
+ }
+
}
diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineReuseService.scala b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/exception/HttpClientRetryException.scala
similarity index 72%
copy from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineReuseService.scala
copy to linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/exception/HttpClientRetryException.scala
index 903a41f..dc023f0 100644
--- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/EngineReuseService.scala
+++ b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/exception/HttpClientRetryException.scala
@@ -5,26 +5,25 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.linkis.manager.am.service.engine
-import org.apache.linkis.common.exception.LinkisRetryException
-import org.apache.linkis.manager.common.entity.node.EngineNode
-import org.apache.linkis.manager.common.protocol.engine.EngineReuseRequest
+package org.apache.linkis.httpclient.exception
+import org.apache.linkis.common.exception.LinkisRetryException
-trait EngineReuseService {
+class HttpClientRetryException(errorDesc: String) extends LinkisRetryException(10900, errorDesc) {
- @throws[LinkisRetryException]
- def reuseEngine(engineReuseRequest: EngineReuseRequest): EngineNode
+ def this(errorDesc: String, throwable: Throwable) = {
+ this(errorDesc)
+ initCause(throwable)
+ }
}
diff --git a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/loadbalancer/DefaultLoadbalancerStrategy.scala b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/loadbalancer/DefaultLoadbalancerStrategy.scala
index ff48947..d2a0ab3 100644
--- a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/loadbalancer/DefaultLoadbalancerStrategy.scala
+++ b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/loadbalancer/DefaultLoadbalancerStrategy.scala
@@ -25,6 +25,9 @@ object DefaultLoadbalancerStrategy extends LoadBalancerStrategy {
override def chooseServerUrl(requestBody: String): String = {
val serverUrls = getAllServerUrls.filterNot(getAllUnhealthyServerUrls.contains)
+ if (serverUrls.isEmpty) {
+ return null
+ }
Random.shuffle(serverUrls.toList).head
}
diff --git a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/request/DownloadAction.scala b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/request/DownloadAction.scala
index 5824db9..5826bcd 100644
--- a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/request/DownloadAction.scala
+++ b/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/request/DownloadAction.scala
@@ -17,11 +17,22 @@
package org.apache.linkis.httpclient.request
+import org.apache.http.HttpResponse
+import org.apache.linkis.common.utils.Logging
+
import scala.tools.nsc.interpreter.InputStream
-trait DownloadAction {
+trait DownloadAction extends Logging {
+
+ def getResponse: HttpResponse
+
+ def setResponse(response: HttpResponse)
def write(inputStream: InputStream): Unit
+ def write(inputStream: InputStream, response: HttpResponse): Unit = {
+ write(inputStream)
+ setResponse(response)
+ }
}
diff --git a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala
index 6e077a2..921b0d1 100644
--- a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala
+++ b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/conf/ServerConfiguration.scala
@@ -100,4 +100,6 @@ object ServerConfiguration extends Logging{
val BDP_SERVER_RESTFUL_REGISTER_CLASSES = CommonVars("wds.linkis.server.restful.register.classes", "")
// val BDP_SERVER_SOCKET_SERVICE_SCAN_PACKAGES = CommonVars("wds.linkis.server.socket.service.scan.packages", BDP_SERVER_RESTFUL_SCAN_PACKAGES.getValue)
val IS_GATEWAY = CommonVars("wds.linkis.is.gateway", "false")
+ val BDP_SERVER_WEB_ALLOW_ORIGIN = CommonVars("wds.linkis.server.web.alloworigin", "*")
+ val BDP_SERVER_WEB_ALLOW_METHOD = CommonVars("wds.linkis.server.web.allowmethod", "POST,GET,OPTIONS,PUT,HEAD,DELETE")
}
diff --git a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SSOUtils.scala b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SSOUtils.scala
index fbd32bd..c7933ca 100644
--- a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SSOUtils.scala
+++ b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SSOUtils.scala
@@ -151,4 +151,6 @@ object SSOUtils extends Logging {
}
}
+ def getSessionTimeOut(): Long = sessionTimeout
+
}
diff --git a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala
index 64a6043..d9cce5d 100644
--- a/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala
+++ b/linkis-commons/linkis-module/src/main/scala/org/apache/linkis/server/security/SecurityFilter.scala
@@ -106,10 +106,10 @@ class SecurityFilter extends Filter {
}
protected def addAccessHeaders(response: HttpServletResponse) {
- response.setHeader("Access-Control-Allow-Origin", "*")
+ response.setHeader("Access-Control-Allow-Origin", ServerConfiguration.BDP_SERVER_WEB_ALLOW_ORIGIN.getValue)
response.setHeader("Access-Control-Allow-Credentials", "true")
response.setHeader("Access-Control-Allow-Headers", "authorization,Content-Type")
- response.setHeader("Access-Control-Allow-Methods", "POST, GET, OPTIONS, PUT, HEAD, DELETE")
+ response.setHeader("Access-Control-Allow-Methods", ServerConfiguration.BDP_SERVER_WEB_ALLOW_METHOD.getValue)
val fullDateFormatEN = DateFormat.getDateTimeInstance(DateFormat.FULL, DateFormat.FULL, new Locale("EN", "en"))
response.setHeader("Date", fullDateFormatEN.format(new Date))
}
diff --git a/linkis-commons/linkis-mybatis/pom.xml b/linkis-commons/linkis-mybatis/pom.xml
index c89898c..9225593 100644
--- a/linkis-commons/linkis-mybatis/pom.xml
+++ b/linkis-commons/linkis-mybatis/pom.xml
@@ -69,6 +69,13 @@
<artifactId>pagehelper</artifactId>
<version>5.1.4</version>
</dependency>
+
+ <dependency>
+ <groupId>com.alibaba</groupId>
+ <artifactId>druid</artifactId>
+ <version>${druid.version}</version>
+ </dependency>
+
</dependencies>
<build>
<plugins>
diff --git a/linkis-commons/linkis-mybatis/src/main/java/org/apache/linkis/mybatis/DataSourceConfig.java b/linkis-commons/linkis-mybatis/src/main/java/org/apache/linkis/mybatis/DataSourceConfig.java
index 9144b35..69bfb6e 100644
--- a/linkis-commons/linkis-mybatis/src/main/java/org/apache/linkis/mybatis/DataSourceConfig.java
+++ b/linkis-commons/linkis-mybatis/src/main/java/org/apache/linkis/mybatis/DataSourceConfig.java
@@ -20,8 +20,6 @@ package org.apache.linkis.mybatis;
import org.apache.linkis.common.utils.JavaLog;
import org.apache.linkis.mybatis.conf.MybatisConfiguration;
-import org.apache.commons.dbcp.BasicDataSource;
-
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
@@ -29,6 +27,7 @@ import org.springframework.context.annotation.Configuration;
import javax.sql.DataSource;
+import com.alibaba.druid.pool.DruidDataSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -69,7 +68,7 @@ public class DataSourceConfig extends JavaLog {
MybatisConfiguration.MYBATIS_DATASOURCE_REMOVE_ABANDONED_ENABLED.getValue();
int removeAbandonedTimeout =
MybatisConfiguration.MYBATIS_DATASOURCE_REMOVE_ABANDONED_TIMEOUT.getValue();
- BasicDataSource datasource = new BasicDataSource();
+ DruidDataSource datasource = new DruidDataSource();
info("Database connection address information(数据库连接地址信息)=" + dbUrl);
datasource.setUrl(dbUrl);
datasource.setUsername(username);
diff --git a/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/constants/TaskConstant.java b/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/constants/TaskConstant.java
index 88d3e07..f5cca82 100644
--- a/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/constants/TaskConstant.java
+++ b/linkis-commons/linkis-protocol/src/main/java/org/apache/linkis/protocol/constants/TaskConstant.java
@@ -54,6 +54,12 @@ public interface TaskConstant {
String ENTRANCEJOB_SCHEDULE_TIME = "scheduleTime";
String ENTRANCEJOB_TO_ORCHESTRATOR = "timeToOrchestrator";
String ENTRANCEJOB_COMPLETE_TIME = "completeTime";
+ String ENTRANCEJOB_YARN_METRICS = "yarnMetrics";
+ String ENTRANCEJOB_YARNRESOURCE = "yarnResource";
+ String ENTRANCEJOB_CORE_PERCENT = "corePercent";
+ String ENTRANCEJOB_MEMORY_PERCENT = "memoryPercent";
+ String ENTRANCEJOB_CORE_RGB = "coreRGB";
+ String ENTRANCEJOB_MEMORY_RGB = "memoryRGB";
String PARAMS_DATA_SOURCE = "dataSources";
diff --git a/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/NoneDelegate.java b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/NoneDelegate.java
index 2394bae..2813cc8 100644
--- a/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/NoneDelegate.java
+++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/NoneDelegate.java
@@ -50,32 +50,4 @@ public class NoneDelegate implements Delegate<Option> {
public Class<?> typeClass() {
return Option.class;
}
-
- /*@Override
- public WireFormat.FieldType getFieldType() {
- return WireFormat.FieldType.UINT32;
- }
-
- @Override
- public None$ readFrom(Input input) throws IOException {
- if(0 != input.readUInt32())
- throw new ProtostuffException("Corrupt input.");
-
- return None$.MODULE$;
- }
-
- @Override
- public void writeTo(Output output, int number, None$ none$, boolean repeated) throws IOException {
- output.writeFixed64(number, 0, repeated);
- }
-
- @Override
- public void transfer(Pipe pipe, Input input, Output output, int number, boolean repeated) throws IOException {
- output.writeUInt32(number, input.readUInt32(), repeated);
- }
-
- @Override
- public Class<?> typeClass() {
- return None$.class;
- }*/
}
diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-client/src/main/java/org/apache/linkis/manager/client/annotation/resource/RegisterResource.java b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/annotation/Chain.java
similarity index 89%
copy from linkis-computation-governance/linkis-manager/linkis-manager-client/src/main/java/org/apache/linkis/manager/client/annotation/resource/RegisterResource.java
copy to linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/annotation/Chain.java
index 8948f80..43d537f 100644
--- a/linkis-computation-governance/linkis-manager/linkis-manager-client/src/main/java/org/apache/linkis/manager/client/annotation/resource/RegisterResource.java
+++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/annotation/Chain.java
@@ -5,25 +5,26 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.linkis.manager.client.annotation.resource;
+
+package org.apache.linkis.rpc.serializer.annotation;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
-
@Target({ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
-public @interface RegisterResource {
+public @interface Chain {
+
+ String value() default "default";
}
diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-client/src/main/java/org/apache/linkis/manager/client/annotation/resource/EnableResourceManager.java b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/annotation/Method.java
similarity index 89%
rename from linkis-computation-governance/linkis-manager/linkis-manager-client/src/main/java/org/apache/linkis/manager/client/annotation/resource/EnableResourceManager.java
rename to linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/annotation/Method.java
index 817d6da..c1083ba 100644
--- a/linkis-computation-governance/linkis-manager/linkis-manager-client/src/main/java/org/apache/linkis/manager/client/annotation/resource/EnableResourceManager.java
+++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/annotation/Method.java
@@ -5,25 +5,25 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.linkis.manager.client.annotation.resource;
+
+package org.apache.linkis.rpc.serializer.annotation;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
-
@Target({ElementType.TYPE})
@Retention(RetentionPolicy.RUNTIME)
-public @interface EnableResourceManager {
-}
\ No newline at end of file
+public @interface Method {
+ String value() default "";
+}
diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-client/src/main/java/org/apache/linkis/manager/client/annotation/resource/RegisterResource.java b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/annotation/Order.java
similarity index 89%
copy from linkis-computation-governance/linkis-manager/linkis-manager-client/src/main/java/org/apache/linkis/manager/client/annotation/resource/RegisterResource.java
copy to linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/annotation/Order.java
index 8948f80..8155760 100644
--- a/linkis-computation-governance/linkis-manager/linkis-manager-client/src/main/java/org/apache/linkis/manager/client/annotation/resource/RegisterResource.java
+++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/annotation/Order.java
@@ -5,25 +5,25 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.linkis.manager.client.annotation.resource;
+
+package org.apache.linkis.rpc.serializer.annotation;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
-
@Target({ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
-public @interface RegisterResource {
+public @interface Order {
+ int value() default 2147483647;
}
diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-client/src/main/java/org/apache/linkis/manager/client/annotation/resource/RegisterResource.java b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/annotation/Receiver.java
similarity index 90%
rename from linkis-computation-governance/linkis-manager/linkis-manager-client/src/main/java/org/apache/linkis/manager/client/annotation/resource/RegisterResource.java
rename to linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/annotation/Receiver.java
index 8948f80..9440501 100644
--- a/linkis-computation-governance/linkis-manager/linkis-manager-client/src/main/java/org/apache/linkis/manager/client/annotation/resource/RegisterResource.java
+++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/annotation/Receiver.java
@@ -5,25 +5,23 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.linkis.manager.client.annotation.resource;
+
+package org.apache.linkis.rpc.serializer.annotation;
import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
-
@Target({ElementType.METHOD})
@Retention(RetentionPolicy.RUNTIME)
-public @interface RegisterResource {
-}
+public @interface Receiver {}
diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-resource-manager-common/src/main/java/org/apache/linkis/resourcemanager/exception/RMErrorException.java b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/exception/MessageErrorException.java
similarity index 71%
copy from linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-resource-manager-common/src/main/java/org/apache/linkis/resourcemanager/exception/RMErrorException.java
copy to linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/exception/MessageErrorException.java
index e7faa9d..3335e65 100644
--- a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-resource-manager-common/src/main/java/org/apache/linkis/resourcemanager/exception/RMErrorException.java
+++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/exception/MessageErrorException.java
@@ -15,22 +15,18 @@
* limitations under the License.
*/
-package org.apache.linkis.resourcemanager.exception;
+package org.apache.linkis.rpc.serializer.exception;
import org.apache.linkis.common.exception.ErrorException;
-public class RMErrorException extends ErrorException {
+public class MessageErrorException extends ErrorException {
- public RMErrorException(int errCode, String desc) {
+ public MessageErrorException(int errCode, String desc) {
super(errCode, desc);
}
- public RMErrorException(int errCode, String desc, Throwable t) {
+ public MessageErrorException(int errCode, String desc, Throwable t) {
super(errCode, desc);
initCause(t);
}
-
- public RMErrorException(int errCode, String desc, String ip, int port, String serviceKind) {
- super(errCode, desc, ip, port, serviceKind);
- }
}
diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-resource-manager-common/src/main/java/org/apache/linkis/resourcemanager/exception/RMWarnException.java b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/exception/MessageWarnException.java
similarity index 74%
copy from linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-resource-manager-common/src/main/java/org/apache/linkis/resourcemanager/exception/RMWarnException.java
copy to linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/exception/MessageWarnException.java
index 5c7aaa1..6b4f387 100644
--- a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-resource-manager-common/src/main/java/org/apache/linkis/resourcemanager/exception/RMWarnException.java
+++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/exception/MessageWarnException.java
@@ -15,17 +15,18 @@
* limitations under the License.
*/
-package org.apache.linkis.resourcemanager.exception;
+package org.apache.linkis.rpc.serializer.exception;
import org.apache.linkis.common.exception.WarnException;
-public class RMWarnException extends WarnException {
+public class MessageWarnException extends WarnException {
- public RMWarnException(int errCode, String desc) {
+ public MessageWarnException(int errCode, String desc) {
super(errCode, desc);
}
- public RMWarnException(int errCode, String desc, String ip, int port, String serviceKind) {
- super(errCode, desc, ip, port, serviceKind);
+ public MessageWarnException(int errCode, String desc, Throwable t) {
+ super(errCode, desc);
+ initCause(t);
}
}
diff --git a/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/method/MessageExecutor.java b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/method/MessageExecutor.java
new file mode 100644
index 0000000..fb858e6
--- /dev/null
+++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/method/MessageExecutor.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.rpc.serializer.method;
+
+import org.apache.linkis.common.utils.JavaLog;
+import org.apache.linkis.protocol.message.RequestProtocol;
+import org.apache.linkis.rpc.Sender;
+import org.apache.linkis.rpc.serializer.exception.MessageErrorException;
+import org.apache.linkis.rpc.serializer.exception.MessageWarnException;
+import org.apache.linkis.rpc.serializer.utils.MessageUtils;
+
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class MessageExecutor extends JavaLog {
+
+ private List<MethodExecuteWrapper> getMinOrderMethodWrapper(
+ Map<String, List<MethodExecuteWrapper>> methodWrappers) {
+ // get min key order
+ List<MethodExecuteWrapper> minOrderMethodWrapper = new ArrayList<>();
+ methodWrappers.forEach(
+ (k, v) ->
+ v.forEach(
+ m -> {
+ if (MessageUtils.orderIsMin(m, v)) {
+ minOrderMethodWrapper.add(m);
+ }
+ }));
+ return minOrderMethodWrapper;
+ }
+
+ public Object execute(
+ RequestProtocol requestProtocol,
+ Map<String, List<MethodExecuteWrapper>> methodWrappers,
+ Sender sender)
+ throws InterruptedException, MessageErrorException {
+ Integer count = methodWrappers.values().stream().map(List::size).reduce(0, Integer::sum);
+ if (count == 1) {
+ return executeOneMethod(requestProtocol, methodWrappers, sender);
+ } else {
+ throw new MessageErrorException(
+ 120001,
+ String.format(
+ "find %s method for the rpc request:%s",
+ count, requestProtocol.getClass().getName()));
+ }
+ }
+
+ private Object executeOneMethod(
+ RequestProtocol requestProtocol,
+ Map<String, List<MethodExecuteWrapper>> methodWrappers,
+ Sender sender) {
+ List<MethodExecuteWrapper> methodExecuteWrappers = getMinOrderMethodWrapper(methodWrappers);
+ Object result = null;
+ if (methodExecuteWrappers.size() == 1) {
+ MethodExecuteWrapper methodWrapper = methodExecuteWrappers.get(0);
+ try {
+ if (!methodWrapper.shouldSkip) {
+ Method method = methodWrapper.getMethod();
+ Object service = methodWrapper.getService();
+ if (methodWrapper.isHasSender()) {
+ if (methodWrapper.isSenderOnLeft()) {
+ result = method.invoke(service, sender, requestProtocol);
+ } else {
+ result = method.invoke(service, requestProtocol, sender);
+ }
+ } else {
+ result = method.invoke(service, requestProtocol);
+ }
+ }
+ } catch (Throwable t) {
+ logger().error(String.format("method %s call failed", methodWrapper.getAlias()), t);
+ throw new MessageWarnException(10000, "method call failed", t);
+ }
+ }
+ return result;
+ }
+}
diff --git a/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/method/MethodExecuteWrapper.java b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/method/MethodExecuteWrapper.java
new file mode 100644
index 0000000..075d4f7
--- /dev/null
+++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/method/MethodExecuteWrapper.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.rpc.serializer.method;
+
+import org.apache.linkis.rpc.serializer.parser.ServiceMethod;
+
+import java.lang.reflect.Method;
+
+public class MethodExecuteWrapper {
+
+ public MethodExecuteWrapper(ServiceMethod serviceMethod) {
+ this.serviceMethod = serviceMethod;
+ }
+
+ private final ServiceMethod serviceMethod;
+
+ public boolean shouldSkip;
+
+ public boolean isShouldSkip() {
+ return shouldSkip;
+ }
+
+ public void setShouldSkip(boolean shouldSkip) {
+ this.shouldSkip = shouldSkip;
+ }
+
+ public Method getMethod() {
+ return this.serviceMethod.getMethod();
+ }
+
+ public Object getService() {
+ return this.serviceMethod.getService();
+ }
+
+ public String getAlias() {
+ return this.serviceMethod.getAlias();
+ }
+
+ public int getOrder() {
+ return this.serviceMethod.getOrder();
+ }
+
+ public String getChainName() {
+ return this.serviceMethod.getChainName();
+ }
+
+ public boolean isHasSender() {
+ return this.serviceMethod.isHasSender();
+ }
+
+ public boolean isSenderOnLeft() {
+ return this.serviceMethod.isSenderOnLeft();
+ }
+}
diff --git a/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/method/ReceiverMethodSearcher.java b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/method/ReceiverMethodSearcher.java
new file mode 100644
index 0000000..b92cafb
--- /dev/null
+++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/method/ReceiverMethodSearcher.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.rpc.serializer.method;
+
+import org.apache.linkis.protocol.message.RequestProtocol;
+import org.apache.linkis.rpc.serializer.exception.MessageWarnException;
+import org.apache.linkis.rpc.serializer.parser.ServiceMethod;
+import org.apache.linkis.rpc.serializer.registry.SpringServiceRegistry;
+import org.apache.linkis.rpc.serializer.utils.MessageUtils;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+public class ReceiverMethodSearcher {
+
+ /** key是requestProtocol的全类名，Map中，key是groupName */
+ private final Map<String, Map<String, List<ServiceMethod>>> protocolServiceMethodCache =
+ new ConcurrentHashMap<>();
+
+ private SpringServiceRegistry serviceRegistry;
+
+ private void initRegistry() {
+ serviceRegistry = new SpringServiceRegistry();
+ }
+
+ public Map<String, List<MethodExecuteWrapper>> getMethodExecuteWrappers(
+ RequestProtocol requestProtocol) {
+ if (serviceRegistry == null) {
+ synchronized (ReceiverMethodSearcher.class) {
+ if (serviceRegistry == null) {
+ initRegistry();
+ }
+ }
+ }
+ String protocolName = requestProtocol.getClass().getName();
+ Map<String, List<ServiceMethod>> protocolServiceMethods =
+ this.protocolServiceMethodCache.get(protocolName);
+ // 静态信息，无需加锁
+ if (protocolServiceMethods == null) {
+ Map<String, List<ServiceMethod>> serviceMethodCache =
+ serviceRegistry.getServiceMethodCache();
+ // 找出注册方法中，参数是当前请求的父类的
+ Map<String, List<ServiceMethod>> serviceMatchs =
+ serviceMethodCache.entrySet().stream()
+ .filter(e -> MessageUtils.isAssignableFrom(e.getKey(), protocolName))
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ // group by chain name 扁平化后再group，这时protocol的父类可能和转换的处于同一个chain中
+ serviceMatchs =
+ serviceMatchs.values().stream()
+ .flatMap(Collection::stream)
+ .collect(Collectors.groupingBy(ServiceMethod::getChainName));
+ // order判断
+ for (List<ServiceMethod> value : serviceMatchs.values()) {
+ Integer repeatOrder = MessageUtils.repeatOrder(value);
+ if (repeatOrder != null && !MessageUtils.orderIsLast(repeatOrder, value)) {
+ throw new MessageWarnException(
+ 10000,
+ String.format(
+ "repeat order : %s for request %s", repeatOrder, protocolName));
+ }
+ }
+ this.protocolServiceMethodCache.put(protocolName, serviceMatchs);
+ }
+ // clone 对象并返回
+ return serviceMethod2Wrapper(this.protocolServiceMethodCache.get(protocolName));
+ }
+
+ private Map<String, List<MethodExecuteWrapper>> serviceMethod2Wrapper(
+ Map<String, List<ServiceMethod>> source) {
+ HashMap<String, List<MethodExecuteWrapper>> target = new HashMap<>();
+ source.forEach(
+ (k, v) ->
+ target.put(
+ k,
+ v.stream()
+ .map(MethodExecuteWrapper::new)
+ .collect(Collectors.toList())));
+ return target;
+ }
+}
diff --git a/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/parser/DefaultServiceParser.java b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/parser/DefaultServiceParser.java
new file mode 100644
index 0000000..760684f
--- /dev/null
+++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/parser/DefaultServiceParser.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.rpc.serializer.parser;
+
+import org.apache.linkis.rpc.Sender;
+import org.apache.linkis.rpc.serializer.annotation.Chain;
+import org.apache.linkis.rpc.serializer.annotation.Order;
+import org.apache.linkis.rpc.serializer.annotation.Receiver;
+
+import org.springframework.aop.support.AopUtils;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.Parameter;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class DefaultServiceParser implements ServiceParser {
+
+ Logger logger = LoggerFactory.getLogger(DefaultServiceParser.class);
+
+ @Override
+ public Map<String, List<ServiceMethod>> parse(Object service) {
+ // TODO: 2020/7/15 more analysis
+ // Method[] methods = service.getClass().getMethods();
+ Method[] methods = AopUtils.getTargetClass(service).getMethods();
+ return Arrays.stream(methods)
+ .filter(this::methodFilterPredicate)
+ .map(m -> getServiceMethod(m, service))
+ .collect(Collectors.groupingBy(ServiceMethod::getProtocolName));
+ }
+
+ private ServiceMethod getServiceMethod(Method method, Object service) {
+ ServiceMethod serviceMethod = new ServiceMethod();
+ serviceMethod.setMethod(method);
+ serviceMethod.setService(service);
+ serviceMethod.setAlias(
+ String.format("%s.%s", service.getClass().getName(), method.getName()));
+ Order order = method.getAnnotation(Order.class);
+ if (order != null) {
+ serviceMethod.setOrder(order.value());
+ }
+ Chain chain = method.getAnnotation(Chain.class);
+ if (chain != null) {
+ serviceMethod.setChainName(chain.value());
+ }
+ Parameter[] parameters = method.getParameters();
+ if (parameters.length == 2) {
+ serviceMethod.setHasSender(true);
+ if (Sender.class.isAssignableFrom(parameters[0].getType()))
+ serviceMethod.setSenderOnLeft(true);
+ }
+ logger.info(
+ method
+ + " parameter:"
+ + Arrays.toString(
+ Arrays.stream(parameters).map(Parameter::getName).toArray()));
+ @SuppressWarnings("all")
+ Parameter parameter =
+ Arrays.stream(parameters)
+ .filter(p -> !Sender.class.isAssignableFrom(p.getType()))
+ .findFirst()
+ .get();
+ serviceMethod.setProtocolName(parameter.getType().getName());
+ return serviceMethod;
+ }
+
+ /**
+ * 标注了@Receiver注解，方法至少一个参数 1个参数：非ServiceMethodContext 子类即可 2个参数 其中一个需要是ServiceMethodContext 的子类
+ * && 2个参数都非ServiceMethodContext 子类即可
+ *
+ * @param method
+ * @return
+ */
+ private boolean methodFilterPredicate(Method method) {
+ if (method.getAnnotation(Receiver.class) != null) {
+ Class<?>[] parameterTypes = method.getParameterTypes();
+ if (method.getParameterCount() == 1) {
+ return !Sender.class.isAssignableFrom(parameterTypes[0]);
+ } else if (method.getParameterCount() == 2) {
+ boolean hasContext =
+ Arrays.stream(parameterTypes).anyMatch(Sender.class::isAssignableFrom);
+ boolean allContext =
+ Arrays.stream(parameterTypes).allMatch(Sender.class::isAssignableFrom);
+ return hasContext && !allContext;
+ }
+ }
+ return false;
+ }
+}
diff --git a/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/parser/ServiceMethod.java b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/parser/ServiceMethod.java
new file mode 100644
index 0000000..99aba6f
--- /dev/null
+++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/parser/ServiceMethod.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.rpc.serializer.parser;
+
+import java.lang.reflect.Method;
+
+public class ServiceMethod {
+
+ private Object service;
+
+ private Method method;
+
+ private String alias;
+
+ private String protocolName;
+
+ private int order = 2147483647;
+
+ private boolean allowImplicit = true;
+
+ private boolean hasSender;
+
+ private boolean senderOnLeft;
+
+ private String chainName = "default";
+
+ public String getChainName() {
+ return chainName;
+ }
+
+ public void setChainName(String chainName) {
+ this.chainName = chainName;
+ }
+
+ public boolean isHasSender() {
+ return hasSender;
+ }
+
+ public void setHasSender(boolean hasSender) {
+ this.hasSender = hasSender;
+ }
+
+ public boolean isAllowImplicit() {
+ return allowImplicit;
+ }
+
+ public void setAllowImplicit(boolean allowImplicit) {
+ this.allowImplicit = allowImplicit;
+ }
+
+ public int getOrder() {
+ return order;
+ }
+
+ public void setOrder(int order) {
+ this.order = order;
+ }
+
+ public Object getService() {
+ return service;
+ }
+
+ public void setService(Object service) {
+ this.service = service;
+ }
+
+ public Method getMethod() {
+ return method;
+ }
+
+ public void setMethod(Method method) {
+ this.method = method;
+ }
+
+ public String getAlias() {
+ return alias;
+ }
+
+ public void setAlias(String alias) {
+ this.alias = alias;
+ }
+
+ public String getProtocolName() {
+ return protocolName;
+ }
+
+ public void setProtocolName(String protocolName) {
+ this.protocolName = protocolName;
+ }
+
+ public boolean isSenderOnLeft() {
+ return senderOnLeft;
+ }
+
+ public void setSenderOnLeft(boolean senderOnLeft) {
+ this.senderOnLeft = senderOnLeft;
+ }
+}
diff --git a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/request/DownloadAction.scala b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/parser/ServiceParser.java
similarity index 80%
copy from linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/request/DownloadAction.scala
copy to linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/parser/ServiceParser.java
index 5824db9..6e09e9c 100644
--- a/linkis-commons/linkis-httpclient/src/main/scala/org/apache/linkis/httpclient/request/DownloadAction.scala
+++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/parser/ServiceParser.java
@@ -5,23 +5,22 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.linkis.httpclient.request
-import scala.tools.nsc.interpreter.InputStream
+package org.apache.linkis.rpc.serializer.parser;
+import java.util.List;
+import java.util.Map;
-trait DownloadAction {
-
- def write(inputStream: InputStream): Unit
+public interface ServiceParser {
+ Map<String, List<ServiceMethod>> parse(Object service);
}
diff --git a/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/registry/AbstractServiceRegistry.java b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/registry/AbstractServiceRegistry.java
new file mode 100644
index 0000000..331f99b
--- /dev/null
+++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/registry/AbstractServiceRegistry.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.rpc.serializer.registry;
+
+import org.apache.linkis.common.utils.JavaLog;
+import org.apache.linkis.rpc.serializer.parser.DefaultServiceParser;
+import org.apache.linkis.rpc.serializer.parser.ServiceMethod;
+import org.apache.linkis.rpc.serializer.parser.ServiceParser;
+
+import org.springframework.aop.support.AopUtils;
+
+import com.google.common.collect.Interner;
+import com.google.common.collect.Interners;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+public abstract class AbstractServiceRegistry extends JavaLog implements ServiceRegistry {
+
+ @SuppressWarnings("all")
+ public final Interner<String> lock = Interners.<String>newWeakInterner();
+ /** key:requestprotocol or custom implicit object class name */
+ private final Map<String, List<ServiceMethod>> serviceMethodCache = new ConcurrentHashMap<>();
+
+ private final Map<String, Object> registedServieMap = new ConcurrentHashMap<>();
+
+ private final ServiceParser serviceParser = new DefaultServiceParser();
+
+ @SuppressWarnings("all")
+ @Override
+ public void register(Object service) {
+ // 防止不同方式注册时候的并发，比如spring和手动注册,同时防止不同包名下类名一样的service
+ // 默认用getName，则会拿到spring代理过的类，没有方法的注解，比如
+ // org.apache.linkis.jobhistory.service.impl.JobHistoryQueryServiceImpl$$EnhancerBySpringCGLIB$$5fa1dd59
+ // String serviceName = service.getClass().getName();
+ // 下面会拿到原始的class，包含方法注解信息
+ String serviceName = AopUtils.getTargetClass(service).getName();
+ synchronized (this.lock.intern(serviceName)) {
+ // 1.是否注册过
+ Object o = this.registedServieMap.get(serviceName);
+ if (o != null) return;
+ // 2..解析
+ Map<String, List<ServiceMethod>> serviceMethods = serviceParser.parse(service);
+ serviceMethods.entrySet().stream()
+ .forEach(
+ entry -> {
+ if (entry.getValue().size() != 1) {
+ error(
+ String.format(
+ "rpc receive method init error! find %s method for the request:%s, this type of rpc request will not be handled!",
+ entry.getValue().size(), entry.getKey()));
+ }
+ });
+ // 3.注册
+ serviceMethods.forEach(this::register);
+ this.registedServieMap.put(serviceName, service);
+ }
+ }
+
+ /**
+ * @param key
+ * @param value
+ * @throws MessageWarnException
+ */
+ @SuppressWarnings("all")
+ private void register(String key, List<ServiceMethod> serviceMethods) {
+ // 防止相同key在不同service的并发注册
+ synchronized (this.lock.intern(key)) {
+ // 1.添加cache
+ refreshServiceMethodCache(key, serviceMethods);
+ }
+ }
+
+ private void refreshServiceMethodCache(String key, List<ServiceMethod> serviceMethods) {
+ this.serviceMethodCache.computeIfAbsent(key, k -> new ArrayList<>()).addAll(serviceMethods);
+ }
+
+ public Map<String, List<ServiceMethod>> getServiceMethodCache() {
+ return this.serviceMethodCache;
+ }
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/conf/ECMErrorCode.scala b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/registry/ServiceRegistry.java
similarity index 85%
copy from linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/conf/ECMErrorCode.scala
copy to linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/registry/ServiceRegistry.java
index 2b1456c..2435648 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/conf/ECMErrorCode.scala
+++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/registry/ServiceRegistry.java
@@ -5,23 +5,19 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.linkis.ecm.core.conf
+package org.apache.linkis.rpc.serializer.registry;
-object ECMErrorCode {
-
- val PROCESS_WAITFOR_ERROR = 20001
-
- val EC_START_FAILED = 11102
+public interface ServiceRegistry {
+ void register(Object service);
}
diff --git a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineSwitchService.scala b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/registry/SpringServiceRegistry.java
similarity index 50%
copy from linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineSwitchService.scala
copy to linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/registry/SpringServiceRegistry.java
index b6f7be7..50389ac 100644
--- a/linkis-computation-governance/linkis-manager/linkis-application-manager/src/main/scala/org/apache/linkis/manager/am/service/engine/DefaultEngineSwitchService.scala
+++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/registry/SpringServiceRegistry.java
@@ -5,28 +5,37 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.linkis.manager.am.service.engine
-import org.apache.linkis.manager.common.entity.node.EngineNode
-import org.apache.linkis.manager.common.protocol.engine.EngineSwitchRequest
-import org.apache.linkis.message.annotation.Receiver
-import org.springframework.stereotype.Service
+package org.apache.linkis.rpc.serializer.registry;
+import org.apache.linkis.rpc.conf.RPCConfiguration;
+import org.apache.linkis.rpc.serializer.annotation.Receiver;
+import org.apache.linkis.rpc.serializer.utils.MessageUtils;
-@Service
-class DefaultEngineSwitchService extends EngineSwitchService {
+import java.lang.reflect.Method;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
- @Receiver
- override def switchEngine(engineSwitchRequest: EngineSwitchRequest): EngineNode = null
+public class SpringServiceRegistry extends AbstractServiceRegistry {
+ public SpringServiceRegistry() {
+ Set<? extends Class<?>> services =
+ RPCConfiguration.REFLECTIONS().getMethodsAnnotatedWith(Receiver.class).stream()
+ .map(Method::getDeclaringClass)
+ .collect(Collectors.toSet());
+ services.stream()
+ .map(MessageUtils::getBean)
+ .filter(Objects::nonNull)
+ .forEach(this::register);
+ }
}
diff --git a/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/utils/MessageUtils.java b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/utils/MessageUtils.java
new file mode 100644
index 0000000..04d9005
--- /dev/null
+++ b/linkis-commons/linkis-rpc/src/main/java/org/apache/linkis/rpc/serializer/utils/MessageUtils.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.rpc.serializer.utils;
+
+import org.apache.linkis.DataWorkCloudApplication;
+import org.apache.linkis.rpc.serializer.method.MethodExecuteWrapper;
+import org.apache.linkis.rpc.serializer.parser.ServiceMethod;
+
+import org.springframework.beans.factory.NoSuchBeanDefinitionException;
+import org.springframework.context.ApplicationContext;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class MessageUtils {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(MessageUtils.class);
+
+ public static <T> T getBean(Class<T> tClass) {
+ T t = null;
+ ApplicationContext applicationContext = DataWorkCloudApplication.getApplicationContext();
+ if (applicationContext != null) {
+ try {
+ t = applicationContext.getBean(tClass);
+ } catch (NoSuchBeanDefinitionException e) {
+ LOGGER.warn(String.format("can not get bean from spring ioc:%s", tClass.getName()));
+ }
+ }
+ return t;
+ }
+
+ public static boolean isAssignableFrom(String supperClassName, String className) {
+ try {
+ return Class.forName(supperClassName).isAssignableFrom(Class.forName(className));
+ } catch (ClassNotFoundException e) {
+ LOGGER.error("class not found", e);
+ return false;
+ }
+ }
+
+ public static boolean orderIsMin(
+ MethodExecuteWrapper methodExecuteWrapper,
+ List<MethodExecuteWrapper> methodExecuteWrappers) {
+ for (MethodExecuteWrapper tmp : methodExecuteWrappers) {
+ if (tmp.getOrder() < methodExecuteWrapper.getOrder()) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public static boolean orderIsLast(int order, List<ServiceMethod> serviceMethods) {
+ // TODO: 2020/8/5 方法判断修改为重复的order 支持头部
+ if (order == 2147483647) return true;
+ for (ServiceMethod serviceMethod : serviceMethods) {
+ if (serviceMethod.getOrder() > order) {
+ return false;
+ }
+ }
+ return false;
+ }
+
+ public static Integer repeatOrder(List<ServiceMethod> serviceMethods) {
+ Map<Integer, Integer> tmp = new HashMap<>();
+ for (ServiceMethod serviceMethod : serviceMethods) {
+ int order = serviceMethod.getOrder();
+ if (tmp.get(order) == null) {
+ tmp.put(order, order);
+ } else {
+ return order;
+ }
+ }
+ return null;
+ }
+}
diff --git a/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/MessageReceiver.scala b/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/MessageReceiver.scala
new file mode 100644
index 0000000..37b46dc
--- /dev/null
+++ b/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/MessageReceiver.scala
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.rpc
+
+import org.apache.linkis.protocol.message.RequestProtocol
+import org.apache.linkis.rpc.serializer.method.{MessageExecutor, ReceiverMethodSearcher}
+
+import scala.concurrent.duration.Duration
+import scala.language.implicitConversions
+
+
+class MessageReceiver extends Receiver {
+
+ private val receiverMethodSearcher = new ReceiverMethodSearcher
+
+ private val messageExecutor = new MessageExecutor
+
+ override def receive(message: Any, sender: Sender): Unit = {
+ receiveAndReply(message, sender)
+ }
+
+ override def receiveAndReply(message: Any, sender: Sender): Any = {
+ message match {
+ case requestProtocol: RequestProtocol => {
+ val methodExecuteWrapper = receiverMethodSearcher.getMethodExecuteWrappers(requestProtocol)
+ messageExecutor.execute(requestProtocol, methodExecuteWrapper, sender)
+ }
+ case _ =>
+ }
+ }
+
+ // TODO
+ override def receiveAndReply(message: Any, duration: Duration, sender: Sender): Any = {
+
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/resource/ResourceProtocol.scala b/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/MessageReceiverChooser.scala
similarity index 69%
copy from linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/resource/ResourceProtocol.scala
copy to linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/MessageReceiverChooser.scala
index ddf0252..ebb0ce3 100644
--- a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-manager-common/src/main/scala/org/apache/linkis/manager/common/protocol/resource/ResourceProtocol.scala
+++ b/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/MessageReceiverChooser.scala
@@ -5,25 +5,30 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.linkis.manager.common.protocol.resource
-import org.apache.linkis.common.ServiceInstance
-import org.apache.linkis.manager.common.entity.resource.NodeResource
+package org.apache.linkis.rpc
+
import org.apache.linkis.protocol.message.RequestProtocol
+import java.util
-trait ResourceProtocol extends RequestProtocol
-case class ResourceUsedProtocol(serviceInstance: ServiceInstance, engineResource: NodeResource, ticketId: String = null) extends RequestProtocol
+class MessageReceiverChooser extends ReceiverChooser {
+ val receiver: Option[Receiver] = Option(new MessageReceiver)
+ override def chooseReceiver(event: RPCMessageEvent): Option[Receiver] = event.message match {
+ case _: util.Map[String, Object] => receiver
+ case _: RequestProtocol => receiver
+ case _ => None
+ }
+}
diff --git a/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/RPCReceiveRestful.scala b/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/RPCReceiveRestful.scala
index 0d9175e..26cf5b6 100644
--- a/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/RPCReceiveRestful.scala
+++ b/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/RPCReceiveRestful.scala
@@ -70,6 +70,8 @@ private[rpc] class RPCReceiveRestful extends RPCReceiveRemote with Logging {
def initListenerBus(): Unit = {
if(!receiverChoosers.exists(_.isInstanceOf[CommonReceiverChooser]))
receiverChoosers = receiverChoosers :+ new CommonReceiverChooser
+ if(!receiverChoosers.exists(_.isInstanceOf[MessageReceiverChooser]))
+ receiverChoosers = receiverChoosers :+ new MessageReceiverChooser
info("init all receiverChoosers in spring beans, list => " + receiverChoosers.toList)
if(!receiverSenderBuilders.exists(_.isInstanceOf[CommonReceiverSenderBuilder]))
receiverSenderBuilders = receiverSenderBuilders :+ new CommonReceiverSenderBuilder
@@ -108,7 +110,7 @@ private[rpc] class RPCReceiveRestful extends RPCReceiveRemote with Logging {
}
private implicit def toMessage(obj: Any): Message = obj match {
- case Unit | () =>
+ case Unit | () | null =>
RPCProduct.getRPCProduct.ok()
case _: BoxedUnit => RPCProduct.getRPCProduct.ok()
case _ =>
diff --git a/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/conf/RPCConfiguration.scala b/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/conf/RPCConfiguration.scala
index c23d289..305e69d 100644
--- a/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/conf/RPCConfiguration.scala
+++ b/linkis-commons/linkis-rpc/src/main/scala/org/apache/linkis/rpc/conf/RPCConfiguration.scala
@@ -18,6 +18,8 @@
package org.apache.linkis.rpc.conf
import org.apache.linkis.common.conf.{CommonVars, TimeType}
+import org.reflections.Reflections
+import org.reflections.scanners.{MethodAnnotationsScanner, SubTypesScanner, TypeAnnotationsScanner}
object RPCConfiguration {
@@ -25,7 +27,7 @@ object RPCConfiguration {
val BDP_RPC_BROADCAST_THREAD_SIZE = CommonVars("wds.linkis.rpc.broadcast.thread.num", new Integer(25))
val BDP_RPC_EUREKA_SERVICE_REFRESH_INTERVAL = CommonVars("wds.linkis.rpc.eureka.client.refresh.interval", new TimeType("1s"))
- val BDP_RPC_EUREKA_SERVICE_REFRESH_MAX_WAIT_TIME = CommonVars("wds.linkis.rpc.eureka.client.refresh.wait.time.max", new TimeType("1m"))
+ val BDP_RPC_EUREKA_SERVICE_REFRESH_MAX_WAIT_TIME = CommonVars("wds.linkis.rpc.eureka.client.refresh.wait.time.max", new TimeType("30s"))
val BDP_RPC_RECEIVER_ASYN_CONSUMER_THREAD_MAX = CommonVars("wds.linkis.rpc.receiver.asyn.consumer.thread.max", 400)
val BDP_RPC_RECEIVER_ASYN_CONSUMER_THREAD_FREE_TIME_MAX = CommonVars("wds.linkis.rpc.receiver.asyn.consumer.freeTime.max", new TimeType("2m"))
val BDP_RPC_RECEIVER_ASYN_QUEUE_CAPACITY = CommonVars("wds.linkis.rpc.receiver.asyn.queue.size.max", 5000)
@@ -46,4 +48,8 @@ object RPCConfiguration {
val ENABLE_LOCAL_MESSAGE = CommonVars("wds.linkis.rpc.conf.enable.local.message", false)
val LOCAL_APP_LIST = CommonVars("wds.linkis.rpc.conf.local.app.list", "").getValue.split(",")
+ val SERVICE_SCAN_PACKAGE = CommonVars("wds.linkis.ms.service.scan.package", "org.apache.linkis").getValue
+
+ val REFLECTIONS = new Reflections(SERVICE_SCAN_PACKAGE, new MethodAnnotationsScanner(), new TypeAnnotationsScanner(), new SubTypesScanner())
+
}
diff --git a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/LoopArrayQueue.scala b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/LoopArrayQueue.scala
index bf49b37..e4a83d6 100644
--- a/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/LoopArrayQueue.scala
+++ b/linkis-commons/linkis-scheduler/src/main/scala/org/apache/linkis/scheduler/queue/LoopArrayQueue.scala
@@ -17,9 +17,10 @@
package org.apache.linkis.scheduler.queue
+import org.apache.linkis.common.utils.Logging
-class LoopArrayQueue(var group: Group) extends ConsumeQueue{
+class LoopArrayQueue(var group: Group) extends ConsumeQueue with Logging {
private val eventQueue: Array[Any] = new Array[Any](group.getMaximumCapacity)
private val maxCapacity: Int = group.getMaximumCapacity
@@ -198,12 +199,12 @@ override def offer(event: SchedulerEvent): Option[Int] = {
val _max = max
if(takeIndex < _min) takeIndex = _min
else if(takeIndex > _max) {
- println(s"none, notice...max: ${_max}, takeIndex: $takeIndex, realSize: $realSize.")
+ logger.info(s"none, notice...max: ${_max}, takeIndex: $takeIndex, realSize: $realSize.")
return None
}
val t = get(takeIndex)
if(t == null) {
- println("null, notice...")
+ logger.info("null, notice...")
}
takeIndex += 1
t
diff --git a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/HDFSFileSystem.java b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/HDFSFileSystem.java
index f658b11..e14d51c 100644
--- a/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/HDFSFileSystem.java
+++ b/linkis-commons/linkis-storage/src/main/java/org/apache/linkis/storage/fs/impl/HDFSFileSystem.java
@@ -18,6 +18,7 @@
package org.apache.linkis.storage.fs.impl;
import org.apache.linkis.common.io.FsPath;
+import org.apache.linkis.hadoop.common.conf.HadoopConf;
import org.apache.linkis.hadoop.common.utils.HDFSUtils;
import org.apache.linkis.storage.domain.FsPathListWithError;
import org.apache.linkis.storage.fs.FileSystem;
@@ -28,7 +29,6 @@ import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
@@ -50,6 +50,7 @@ public class HDFSFileSystem extends FileSystem {
public static final String HDFS_PREFIX_WITHOUT_AUTH = "hdfs:///";
public static final String HDFS_PREFIX_WITH_AUTH = "hdfs://";
+ private static final String FS_CLOSED_ERROR = "Filesystem closed";
private org.apache.hadoop.fs.FileSystem fs = null;
private Configuration conf = null;
@@ -192,7 +193,9 @@ public class HDFSFileSystem extends FileSystem {
}
}
}
- /*conf.set("fs.hdfs.impl.disable.cache","true");*/
+ if (StorageConfiguration.FS_CACHE_DISABLE().getValue()) {
+ conf.set("fs.hdfs.impl.disable.cache", "true");
+ }
fs = HDFSUtils.getHDFSUserFileSystem(user, conf);
if (fs == null) {
throw new IOException("init HDFS FileSystem failed!");
@@ -238,7 +241,7 @@ public class HDFSFileSystem extends FileSystem {
if (!overwrite) {
return fs.append(new Path(path));
} else {
- FSDataOutputStream out = fs.create(new Path(path), true);
+ OutputStream out = fs.create(new Path(path), true);
this.setPermission(dest, this.getDefaultFilePerm());
return out;
}
@@ -294,7 +297,33 @@ public class HDFSFileSystem extends FileSystem {
@Override
public boolean exists(FsPath dest) throws IOException {
- return fs.exists(new Path(checkHDFSPath(dest.getPath())));
+ try {
+ return fs.exists(new Path(checkHDFSPath(dest.getPath())));
+ } catch (IOException e) {
+ if (null != e.getMessage() && e.getMessage().contains(FS_CLOSED_ERROR)) {
+ logger.info("Failed to execute exists, retry", e);
+ resetRootHdfs();
+ return fs.exists(new Path(checkHDFSPath(dest.getPath())));
+ } else {
+ throw e;
+ }
+ }
+ }
+
+ private void resetRootHdfs() {
+ if (fs != null) {
+ synchronized (this) {
+ if (fs != null) {
+ if (HadoopConf.HDFS_ENABLE_CACHE()) {
+ HDFSUtils.closeHDFSFIleSystem(fs, user, true);
+ } else {
+ HDFSUtils.closeHDFSFIleSystem(fs, user);
+ }
+ logger.warn(user + "FS reset close.");
+ fs = HDFSUtils.getHDFSUserFileSystem(user, conf);
+ }
+ }
+ }
}
@Override
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala
index 1aa48c1..3074862 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/domain/DataType.scala
@@ -74,22 +74,31 @@ object DataType extends Logging{
def toValue(dataType: DataType, value: String): Any = Utils.tryCatch(dataType match {
case NullType => null
case StringType | CharType | VarcharType | StructType | ListType | ArrayType | MapType => value
- case BooleanType => if(isNull(value)) null else value.toBoolean
- case ShortIntType => if(isNull(value)) null else value.toShort
- case IntType =>if(isNull(value)) null else value.toInt
- case LongType | BigIntType => if(isNull(value)) null else value.toLong
- case FloatType => if(isNull(value)) null else value.toFloat
- case DoubleType => if(isNull(value)) null else value.toDouble
- case DecimalType => if(isNull(value)) null else BigDecimal(value)
- case DateType => if(isNull(value)) null else Date.valueOf(value)
- case TimestampType => if(isNull(value)) null else Timestamp.valueOf(value).toString.stripSuffix(".0")
+ case BooleanType => if(isNumberNull(value)) null else value.toBoolean
+ case ShortIntType => if(isNumberNull(value)) null else value.toShort
+ case IntType =>if(isNumberNull(value)) null else value.toInt
+ case LongType | BigIntType => if(isNumberNull(value)) null else value.toLong
+ case FloatType => if(isNumberNull(value)) null else value.toFloat
+ case DoubleType => if(isNumberNull(value)) null else value.toDouble
+ case DecimalType => if(isNumberNull(value)) null else BigDecimal(value)
+ case DateType => if(isNumberNull(value)) null else Date.valueOf(value)
+ case TimestampType => if(isNumberNull(value)) null else Timestamp.valueOf(value).toString.stripSuffix(".0")
case BinaryType => if(isNull(value)) null else value.getBytes()
case _ => value
}){
- t => warn(s"Failed to $value switch to dataType:",t)
+ t =>
+ logger.debug(s"Failed to $value switch to dataType:", t)
value
}
- def isNull(value:String):Boolean= if(value == null || value == NULL_VALUE || value.trim == "") true else false
+
+ def isNull(value: String): Boolean = if (value == null || value == NULL_VALUE || value.trim == "") true else false
+
+ def isNumberNull(value: String): Boolean = if (null == value || NULL_VALUE.equalsIgnoreCase(value) || value.trim == "") {
+ true
+ } else {
+ false
+ }
+
}
abstract class DataType(val typeName:String,
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala
index 3de246a..71ff18d 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/ResultSetReader.scala
@@ -45,7 +45,12 @@ object ResultSetReader {
val resultSet = rsFactory.getResultSetByPath(resPath)
val fs = FSFactory.getFs(resPath)
fs.init(null)
- ResultSetReader.getResultSetReader(resultSet, fs.read(resPath))
+ val reader = ResultSetReader.getResultSetReader(resultSet, fs.read(resPath))
+ reader match {
+ case storageResultSetReader: StorageResultSetReader[_, _] => storageResultSetReader.setFs(fs)
+ case _ =>
+ }
+ reader
}
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala
index cee259e..d35a509 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/resultset/StorageResultSetReader.scala
@@ -20,8 +20,8 @@ package org.apache.linkis.storage.resultset
import java.io.{ByteArrayInputStream, IOException, InputStream}
import org.apache.linkis.common.io.resultset.{ResultSet, ResultSetReader}
-import org.apache.linkis.common.io.{MetaData, Record}
-import org.apache.linkis.common.utils.Logging
+import org.apache.linkis.common.io.{Fs, MetaData, Record}
+import org.apache.linkis.common.utils.{Logging, Utils}
import org.apache.linkis.storage.domain.Dolphin
import org.apache.linkis.storage.exception.StorageWarnException
import org.apache.linkis.storage.utils.StorageUtils
@@ -38,6 +38,8 @@ class StorageResultSetReader[K <: MetaData, V <: Record](resultSet: ResultSet[K,
private var colCount = 0
private var rowCount = 0
+ private var fs: Fs = _
+
private val READ_CACHE = 1024
private val bytes = new Array[Byte](READ_CACHE)
@@ -96,6 +98,9 @@ class StorageResultSetReader[K <: MetaData, V <: Record](resultSet: ResultSet[K,
row
}
+ def setFs(fs:Fs):Unit = this.fs = fs
+ def getFs:Fs = this.fs
+
@scala.throws[IOException]
override def getMetaData: MetaData = {
if(metaData == null) init()
@@ -108,8 +113,12 @@ class StorageResultSetReader[K <: MetaData, V <: Record](resultSet: ResultSet[K,
if(recordNum < 0 ) return -1
if(metaData == null) getMetaData
- for(i <- recordNum until (0, -1)){
- try inputStream.skip(Dolphin.readInt(inputStream)) catch { case t: Throwable => return -1}
+ for(i <- recordNum until (0, -1)) {
+ try inputStream.skip(Dolphin.readInt(inputStream))
+ catch {
+ case t: Throwable =>
+ return recordNum - i
+ }
}
recordNum
}
@@ -130,5 +139,8 @@ class StorageResultSetReader[K <: MetaData, V <: Record](resultSet: ResultSet[K,
@scala.throws[IOException]
override def available: Long = inputStream.available()
- override def close(): Unit = inputStream.close()
+ override def close(): Unit = {
+ inputStream.close()
+ if(this.fs != null) Utils.tryQuietly(this.fs.close())
+ }
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/reader/StorageScriptFsReader.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/reader/StorageScriptFsReader.scala
index 1a10912..c7b06a1 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/reader/StorageScriptFsReader.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/script/reader/StorageScriptFsReader.scala
@@ -66,7 +66,11 @@ class StorageScriptFsReader(val path: FsPath, val charset: String, val inputStre
}
@scala.throws[IOException]
- override def skip(recordNum: Int): Int = -1
+ override def skip(recordNum: Int): Int = {
+ if(recordNum < 0 ) return -1
+ if(metadata == null) getMetaData
+ try bufferedReader.skip(recordNum).toInt catch { case t: Throwable => recordNum }
+ }
@scala.throws[IOException]
override def getPosition: Long = -1L
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/AbstractFileSource.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/AbstractFileSource.scala
index 424e739..52aad63 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/AbstractFileSource.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/AbstractFileSource.scala
@@ -62,4 +62,6 @@ abstract class AbstractFileSource(var fileSplits: Array[FileSplit]) extends File
override def getTypes: Array[String] = this.fileSplits.map(_.`type`)
+ override def getFileInfo(needToCountRowNumber: Int = 5000): Array[Pair[Int, Int]] = fileSplits.map(_.getFileInfo(needToCountRowNumber))
+
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala
index f13a0f5..43138af 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSource.scala
@@ -36,6 +36,8 @@ trait FileSource extends Closeable {
def collect(): Array[Pair[Object, util.ArrayList[Array[String]]]]
+ def getFileInfo(needToCountRowNumber: Int = 5000): Array[Pair[Int, Int]]
+
def write[K <: MetaData, V <: Record](fsWriter: FsWriter[K, V]): Unit
def addParams(params: util.Map[String, String]): FileSource
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala
index 8191c3a..ff0b70b 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/FileSplit.scala
@@ -78,6 +78,25 @@ class FileSplit(var fsReader: FsReader[_ <: MetaData, _ <: Record], var `type`:
t
}
+ /**
+ * Get the colNumber and rowNumber of the row to be counted
+ * @param needToCountRowNumber
+ * @return colNumber, rowNumber
+ */
+ def getFileInfo(needToCountRowNumber: Int = 5000): Pair[Int, Int] = {
+ val metaData = fsReader.getMetaData
+ val colNumber = metaData match {
+ case tableMetaData: TableMetaData => tableMetaData.columns.length
+ case _ => 1
+ }
+ val rowNumber = if (needToCountRowNumber == -1) {
+ fsReader.skip(Int.MaxValue)
+ } else {
+ fsReader.skip(needToCountRowNumber)
+ }
+ new Pair(colNumber, rowNumber)
+ }
+
def write[K <: MetaData, V <: Record](fsWriter: FsWriter[K, V]): Unit = {
`while`(fsWriter.addMetaData, fsWriter.addRecord)
}
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/ResultsetFileSource.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/ResultsetFileSource.scala
index 33638d1..a6b14fb 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/ResultsetFileSource.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/source/ResultsetFileSource.scala
@@ -25,7 +25,8 @@ class ResultsetFileSource(fileSplits: Array[FileSplit]) extends AbstractFileSour
shuffle({
case t: TableRecord => new TableRecord(t.row.map {
- case null | "NULL" | "" => getParams.getOrDefault("nullValue", "NULL")
+ case null | "NULL" => getParams.getOrDefault("nullValue", "NULL")
+ case "" => getParams.getOrDefault("nullValue", "")
case value: Double => StorageUtils.doubleToString(value)
case r => r
})
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala
index f40d83e..491536b 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/FileSystemUtils.scala
@@ -29,7 +29,7 @@ import org.apache.linkis.storage.fs.impl.LocalFileSystem
object FileSystemUtils extends Logging{
def copyFile(filePath: FsPath, origin: FsPath, user: String): Unit = {
- val fileSystem = FSFactory.getFsByProxyUser(filePath,user).asInstanceOf[FileSystem]
+ val fileSystem = FSFactory.getFsByProxyUser(filePath, user).asInstanceOf[FileSystem]
Utils.tryFinally {
fileSystem.init(null)
if (!fileSystem.exists(filePath)) {
@@ -43,41 +43,36 @@ object FileSystemUtils extends Logging{
}
/**
- * Create a new file(创建新文件)
- * @param filePath
- * @param createParentWhenNotExists Whether to recursively create a directory(是否递归创建目录)
- */
+ * Create a new file(创建新文件)
+ *
+ * @param filePath
+ * @param createParentWhenNotExists Whether to recursively create a directory(是否递归创建目录)
+ */
def createNewFile(filePath: FsPath, createParentWhenNotExists: Boolean): Unit = {
- val fileSystem = FSFactory.getFs(filePath).asInstanceOf[FileSystem]
+ createNewFile(filePath, StorageUtils.getJvmUser, createParentWhenNotExists)
+ }
+
+ def createNewFile(filePath: FsPath, user: String, createParentWhenNotExists: Boolean): Unit = {
+ val fileSystem = FSFactory.getFsByProxyUser(filePath, user).asInstanceOf[FileSystem]
Utils.tryFinally {
fileSystem.init(null)
- if (!fileSystem.exists(filePath)) {
- if (!fileSystem.exists(filePath.getParent)) {
- if(!createParentWhenNotExists) throw new IOException("parent dir " + filePath.getParent.getPath + " dose not exists.")
- fileSystem.mkdirs(filePath.getParent)
- }
- fileSystem.createNewFile(filePath)
- }
+ createNewFileWithFileSystem(fileSystem, filePath, user, createParentWhenNotExists)
}(Utils.tryQuietly(fileSystem.close()))
}
- def createNewFile(filePath: FsPath, user:String,createParentWhenNotExists: Boolean): Unit = {
- val fileSystem = FSFactory.getFsByProxyUser(filePath,user).asInstanceOf[FileSystem]
- Utils.tryFinally {
- fileSystem.init(null)
- if (!fileSystem.exists(filePath)) {
- if (!fileSystem.exists(filePath.getParent)) {
- if(!createParentWhenNotExists) throw new IOException("parent dir " + filePath.getParent.getPath + " dose not exists.")
- mkdirs(fileSystem,filePath.getParent, user)
- }
- fileSystem.createNewFile(filePath)
- fileSystem match {
- case l:LocalFileSystem => fileSystem.setOwner(filePath,user)
- case _ => info(s"doesn't need to call setOwner")
- }
- /*fileSystem.setOwner(filePath,user,StorageConfiguration.STORAGE_HDFS_GROUP.getValue)*/
+ def createNewFileWithFileSystem(fileSystem: FileSystem, filePath: FsPath, user: String, createParentWhenNotExists: Boolean): Unit = {
+ if (!fileSystem.exists(filePath)) {
+ if (!fileSystem.exists(filePath.getParent)) {
+ if (!createParentWhenNotExists) throw new IOException("parent dir " + filePath.getParent.getPath + " dose not exists.")
+ mkdirs(fileSystem, filePath.getParent, user)
}
- }(Utils.tryQuietly(fileSystem.close()))
+ fileSystem.createNewFile(filePath)
+ fileSystem match {
+ case l: LocalFileSystem => fileSystem.setOwner(filePath, user)
+ case _ => info(s"doesn't need to call setOwner")
+ }
+ /*fileSystem.setOwner(filePath,user,StorageConfiguration.STORAGE_HDFS_GROUP.getValue)*/
+ }
}
/**
@@ -89,22 +84,22 @@ object FileSystemUtils extends Logging{
* @return
*/
@throws[IOException]
- def mkdirs(fileSystem: FileSystem,dest: FsPath, user: String): Boolean = {
+ def mkdirs(fileSystem: FileSystem, dest: FsPath, user: String): Boolean = {
var parentPath = dest.getParent
val dirsToMake = new util.Stack[FsPath]()
dirsToMake.push(dest)
- while (!fileSystem.exists(parentPath)){
+ while (!fileSystem.exists(parentPath)) {
dirsToMake.push(parentPath)
parentPath = parentPath.getParent
}
- if(! fileSystem.canExecute(parentPath)){
+ if (!fileSystem.canExecute(parentPath)) {
throw new IOException("You have not permission to access path " + dest.getPath)
}
- while (!dirsToMake.empty()){
+ while (!dirsToMake.empty()) {
val path = dirsToMake.pop()
fileSystem.mkdir(path)
fileSystem match {
- case l:LocalFileSystem => fileSystem.setOwner(path,user)
+ case l: LocalFileSystem => fileSystem.setOwner(path, user)
case _ => info(s"doesn't need to call setOwner")
}
//fileSystem.setOwner(path,user,StorageConfiguration.STORAGE_HDFS_GROUP.getValue)
diff --git a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageConfiguration.scala b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageConfiguration.scala
index e384fcc..96cc891 100644
--- a/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageConfiguration.scala
+++ b/linkis-commons/linkis-storage/src/main/scala/org/apache/linkis/storage/utils/StorageConfiguration.scala
@@ -66,4 +66,6 @@ object StorageConfiguration {
val HDFS_PATH_PREFIX_REMOVE = CommonVars[Boolean]("wds.linkis.storage.hdfs.prefxi.remove", true)
+ val FS_CACHE_DISABLE = CommonVars[java.lang.Boolean]("wds.linkis.fs.hdfs.impl.disable.cache", false)
+
}
diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/pom.xml b/linkis-computation-governance/linkis-client/linkis-computation-client/pom.xml
index 577cab0..ab78905 100644
--- a/linkis-computation-governance/linkis-client/linkis-computation-client/pom.xml
+++ b/linkis-computation-governance/linkis-client/linkis-computation-client/pom.xml
@@ -62,6 +62,20 @@
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-bml-client</artifactId>
<version>${linkis.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.linkis</groupId>
+ <artifactId>linkis-storage</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.linkis</groupId>
+ <artifactId>linkis-hadoop-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.json4s</groupId>
+ <artifactId>json4s-jackson_${scala.binary.version}</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/LinkisJobBuilder.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/LinkisJobBuilder.scala
index 8ec2f66..a1f2bf9 100644
--- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/LinkisJobBuilder.scala
+++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/LinkisJobBuilder.scala
@@ -14,7 +14,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
package org.apache.linkis.computation.client
import java.util
@@ -40,10 +40,11 @@ trait LinkisJobBuilder[Job <: LinkisJob] {
protected var params: util.Map[String, Any] = _
protected var source: util.Map[String, Any] = _
- protected def ensureNotNull(obj: Any, errorMsg: String): Unit = if(obj == null)
+ protected def ensureNotNull(obj: Any, errorMsg: String): Unit = if (obj == null) {
throw new UJESJobException(s"$errorMsg cannot be null.")
+ }
- protected def nullThenSet(obj: Any)(setTo: => Unit): Unit = if(obj == null) setTo
+ protected def nullThenSet(obj: Any)(setTo: => Unit): Unit = if (obj == null) setTo
def addExecuteUser(executeUser: String): this.type = {
this.executeUser = executeUser
@@ -56,7 +57,7 @@ trait LinkisJobBuilder[Job <: LinkisJob] {
}
def addJobContent(key: String, value: Any): this.type = {
- if(jobContent == null) jobContent = new util.HashMap[String, Any]
+ if (jobContent == null) jobContent = new util.HashMap[String, Any]
jobContent.put(key, value)
this
}
@@ -67,7 +68,7 @@ trait LinkisJobBuilder[Job <: LinkisJob] {
}
def addLabel(key: String, value: Any): this.type = {
- if(labels == null) labels = new util.HashMap[String, Any]
+ if (labels == null) labels = new util.HashMap[String, Any]
labels.put(key, value)
this
}
@@ -83,7 +84,7 @@ trait LinkisJobBuilder[Job <: LinkisJob] {
}
def addSource(key: String, value: Any): this.type = {
- if(source == null) source = new util.HashMap[String, Any]
+ if (source == null) source = new util.HashMap[String, Any]
source.put(key, value)
this
}
@@ -165,7 +166,7 @@ object LinkisJobBuilder {
def setDefaultClientConfig(clientConfig: DWSClientConfig): Unit = this.clientConfig = clientConfig
def getDefaultClientConfig: DWSClientConfig = {
- if(clientConfig == null) synchronized {
+ if (clientConfig == null) synchronized {
if (clientConfig == null) buildDefaultConfig()
}
clientConfig
@@ -174,9 +175,9 @@ object LinkisJobBuilder {
def setDefaultUJESClient(ujesClient: UJESClient): Unit = this.ujesClient = ujesClient
def getDefaultServerUrl: String = {
- if(StringUtils.isEmpty(serverUrl)) {
+ if (StringUtils.isEmpty(serverUrl)) {
serverUrl = Configuration.getGateWayURL()
- if(StringUtils.isEmpty(serverUrl)) throw new UJESClientBuilderException("serverUrl must be set!")
+ if (StringUtils.isEmpty(serverUrl)) throw new UJESClientBuilderException("serverUrl must be set!")
}
serverUrl
}
@@ -188,8 +189,8 @@ object LinkisJobBuilder {
private[client] def justGetDefaultUJESClient: UJESClient = ujesClient
def getDefaultUJESClient: UJESClient = {
- if(ujesClient == null) synchronized {
- if(clientConfig == null) buildDefaultConfig()
+ if (ujesClient == null) synchronized {
+ if (clientConfig == null) buildDefaultConfig()
if(ujesClient == null) {
ujesClient = new UJESClientImpl(clientConfig)
}
diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/LinkisJobClient.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/LinkisJobClient.scala
index d5e6974..ef452fb 100644
--- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/LinkisJobClient.scala
+++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/LinkisJobClient.scala
@@ -35,7 +35,7 @@ object LinkisJobClient extends Closeable {
val once = OnceJob
override def close(): Unit = {
- if(config.justGetDefaultUJESClient != null) {
+ if (config.justGetDefaultUJESClient != null) {
config.justGetDefaultUJESClient.close()
}
}
diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/interactive/InteractiveJobBuilder.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/interactive/InteractiveJobBuilder.scala
index 90ca613..f1af58c 100644
--- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/interactive/InteractiveJobBuilder.scala
+++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/interactive/InteractiveJobBuilder.scala
@@ -41,14 +41,15 @@ class InteractiveJobBuilder private[interactive]()
def setCode(code: String): this.type = addJobContent("code", code)
- def setRunType(runType: RunType): this.type= addJobContent("runType", runType.toString)
+ def setRunType(runType: RunType): this.type = addJobContent("runType", runType.toString)
def setRunTypeStr(runType: String): this.type = addJobContent("runType", runType)
override protected def validate(): Unit = {
- if(labels != null && !labels.containsKey(LabelKeyUtils.USER_CREATOR_LABEL_KEY)
- && StringUtils.isNotBlank(creator))
+ if (labels != null && !labels.containsKey(LabelKeyUtils.USER_CREATOR_LABEL_KEY)
+ && StringUtils.isNotBlank(creator)) {
addLabel(LabelKeyUtils.USER_CREATOR_LABEL_KEY, executeUser + "-" + creator)
+ }
super.validate()
}
diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/LinkisManagerClient.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/LinkisManagerClient.scala
index 25f8f90..48de998 100644
--- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/LinkisManagerClient.scala
+++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/LinkisManagerClient.scala
@@ -34,7 +34,8 @@ trait LinkisManagerClient extends Closeable {
def killEngineConn(killEngineConnAction: KillEngineConnAction): KillEngineConnResult
- def executeEngineConnOperation(engineOperateAction: EngineConnOperateAction): EngineConnOperateResult
+ def executeEngineConnOperation(engineConnOperateAction: EngineConnOperateAction): EngineConnOperateResult
+
}
object LinkisManagerClient {
diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/EngineConnOperateResult.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/EngineConnOperateResult.scala
index 0cdecb4..b2bf3ef 100644
--- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/EngineConnOperateResult.scala
+++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/result/EngineConnOperateResult.scala
@@ -39,13 +39,12 @@ class EngineConnOperateResult extends LinkisManagerResult {
def setIsError(isError: Boolean): Unit = this.isError = isError
- def getResult: util.Map[String, Any] = if(isError) throw new UJESJobException(20301, errorMsg) else result
+ def getResult: util.Map[String, Any] = if (isError) throw new UJESJobException(20301, errorMsg) else result
def getAsOption[T](key: String): Option[T] = Option(getAs(key))
- def getAs[T](key: String): T = getAs(key, null.asInstanceOf[T])
+ def getAs[T](key: String): T = getAs[T](key, null.asInstanceOf[T])
- def getAs[T](key: String, defaultValue: T): T = if(getResult != null && result.containsKey(key)) result.get(key).asInstanceOf[T]
- else defaultValue
-
-}
+ def getAs[T](key: String, defaultValue: T): T = if (getResult != null && result.containsKey(key)) result.get(key).asInstanceOf[T]
+ else defaultValue
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/simple/SimpleOnceJobBuilder.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/simple/SimpleOnceJobBuilder.scala
index 186ac82..c020390 100644
--- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/simple/SimpleOnceJobBuilder.scala
+++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/computation/client/once/simple/SimpleOnceJobBuilder.scala
@@ -78,15 +78,16 @@ class SimpleOnceJobBuilder private[simple]()
override def build(): SubmittableSimpleOnceJob = {
ensureNotNull(labels, "labels")
ensureNotNull(jobContent, "jobContent")
- nullThenSet(params){
+ nullThenSet(params) {
params = new util.HashMap[String, Any]
}
- nullThenSet(source){
+ nullThenSet(source) {
source = new util.HashMap[String, Any]()
}
addStartupParam("label." + LabelKeyConstant.CODE_TYPE_KEY, jobContent.get("runType"))
- if(!labels.containsKey(SimpleOnceJobBuilder.ONCE_ENGINE_CONN_MODE_LABEL_KEY))
+ if (!labels.containsKey(SimpleOnceJobBuilder.ONCE_ENGINE_CONN_MODE_LABEL_KEY)) {
addLabel(ONCE_ENGINE_CONN_MODE_LABEL_KEY, ONCE_ENGINE_CONN_MODE_LABEL_VALUE)
+ }
val properties = new util.HashMap[String, String]
properties.put(OnceExecutorContentUtils.ONCE_EXECUTOR_CONTENT_KEY, getOnceExecutorContent)
properties.putAll(TaskUtils.getStartupMap(params))
@@ -114,8 +115,8 @@ object SimpleOnceJobBuilder {
private var bmlClient: BmlClient = _
private var linkisManagerClient: LinkisManagerClient = _
def getBmlClient: BmlClient = {
- if(bmlClient == null) synchronized {
- if(bmlClient == null) {
+ if (bmlClient == null) synchronized {
+ if (bmlClient == null) {
bmlClient = BmlClientFactory.createBmlClient(LinkisJobBuilder.getDefaultClientConfig)
Utils.addShutdownHook(() => bmlClient.close())
}
@@ -123,8 +124,8 @@ object SimpleOnceJobBuilder {
bmlClient
}
def getLinkisManagerClient: LinkisManagerClient = {
- if(linkisManagerClient == null) synchronized {
- if(linkisManagerClient == null) {
+ if (linkisManagerClient == null) synchronized {
+ if (linkisManagerClient == null) {
linkisManagerClient = LinkisManagerClient(LinkisJobBuilder.getDefaultUJESClient)
Utils.addShutdownHook(() => linkisManagerClient.close())
}
diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/UJESClient.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/UJESClient.scala
index 792adbd..1dcb364 100644
--- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/UJESClient.scala
+++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/UJESClient.scala
@@ -91,6 +91,10 @@ abstract class UJESClient extends Closeable {
executeUJESJob(getColumnsAction).asInstanceOf[GetColumnsResult]
}
+ def getTableStatisticInfo(getTableStatisticInfoAction: GetTableStatisticInfoAction): GetTableStatisticInfoResult = {
+ executeUJESJob(getTableStatisticInfoAction).asInstanceOf[GetTableStatisticInfoResult]
+ }
+
}
object UJESClient {
def apply(clientConfig: DWSClientConfig): UJESClient = new UJESClientImpl(clientConfig)
diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/UJESClientImpl.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/UJESClientImpl.scala
index 39e15e0..9c63634 100644
--- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/UJESClientImpl.scala
+++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/UJESClientImpl.scala
@@ -23,7 +23,7 @@ import org.apache.linkis.httpclient.response.Result
import org.apache.linkis.ujes.client.request.UJESJobAction
class UJESClientImpl(clientConfig: DWSClientConfig) extends UJESClient {
- private val dwsHttpClient = new DWSHttpClient(clientConfig, "UJES-Job-Execution-Thread")
+ private val dwsHttpClient = new DWSHttpClient(clientConfig, "Linkis-Job-Execution-Thread")
override protected[client] def executeUJESJob(ujesJobAction: UJESJobAction): Result = ujesJobAction match {
case action: Action => dwsHttpClient.execute(action)
}
diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/request/GetTableStatisticInfoAction.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/request/GetTableStatisticInfoAction.scala
new file mode 100644
index 0000000..e924f03
--- /dev/null
+++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/request/GetTableStatisticInfoAction.scala
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.ujes.client.request
+
+import org.apache.linkis.httpclient.request.GetAction
+import org.apache.linkis.ujes.client.exception.UJESClientBuilderException
+import org.apache.commons.lang.StringUtils
+
+
+
+class GetTableStatisticInfoAction extends GetAction with UJESJobAction {
+ override def suffixURLs: Array[String] = Array("datasource", "getTableStatisticInfo")
+}
+
+object GetTableStatisticInfoAction {
+ def builder(): Builder = new Builder
+ class Builder private[GetTableStatisticInfoAction]() {
+ private var user: String = _
+ private var database: String = _
+ private var tableName: String = _
+ private var pageSize: Int = 100
+ private var partitionSort: String = "desc"
+
+ def setUser(user: String): Builder = {
+ this.user = user
+ this
+ }
+
+ def getUser(): String = user
+
+ def setDatabase(db: String): Builder = {
+ this.database = db
+ this
+ }
+
+ def setTable(table: String): Builder = {
+ this.tableName = table
+ this
+ }
+
+ def setPageSize(pageSize: Int): Builder = {
+ this.pageSize = pageSize
+ this
+ }
+
+ def setPartitionSort(order: String): Builder = {
+ this.partitionSort = order
+ this
+ }
+
+ def builder(): GetTableStatisticInfoAction = {
+ if (StringUtils.isBlank(user)) throw new UJESClientBuilderException("user is needed!")
+ if (StringUtils.isBlank(database)) throw new UJESClientBuilderException("database is needed!")
+ if (StringUtils.isBlank(tableName)) throw new UJESClientBuilderException("table is needed!")
+ val getTableStatisticInfoAction = new GetTableStatisticInfoAction
+ getTableStatisticInfoAction.setUser(user)
+ getTableStatisticInfoAction.setParameter("database", database)
+ getTableStatisticInfoAction.setParameter("tableName", tableName)
+ getTableStatisticInfoAction.setParameter("pageSize", pageSize)
+ getTableStatisticInfoAction.setParameter("partitionSort", partitionSort)
+ getTableStatisticInfoAction
+ }
+
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/request/JobExecuteAction.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/request/JobExecuteAction.scala
index fdf52a7..33aec6c 100644
--- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/request/JobExecuteAction.scala
+++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/request/JobExecuteAction.scala
@@ -216,7 +216,7 @@ object JobExecuteAction {
val PRESTO = new EngineType {
override val toString: String = "presto"
val PSQL = new RunType {
- override val toString: String = "presto"
+ override val toString: String = "psql"
}
override def getDefaultRunType: RunType = PSQL
}
diff --git a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-resource-manager-common/src/main/java/org/apache/linkis/resourcemanager/protocol/TimeoutEMEngineRequest.java b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/GetTableStatisticInfoResult.scala
similarity index 56%
copy from linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-resource-manager-common/src/main/java/org/apache/linkis/resourcemanager/protocol/TimeoutEMEngineRequest.java
copy to linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/GetTableStatisticInfoResult.scala
index c4ba1c4..6ffef19 100644
--- a/linkis-computation-governance/linkis-manager/linkis-manager-commons/linkis-resource-manager-common/src/main/java/org/apache/linkis/resourcemanager/protocol/TimeoutEMEngineRequest.java
+++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/GetTableStatisticInfoResult.scala
@@ -15,29 +15,27 @@
* limitations under the License.
*/
-package org.apache.linkis.resourcemanager.protocol;
+package org.apache.linkis.ujes.client.response
-import org.apache.linkis.manager.common.protocol.em.EMRequest;
+import org.apache.linkis.httpclient.dws.annotation.DWSHttpMessageResult
+import org.apache.linkis.httpclient.dws.response.DWSResult
+import org.apache.linkis.ujes.client.request.UserAction
-public class TimeoutEMEngineRequest implements EMRequest {
+import java.util
+import scala.beans.BeanProperty
- private String user;
- private String ticketId;
- @Override
- public String getUser() {
- return user;
- }
+@DWSHttpMessageResult("/api/rest_j/v\\d+/datasource/getTableStatisticInfo")
+class GetTableStatisticInfoResult extends DWSResult with UserAction {
- public void setUser(String user) {
- this.user = user;
- }
+ @BeanProperty
+ var pageNow : Int = _
+ @BeanProperty
+ var pageSize : Int = _
+ @BeanProperty
+ var tableStatisticInfo : util.Map[String, Any] = _
+ @BeanProperty
+ var totalSize : Int = _
- public String getTicketId() {
- return ticketId;
- }
-
- public void setTicketId(String ticketId) {
- this.ticketId = ticketId;
- }
}
+
diff --git a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/JobInfoResult.scala b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/JobInfoResult.scala
index 2760dc2..0f7f0d9 100644
--- a/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/JobInfoResult.scala
+++ b/linkis-computation-governance/linkis-client/linkis-computation-client/src/main/scala/org/apache/linkis/ujes/client/response/JobInfoResult.scala
@@ -36,6 +36,8 @@ class JobInfoResult extends DWSResult with UserAction with Status {
private var requestPersistTask: RequestPersistTask = _
private var resultSetList: Array[String] = _
+ private var strongerExecId: String = _
+
def setTask(task: util.Map[_, _]): Unit = {
this.task = task
requestPersistTask = new RequestPersistTask
@@ -44,6 +46,7 @@ class JobInfoResult extends DWSResult with UserAction with Status {
task.remove("createdTime")
task.remove("updatedTime")
task.remove("engineStartTime")
+ task.remove("labels")
Utils.tryCatch{
BeanUtils.populate(requestPersistTask, task.asInstanceOf[util.Map[String, _]])
}{
@@ -53,12 +56,20 @@ class JobInfoResult extends DWSResult with UserAction with Status {
requestPersistTask.setCreatedTime(new Date(createdTime))
requestPersistTask.setUpdatedTime(new Date(updatedTime))
requestPersistTask.setEngineStartTime(new Date(updatedTime))
+ if (task.containsKey("strongerExecId")) {
+ this.strongerExecId = task.get("strongerExecId").asInstanceOf[String]
+ }
+
}
def getTask = task
def getRequestPersistTask: RequestPersistTask = requestPersistTask
+ def getStrongerExecId: String = {
+ this.strongerExecId
+ }
+
def getResultSetList(ujesClient: UJESClient): Array[String] = {
if(isSucceed && resultSetList == null) synchronized {
if(resultSetList != null) return resultSetList
diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/JobRequest.java b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/JobRequest.java
index 648966c..9ed1f2a 100644
--- a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/JobRequest.java
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/JobRequest.java
@@ -46,6 +46,9 @@ public class JobRequest {
private Date createdTime;
private Date updatedTime;
private String instances;
+ /** result location */
+ private String resultLocation;
+
private Map<String, Object> metrics = new HashMap<>();
public Long getId() {
@@ -184,6 +187,14 @@ public class JobRequest {
this.metrics = metrics;
}
+ public String getResultLocation() {
+ return resultLocation;
+ }
+
+ public void setResultLocation(String resultLocation) {
+ this.resultLocation = resultLocation;
+ }
+
@Override
public String toString() {
return "JobRequest{"
diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/OnceExecutorContent.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/entity/job/OnceExecutorContent.scala
similarity index 99%
rename from linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/OnceExecutorContent.scala
rename to linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/entity/job/OnceExecutorContent.scala
index 0092e16..f28c6ac 100644
--- a/linkis-computation-governance/linkis-computation-governance-common/src/main/java/org/apache/linkis/governance/common/entity/job/OnceExecutorContent.scala
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/entity/job/OnceExecutorContent.scala
@@ -5,16 +5,16 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
package org.apache.linkis.governance.common.entity.job
import java.util
diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala
index ff637d7..d7ac1d6 100644
--- a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/paser/CodeParser.scala
@@ -260,6 +260,49 @@ class EmptyCodeParser extends SingleCodeParser {
}
+class JsonCodeParser extends SingleCodeParser {
+
+ override val codeType: CodeType = CodeType.JSON
+
+ override def parse(code: String): Array[String] = {
+ // parse json code
+ val codeBuffer = new ArrayBuffer[String]()
+ val statementBuffer = new ArrayBuffer[Char]()
+
+ var status = 0
+ var isBegin = false
+ code.trim.toCharArray().foreach {
+ case '{' => {
+ if (status == 0) {
+ if (isBegin && !statementBuffer.isEmpty) {
+ codeBuffer.append(new String(statementBuffer.toArray))
+ statementBuffer.clear()
+ } else {
+ isBegin = true
+ }
+ }
+ status -= 1
+ statementBuffer.append('{')
+ }
+ case '}' => {
+ status += 1
+ statementBuffer.append('}')
+ }
+ case char: Char => if (status == 0 && isBegin && !statementBuffer.isEmpty) {
+ codeBuffer.append(new String(statementBuffer.toArray))
+ statementBuffer.clear()
+ isBegin = false
+ } else {
+ statementBuffer.append(char)
+ }
+ }
+
+ if(statementBuffer.nonEmpty) codeBuffer.append(new String(statementBuffer.toArray))
+
+ codeBuffer.toArray
+ }
+
+}
object CodeType extends Enumeration {
type CodeType = Value
diff --git a/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala
new file mode 100644
index 0000000..8489732
--- /dev/null
+++ b/linkis-computation-governance/linkis-computation-governance-common/src/main/scala/org/apache/linkis/governance/common/utils/GovernanceUtils.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.governance.common.utils
+
+import org.apache.commons.lang.StringUtils
+import org.apache.linkis.common.conf.Configuration
+import org.apache.linkis.common.utils.{Logging, Utils}
+
+import java.io.File
+
+object GovernanceUtils extends Logging{
+
+
+ def killProcess(pid: String, desc: String, isSudo: Boolean): Unit = {
+ val subProcessKillScriptPath = Configuration.getLinkisHome() + "/sbin/kill-process-by-pid.sh"
+ if (StringUtils.isBlank(subProcessKillScriptPath) || !new File(subProcessKillScriptPath).exists()) {
+ logger.error(s"Failed to locate kill-script, $subProcessKillScriptPath not exist")
+ } else if (StringUtils.isNotBlank(pid)) {
+ val cmd = if (isSudo) {
+ Array("sudo", "sh", subProcessKillScriptPath, String.valueOf(pid))
+ } else {
+ Array("sh", subProcessKillScriptPath, String.valueOf(pid))
+ }
+ logger.info(s"Starting to kill sub-processes. desc: $desc Kill Command: " + cmd.mkString(" "))
+
+ Utils.tryCatch {
+ val output = Utils.exec(cmd, 600 * 1000L)
+ logger.info(s"Kill Success! desc: $desc. msg:\n ${output}")
+ } { t =>
+ logger.error(s"Kill error! desc: $desc.", t)
+ }
+ }
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-linux-launch/src/main/scala/org/apache/linkis/ecm/linux/launch/LinuxProcessEngineConnLaunch.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-linux-launch/src/main/scala/org/apache/linkis/ecm/linux/launch/LinuxProcessEngineConnLaunch.scala
index 1f68186..217f0b2 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-linux-launch/src/main/scala/org/apache/linkis/ecm/linux/launch/LinuxProcessEngineConnLaunch.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-linux-launch/src/main/scala/org/apache/linkis/ecm/linux/launch/LinuxProcessEngineConnLaunch.scala
@@ -17,6 +17,7 @@
package org.apache.linkis.ecm.linux.launch
+import org.apache.linkis.common.utils.Utils
import org.apache.linkis.ecm.core.launch.ProcessEngineConnLaunch
@@ -27,4 +28,21 @@ class LinuxProcessEngineConnLaunch extends ProcessEngineConnLaunch {
override protected def sudoCommand(user: String, command: String): Array[String] =
Array("sudo", "su", "-", user, "-c", command)
+
+ override def getPid(): Option[String] = {
+ Utils.tryCatch {
+ val clazz = Class.forName("java.lang.UNIXProcess");
+ val field = clazz.getDeclaredField("pid");
+ field.setAccessible(true);
+ val pid = field.get(getProcess()).asInstanceOf[Int]
+ if (pid >0) {
+ Some(pid.toString)
+ } else {
+ None
+ }
+ } { t =>
+ info("Failed to acquire pid for shell process", t)
+ None
+ }
+ }
}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/conf/ECMErrorCode.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/conf/ECMErrorCode.scala
index 2b1456c..760993f 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/conf/ECMErrorCode.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/conf/ECMErrorCode.scala
@@ -20,8 +20,14 @@ package org.apache.linkis.ecm.core.conf
object ECMErrorCode {
- val PROCESS_WAITFOR_ERROR = 20001
+ val EC_START_TIME_OUT = 11100
+
+ val EC_INTERRUPT_TIME_OUT = 11101
val EC_START_FAILED = 11102
+ val EC_FETCH_LOG_FAILED = 11110
+
+ val PROCESS_WAITFOR_ERROR = 20001
+
}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/EngineConnManagerEnv.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/EngineConnManagerEnv.scala
index 51fdd03..a1c2bfd 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/EngineConnManagerEnv.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/EngineConnManagerEnv.scala
@@ -17,6 +17,8 @@
package org.apache.linkis.ecm.core.launch
+import scala.collection.mutable
+
trait EngineConnManagerEnv {
@@ -26,7 +28,7 @@ trait EngineConnManagerEnv {
val engineConnTempDirs: String
val engineConnManagerHost: String
val engineConnManagerPort: String
- val linkDirs: Map[String, String]
+ val linkDirs: mutable.HashMap[String, String]
val properties: Map[String, String]
}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala
index e3432b7..7c12d2a 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-core/src/main/scala/org/apache/linkis/ecm/core/launch/ProcessEngineConnLaunch.scala
@@ -125,6 +125,14 @@ trait ProcessEngineConnLaunch extends EngineConnLaunch with Logging {
def getEngineConnPort: String = engineConnPort
+ protected def getProcess(): Process = this.process
+
+ /**
+ * Get the pid of the startup process
+ * @return
+ */
+ def getPid(): Option[String] = None
+
protected def getCommandArgs: Array[String] = {
if (request.creationDesc.properties.exists { case (k, v) => k.contains(" ") || (v != null && v.contains(" ")) })
throw new ErrorException(30000, "Startup parameters contain spaces!(启动参数中包含空格！)") //TODO exception
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/pom.xml b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/pom.xml
index 3ac89d9..4949f45 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/pom.xml
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/pom.xml
@@ -51,7 +51,7 @@
<dependency>
<groupId>org.apache.linkis</groupId>
- <artifactId>linkis-message-scheduler</artifactId>
+ <artifactId>linkis-rpc</artifactId>
<version>${linkis.version}</version>
<scope>provided</scope>
</dependency>
@@ -84,6 +84,12 @@
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>com.github.oshi</groupId>
+ <artifactId>oshi-core</artifactId>
+ <version>5.3.4</version>
+ </dependency>
+
</dependencies>
<build>
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/conf/ECMConfiguration.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/conf/ECMConfiguration.scala
index c39df6c..dae3d4b 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/conf/ECMConfiguration.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/conf/ECMConfiguration.scala
@@ -37,14 +37,17 @@ object ECMConfiguration {
val ECM_ASYNC_BUS_WAITTOEMPTY_TIME: Long = CommonVars("wds.linkis.ecm.async.bus.waittoempty.time", 5000L).getValue
//resource
- val ECM_MAX_MEMORY_AVAILABLE: Long = CommonVars[Long]("wds.linkis.ecm.memory.max", ByteTimeUtils.byteStringAsBytes("80g")).getValue
- val ECM_MAX_CORES_AVAILABLE: Int = CommonVars[Integer]("wds.linkis.ecm.cores.max", 50).getValue
+ val ECM_MAX_MEMORY_AVAILABLE: Long = CommonVars[Long]("wds.linkis.ecm.memory.max", ByteTimeUtils.byteStringAsBytes("100g")).getValue
+ val ECM_MAX_CORES_AVAILABLE: Int = CommonVars[Integer]("wds.linkis.ecm.cores.max", 100).getValue
val ECM_MAX_CREATE_INSTANCES: Int = CommonVars[Integer]("wds.linkis.ecm.engineconn.instances.max", 50).getValue
val ECM_PROTECTED_MEMORY: Long = CommonVars[Long]("wds.linkis.ecm.protected.memory", ByteTimeUtils.byteStringAsBytes("4g")).getValue
+ val ECM_PROTECTED_CPU_LOAD: Double = CommonVars[Double]("wds.linkis.ecm.protected.cpu.load", 0.98D).getValue
val ECM_PROTECTED_CORES: Int = CommonVars[Integer]("wds.linkis.ecm.protected.cores.max", 2).getValue
val ECM_PROTECTED_INSTANCES: Int = CommonVars[Integer]("wds.linkis.ecm.protected.engine.instances", 2).getValue
+ val ECM_PROTECTED_LOAD_ENABLED: Boolean = CommonVars[Boolean]("wds.linkis.ecm.protected.load.enabled", false).getValue
+
val MANAGER_SPRING_NAME: String = GovernanceCommonConf.MANAGER_SPRING_NAME.getValue
val ENGINE_CONN_MANAGER_SPRING_NAME: String = GovernanceCommonConf.ENGINE_CONN_MANAGER_SPRING_NAME.getValue
@@ -75,5 +78,6 @@ object ECMConfiguration {
val ENGINE_START_ERROR_MSG_MAX_LEN = CommonVars("wds.linkis.ecm.engine.start.error.msg.max.len", 500)
+ val ECM_PROCESS_SCRIPT_KILL: Boolean = CommonVars[Boolean]("wds.linkis.ecm.script.kill.engineconn", true).getValue
}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/hook/JarUDFLoadECMHook.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/hook/JarUDFLoadECMHook.scala
index aedabe8..86e5fbe 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/hook/JarUDFLoadECMHook.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/hook/JarUDFLoadECMHook.scala
@@ -16,23 +16,82 @@
*/
package org.apache.linkis.ecm.server.hook
-import org.apache.linkis.common.utils.Logging
+import org.apache.linkis.common.utils.{Logging, Utils}
import org.apache.linkis.ecm.core.engineconn.EngineConn
import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest
-import org.apache.linkis.manager.engineplugin.common.launch.process.{LaunchConstants, ProcessEngineConnLaunchRequest}
+import org.apache.linkis.manager.engineplugin.common.launch.process.{Environment, LaunchConstants, ProcessEngineConnLaunchRequest}
import org.apache.linkis.udf.UDFClient
import org.apache.commons.lang.StringUtils
+import org.apache.linkis.common.io.FsPath
+import org.apache.linkis.ecm.server.service.LocalDirsHandleService
+import org.apache.linkis.ecm.server.service.impl.DefaultLocalDirsHandleService
+import org.apache.linkis.ecm.server.util.ECMUtils
+import org.apache.linkis.manager.common.protocol.bml.BmlResource
+import org.apache.linkis.manager.label.utils.LabelUtil
+import org.apache.linkis.storage.FSFactory
+import org.apache.linkis.storage.fs.FileSystem
+import org.apache.linkis.storage.utils.{FileSystemUtils, StorageUtils}
+
+import java.io.File
+import java.nio.file.Paths
+import scala.collection.mutable
class JarUDFLoadECMHook extends ECMHook with Logging {
+ private var localDirsHandleService: LocalDirsHandleService = new DefaultLocalDirsHandleService
+
+ private val fs: FileSystem = FSFactory.getFs(StorageUtils.FILE).asInstanceOf[FileSystem]
+
+ fs.init(null)
+
override def beforeLaunch(request: EngineConnLaunchRequest, conn: EngineConn): Unit = {
request match {
case pel: ProcessEngineConnLaunchRequest =>
info("start loading UDFs")
- val udfInfos = UDFClient.getUdfInfos(request.user,"udf").filter{ info => info.getUdfType == 0 && info.getExpire == false && StringUtils.isNotBlank(info.getPath) && info.getLoad == true }
- udfInfos.foreach{ udfInfo =>
- LaunchConstants.addPathToUDFPath(pel.environment, udfInfo.getPath)
+ val user = pel.user
+ val ticketId = pel.ticketId
+ val engineType = LabelUtil.getEngineType(pel.labels)
+ val workDir = localDirsHandleService.getEngineConnWorkDir(user, ticketId, engineType)
+ val pubDir = localDirsHandleService.getEngineConnPublicDir
+ val udfDir = workDir + File.separator + "udf"
+ val fsPath = new FsPath(StorageUtils.FILE_SCHEMA + udfDir)
+ if (!fs.exists(fsPath)) {
+ FileSystemUtils.mkdirs(fs, fsPath, Utils.getJvmUser)
+ fs.setPermission(fsPath, "rwxrwxrwx")
+ }
+
+ val udfInfos = UDFClient.getJarUdf(pel.user)
+ val fileNameSet: mutable.HashSet[String] = new mutable.HashSet[String]()
+ import util.control.Breaks._
+ udfInfos.foreach { udfInfo =>
+ val resourceId = udfInfo.getBmlResourceId
+ val version = udfInfo.getBmlResourceVersion
+ val bmlResourceDir = StorageUtils.FILE_SCHEMA + Paths.get(pubDir, resourceId, version).toFile.getPath
+ val fsPath = new FsPath(bmlResourceDir)
+ val bmlResource: BmlResource = new BmlResource
+ bmlResource.setResourceId(resourceId)
+ bmlResource.setVersion(version)
+ // 文件名用path最后一截
+ val fileName = udfInfo.getPath.substring(udfInfo.getPath.lastIndexOf("/") + 1)
+ bmlResource.setFileName(fileName)
+ breakable {
+ if (StringUtils.isEmpty(fileName) || fileNameSet.contains(fileName)) {
+ break()
+ }
+ fileNameSet += bmlResource.getFileName
+ if (!fs.exists(fsPath)) resourceId.intern().synchronized {
+ if (!fs.exists(fsPath)) {
+ ECMUtils.downLoadBmlResourceToLocal(bmlResource, if (udfInfo.getCreateUser.equals("bdp")) "hadoop" else udfInfo.getCreateUser, fsPath.getPath)(fs)
+ info(s"Finished to download bml resource ${fsPath.getPath}")
+ }
+ }
+ conn.getEngineConnLaunchRunner.getEngineConnLaunch.getEngineConnManagerEnv().linkDirs.
+ put(fsPath.getPath + File.separator + bmlResource.getFileName, udfDir + File.separator + bmlResource.getFileName)
+ }
}
+ // LaunchConstants.addPathToClassPath(pel.environment, udfDir + File.separator + "*")
+ val udfJars = fileNameSet.map(udfDir + File.separator + _)
+ pel.environment.put(Environment.UDF_JARS.toString, udfJars.mkString(","));
}
}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala
index bd1e3c6..c446abc 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/operator/EngineConnLogOperator.scala
@@ -20,16 +20,16 @@ package org.apache.linkis.ecm.server.operator
import java.io.{File, RandomAccessFile}
import java.util
-
import org.apache.linkis.DataWorkCloudApplication
import org.apache.linkis.common.conf.CommonVars
import org.apache.linkis.common.utils.{Logging, Utils}
-import org.apache.linkis.ecm.server.exception.{ECMErrorCode, ECMErrorException}
+import org.apache.linkis.ecm.server.exception.ECMErrorException
import org.apache.linkis.ecm.server.service.{EngineConnListService, LocalDirsHandleService}
import org.apache.linkis.manager.common.operator.Operator
import org.apache.linkis.manager.common.protocol.em.ECMOperateRequest
import org.apache.commons.io.IOUtils
import org.apache.commons.lang.StringUtils
+import org.apache.linkis.ecm.core.conf.ECMErrorCode
import scala.collection.JavaConverters.asScalaBufferConverter
import scala.util.matching.Regex
@@ -130,12 +130,12 @@ class EngineConnLogOperator extends Operator with Logging {
.getOrElse {
val creator = getAsThrow[String]("creator")
val engineConnType = getAsThrow[String]("engineConnType")
- localDirsHandleService.getEngineConnLogDir(creator, ticketId)
+ localDirsHandleService.getEngineConnLogDir(creator, ticketId, engineConnType)
}
(logDir, ticketId)
}
- val logPath = new File(engineConnLogDir, EngineConnLogOperator.LOG_FILE_NAME.getValue)
- if(!logPath.exists() || !logPath.isFile) {
+ val logPath = new File(engineConnLogDir, getAs("logType", EngineConnLogOperator.LOG_FILE_NAME.getValue));
+ if (!logPath.exists() || !logPath.isFile) {
throw new ECMErrorException(ECMErrorCode.EC_FETCH_LOG_FAILED, s"LogFile $logPath is not exists or is not a file.")
}
info(s"Try to fetch EngineConn(id: $ticketId, instance: $engineConnInstance) logs from ${logPath.getPath}.")
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/ECMHealthService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/ECMHealthService.scala
index e713561..b596756 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/ECMHealthService.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/ECMHealthService.scala
@@ -18,7 +18,8 @@
package org.apache.linkis.ecm.server.service
import org.apache.linkis.ecm.core.report.ECMHealthReport
-import org.apache.linkis.manager.common.protocol.node.{NodeHeartbeatMsg, NodeHeartbeatRequest}
+import org.apache.linkis.manager.common.entity.enumeration.{NodeHealthy, NodeStatus}
+import org.apache.linkis.manager.common.protocol.node.{NodeHealthyRequest, NodeHeartbeatMsg, NodeHeartbeatRequest}
trait ECMHealthService{
@@ -27,8 +28,20 @@ trait ECMHealthService{
def reportHealth(report: ECMHealthReport): Unit
- def generateHealthReport(reportTime:Long): ECMHealthReport
+ def generateHealthReport(reportTime: Long): ECMHealthReport
def dealNodeHeartbeatRequest(nodeHeartbeatRequest: NodeHeartbeatRequest): NodeHeartbeatMsg
+ def dealNodeHealthyRequest(nodeHealthyRequest: NodeHealthyRequest): Unit
+
+ def getNodeStatus: NodeStatus
+
+ def getNodeHealthy: NodeHealthy
+
+ def isSetByManager: Boolean
+
+ def transitionStatus(toStatus: NodeStatus): Unit
+
+ def transitionHealthy(toHealthy: NodeHealthy): Unit
+
}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/LocalDirsHandleService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/LocalDirsHandleService.scala
index d686c7c..f4cbded 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/LocalDirsHandleService.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/LocalDirsHandleService.scala
@@ -22,13 +22,13 @@ trait LocalDirsHandleService {
def getEngineConnManagerHomeDir: String
- def getEngineConnWorkDir(user: String, ticketId: String): String
+ def getEngineConnWorkDir(user: String, ticketId: String, engineType: String): String
def getEngineConnPublicDir: String
- def getEngineConnLogDir(user: String, ticketId: String): String
+ def getEngineConnLogDir(user: String, ticketId: String, engineType: String): String
- def getEngineConnTmpDir(user: String, ticketId: String): String
+ def getEngineConnTmpDir(user: String, ticketId: String, engineType: String): String
def cleanup(): Unit
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala
index 79a8667..cc70c45 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/AbstractEngineConnLaunchService.scala
@@ -92,28 +92,27 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w
conn.setServiceInstance(serviceInstance)
case _ =>
}
+ afterLaunch(request, conn, duration)
+
val future = Future {
- afterLaunch(request, conn, duration)
+ logger.info(s"wait engineConn ${conn.getServiceInstance} start")
+ waitEngineConnStart(request, conn, duration)
}
future onComplete {
case Failure(t) =>
- throw t
+ logger.error(s"init ${conn.getServiceInstance} failed.${conn.getEngineConnLaunchRunner.getEngineConnLaunch.getEngineConnManagerEnv().engineConnWorkDir}")
case Success(_) =>
- info(s"init ${conn.getServiceInstance} succeed.")
+ logger.info(s"init ${conn.getServiceInstance} succeed.${conn.getEngineConnLaunchRunner.getEngineConnLaunch.getEngineConnManagerEnv().engineConnWorkDir}")
}
- //超时忽略，如果状态翻转了则直接返回
- Utils.tryQuietly(Await.result(future, Duration(WAIT_ENGINECONN_PID.getValue.toLong, TimeUnit.MILLISECONDS)))
- }{
- //failed，1.被ms打断，2.超时，3.普通错误，比如process
- t: Throwable =>
- error(s"init ${conn.getServiceInstance} failed, now stop and delete it. message: ${t.getMessage}")
- conn.getEngineConnLaunchRunner.stop()
- Sender.getSender(MANAGER_SPRING_NAME).send(EngineConnStatusCallbackToAM(conn.getServiceInstance,
- NodeStatus.ShuttingDown, " wait init failed , reason " + ExceptionUtils.getRootCauseMessage(t)))
- LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll(EngineConnStatusChangeEvent(conn.getTickedId, Failed))
- throw t
+ } { t =>
+ error(s"init ${conn.getServiceInstance} failed, ${conn.getEngineConnLaunchRunner.getEngineConnLaunch.getEngineConnManagerEnv().engineConnWorkDir}, now stop and delete it. message: ${t.getMessage}", t)
+ conn.getEngineConnLaunchRunner.stop()
+ Sender.getSender(MANAGER_SPRING_NAME).send(EngineConnStatusCallbackToAM(conn.getServiceInstance,
+ NodeStatus.ShuttingDown, " wait init failed , reason " + ExceptionUtils.getRootCauseMessage(t)))
+ LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll(EngineConnStatusChangeEvent(conn.getTickedId, Failed))
+ throw t
}
val engineNode = new AMEngineNode()
engineNode.setLabels(conn.getLabels)
@@ -124,6 +123,8 @@ abstract class AbstractEngineConnLaunchService extends EngineConnLaunchService w
engineNode
}
+ def waitEngineConnStart(request: EngineConnLaunchRequest, conn: EngineConn, duration: Long): Unit
+
def createEngineConn: EngineConn = new DefaultEngineConn
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/BmlResourceLocalizationService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/BmlResourceLocalizationService.scala
index 485857b..f61aec4 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/BmlResourceLocalizationService.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/BmlResourceLocalizationService.scala
@@ -19,7 +19,6 @@ package org.apache.linkis.ecm.server.service.impl
import java.io.File
import java.nio.file.Paths
-
import org.apache.linkis.DataWorkCloudApplication
import org.apache.linkis.common.conf.Configuration
import org.apache.linkis.common.io.FsPath
@@ -34,6 +33,7 @@ import org.apache.linkis.ecm.server.util.ECMUtils
import org.apache.linkis.manager.common.protocol.bml.BmlResource
import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLaunchRequest
import org.apache.linkis.manager.engineplugin.common.launch.process.ProcessEngineConnLaunchRequest
+import org.apache.linkis.manager.label.utils.LabelUtil
import org.apache.linkis.storage.FSFactory
import org.apache.linkis.storage.fs.FileSystem
import org.apache.linkis.storage.utils.{FileSystemUtils, StorageUtils}
@@ -65,26 +65,20 @@ class BmlResourceLocalizationService extends ResourceLocalizationService with Lo
val linkDirsP = new mutable.HashMap[String, String]
val user = request.user
val ticketId = request.ticketId
- val workDir = createDirIfNotExit(localDirsHandleService.getEngineConnWorkDir(user, ticketId))
+ val engineType = LabelUtil.getEngineType(request.labels)
+ val workDir = createDirIfNotExit(localDirsHandleService.getEngineConnWorkDir(user, ticketId, engineType))
val emHomeDir = createDirIfNotExit(localDirsHandleService.getEngineConnManagerHomeDir)
- val logDirs = createDirIfNotExit(localDirsHandleService.getEngineConnLogDir(user, ticketId))
- val tmpDirs = createDirIfNotExit(localDirsHandleService.getEngineConnTmpDir(user, ticketId))
+ val logDirs = createDirIfNotExit(localDirsHandleService.getEngineConnLogDir(user, ticketId, engineType))
+ val tmpDirs = createDirIfNotExit(localDirsHandleService.getEngineConnTmpDir(user, ticketId, engineType))
files.foreach(downloadBmlResource(request, linkDirsP, _, workDir))
engineConn.getEngineConnLaunchRunner.getEngineConnLaunch.setEngineConnManagerEnv(new EngineConnManagerEnv {
override val engineConnManagerHomeDir: String = emHomeDir
override val engineConnWorkDir: String = workDir
override val engineConnLogDirs: String = logDirs
override val engineConnTempDirs: String = tmpDirs
-
- var hostName = Utils.getComputerName
- val eurekaPreferIp = Configuration.EUREKA_PREFER_IP
- if(eurekaPreferIp){
- hostName = DataWorkCloudApplication.getApplicationContext.getEnvironment().getProperty("spring.cloud.client.ip-address")
- }
- override val engineConnManagerHost: String = hostName
+ override val engineConnManagerHost: String = Utils.getComputerName
override val engineConnManagerPort: String = DataWorkCloudApplication.getApplicationContext.getEnvironment.getProperty("server.port")
- override val linkDirs: Map[String, String] = linkDirsP.toMap
- // TODO: 注册发现信息的配置化
+ override val linkDirs: mutable.HashMap[String, String] = linkDirsP
override val properties: Map[String, String] = Map("eureka.client.serviceUrl.defaultZone" -> ECM_EUREKA_DEFAULTZONE)
})
case _ =>
@@ -112,17 +106,36 @@ class BmlResourceLocalizationService extends ResourceLocalizationService with Lo
val publicDir = localDirsHandleService.getEngineConnPublicDir
val bmlResourceDir = schema + Paths.get(publicDir, resourceId, version).toFile.getPath
val fsPath = new FsPath(bmlResourceDir)
- if (!fs.exists(fsPath)) {
- ECMUtils.downLoadBmlResourceToLocal(resource, user, fsPath.getPath)
- val unzipDir = fsPath.getSchemaPath + File.separator + resource.getFileName.substring(0, resource.getFileName.lastIndexOf("."))
- FileSystemUtils.mkdirs(fs, new FsPath(unzipDir), Utils.getJvmUser)
- ZipUtils.unzip(bmlResourceDir + File.separator + resource.getFileName, unzipDir)
- fs.delete(new FsPath(bmlResourceDir + File.separator + resource.getFileName))
+
+ /**
+ * Prevent concurrent use of resourceId as a lock object
+ */
+ if (!fs.exists(fsPath)) resourceId.intern().synchronized {
+ if (!fs.exists(fsPath)) {
+ ECMUtils.downLoadBmlResourceToLocal(resource, user, fsPath.getPath)
+ val unzipDir = fsPath.getSchemaPath + File.separator + resource.getFileName.substring(0, resource.getFileName.lastIndexOf("."))
+ FileSystemUtils.mkdirs(fs, new FsPath(unzipDir), Utils.getJvmUser)
+ val path = bmlResourceDir + File.separator + resource.getFileName
+ Utils.tryCatch(ZipUtils.unzip(path, unzipDir)) {
+ t: Throwable =>
+ logger.error("Failed to unzip path", t)
+ Utils.tryAndWarn(fs.delete(fsPath))
+ }
+ logger.info(s"Finished to download bml resource ${fsPath.getPath}")
+ fs.delete(new FsPath(bmlResourceDir + File.separator + resource.getFileName))
+ }
}
//2.软连，并且添加到map
val dirAndFileList = fs.listPathWithError(fsPath)
- dirAndFileList.getFsPaths.foreach {
- case path: FsPath =>
+ var paths = dirAndFileList.getFsPaths.toList
+ if (paths.exists(_.getPath.endsWith(".zip"))) {
+ logger.info(s"Start to wait fs path to init ${fsPath.getPath}")
+ resourceId.intern().synchronized {
+ logger.info(s"Finished to wait fs path to init ${fsPath.getPath} ")
+ }
+ paths = fs.listPathWithError(fsPath).getFsPaths.toList
+ }
+ paths.foreach { path =>
val name = new File(path.getPath).getName
linkDirs.put(path.getPath, workDir + seperator + name)
}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMHealthService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMHealthService.scala
index ea83e97..6ddb94d 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMHealthService.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultECMHealthService.scala
@@ -19,33 +19,52 @@ package org.apache.linkis.ecm.server.service.impl
import java.util.Date
import java.util.concurrent.TimeUnit
-
-import org.apache.linkis.common.utils.Utils
+import org.apache.linkis.common.utils.{ByteTimeUtils, OverloadUtils, Utils}
import org.apache.linkis.ecm.core.listener.{ECMEvent, ECMEventListener}
import org.apache.linkis.ecm.core.report.ECMHealthReport
import org.apache.linkis.ecm.server.LinkisECMApplication
+import org.apache.linkis.ecm.server.conf.ECMConfiguration
import org.apache.linkis.ecm.server.conf.ECMConfiguration._
import org.apache.linkis.ecm.server.listener.{ECMClosedEvent, ECMReadyEvent}
import org.apache.linkis.ecm.server.report.DefaultECMHealthReport
import org.apache.linkis.ecm.server.service.{ECMHealthService, EngineConnListService}
+import org.apache.linkis.ecm.server.util.HardwareUtils
import org.apache.linkis.manager.common.entity.enumeration.{NodeHealthy, NodeStatus}
import org.apache.linkis.manager.common.entity.metrics.{NodeHealthyInfo, NodeOverLoadInfo}
import org.apache.linkis.manager.common.entity.resource.{CommonNodeResource, LoadInstanceResource}
-import org.apache.linkis.manager.common.protocol.node.{NodeHeartbeatMsg, NodeHeartbeatRequest}
-import org.apache.linkis.message.annotation.Receiver
+import org.apache.linkis.manager.common.protocol.node.{NodeHealthyRequest, NodeHeartbeatMsg, NodeHeartbeatRequest, NodeHeartbeatResourceMsg}
+import org.apache.linkis.rpc.serializer.annotation.Receiver
import org.apache.linkis.rpc.Sender
import org.springframework.beans.factory.annotation.Autowired
class DefaultECMHealthService extends ECMHealthService with ECMEventListener {
- val maxResource = new LoadInstanceResource(ECM_MAX_MEMORY_AVAILABLE, ECM_MAX_CORES_AVAILABLE, ECM_MAX_CREATE_INSTANCES)
- val minResource = new LoadInstanceResource(ECM_PROTECTED_MEMORY, ECM_PROTECTED_CORES, ECM_PROTECTED_INSTANCES)
+ private val maxResource = new LoadInstanceResource(ECM_MAX_MEMORY_AVAILABLE, ECM_MAX_CORES_AVAILABLE, ECM_MAX_CREATE_INSTANCES)
+
+ private val minResource = new LoadInstanceResource(ECM_PROTECTED_MEMORY, ECM_PROTECTED_CORES, ECM_PROTECTED_INSTANCES)
+
private val runtime: Runtime = Runtime.getRuntime
+ private var status: NodeStatus = NodeStatus.Starting
+
+ private var healthy: NodeHealthy = NodeHealthy.Healthy
+
+ private var setByManager: Boolean = false
+
+ private var lastCpuLoad: Double = 0D
+
+ private var lastFreeMemory: Long = ECM_MAX_MEMORY_AVAILABLE
+
+ private val statusLocker = new Object()
+
+ private val healthyLocker = new Object()
+
+
private val future = Utils.defaultScheduler.scheduleAtFixedRate(new Runnable {
- override def run(): Unit =Utils.tryAndWarn{
+ override def run(): Unit = Utils.tryAndWarn{
if (LinkisECMApplication.isReady) {
+ checkMachinePerformance()
reportHealth(getLastEMHealthReport)
}
}
@@ -57,7 +76,7 @@ class DefaultECMHealthService extends ECMHealthService with ECMEventListener {
override def getLastEMHealthReport: ECMHealthReport = {
val report = new DefaultECMHealthReport
report.setNodeId(LinkisECMApplication.getECMServiceInstance.toString)
- report.setNodeStatus(NodeStatus.Running)
+ report.setNodeStatus(getNodeStatus)
//todo report right metrics
report.setTotalResource(maxResource)
report.setProtectedResource(minResource)
@@ -65,18 +84,27 @@ class DefaultECMHealthService extends ECMHealthService with ECMEventListener {
report.setReportTime(new Date().getTime)
report.setRunningEngineConns(LinkisECMApplication.getContext.getECMMetrics.getRunningEngineConns)
val info = new NodeOverLoadInfo
- info.setMaxMemory(runtime.maxMemory())
- info.setUsedMemory(runtime.totalMemory() - runtime.freeMemory())
- // TODO: 根据系统获取当前操作系统负载
+ val (max, free) = HardwareUtils.getTotalAndAvailableMemory()
+ info.setMaxMemory(max)
+ info.setSystemLeftMemory(free)
+ info.setUsedMemory(max - free)
report.setOverload(info)
report
}
- // TODO: 可能还需要个判断health状态的方法
+
override def reportHealth(report: ECMHealthReport): Unit = {
val heartbeat: NodeHeartbeatMsg = transferECMHealthReportToNodeHeartbeatMsg(report)
+ val resourceHeartbeat: NodeHeartbeatResourceMsg = new NodeHeartbeatResourceMsg
+ resourceHeartbeat.setHealthyInfo(heartbeat.getHealthyInfo)
+ resourceHeartbeat.setHeartBeatMsg(heartbeat.getHeartBeatMsg)
+ resourceHeartbeat.setNodeResource(heartbeat.getNodeResource)
+ resourceHeartbeat.setOverLoadInfo(heartbeat.getOverLoadInfo)
+ resourceHeartbeat.setServiceInstance(heartbeat.getServiceInstance)
+ resourceHeartbeat.setStatus(heartbeat.getStatus)
Sender.getSender(MANAGER_SPRING_NAME).send(heartbeat)
+ Sender.getSender(MANAGER_SPRING_NAME).send(resourceHeartbeat)
}
private def transferECMHealthReportToNodeHeartbeatMsg(report: ECMHealthReport) = {
@@ -93,7 +121,7 @@ class DefaultECMHealthService extends ECMHealthService with ECMEventListener {
heartbeat.setHeartBeatMsg("")
val nodeHealthyInfo = new NodeHealthyInfo
nodeHealthyInfo.setMsg("")
- nodeHealthyInfo.setNodeHealthy(NodeHealthy.Healthy)
+ nodeHealthyInfo.setNodeHealthy(getNodeHealthy)
heartbeat.setHealthyInfo(nodeHealthyInfo)
heartbeat
}
@@ -104,26 +132,29 @@ class DefaultECMHealthService extends ECMHealthService with ECMEventListener {
}
private def emShutdownHealthReport(event: ECMClosedEvent): Unit = {
+ transitionStatus(NodeStatus.ShuttingDown)
val report = getLastEMHealthReport
- report.setNodeStatus(NodeStatus.ShuttingDown)
reportHealth(report)
}
- private def emReadyHealthReport(event: ECMReadyEvent): Unit = reportHealth(getLastEMHealthReport)
+ private def emReadyHealthReport(event: ECMReadyEvent): Unit = {
+ transitionStatus(NodeStatus.Running)
+ reportHealth(getLastEMHealthReport)
+ }
override def onEvent(event: ECMEvent): Unit = event match {
case event: ECMReadyEvent =>
emReadyHealthReport(event)
case event: ECMClosedEvent =>
- cancelHealthReportThread(event)
emShutdownHealthReport(event)
presistenceLeftReports(event)
+ cancelHealthReportThread(event)
case _ =>
}
private def cancelHealthReportThread(event: ECMClosedEvent): Unit = {
-
+ Utils.tryAndWarn(future.cancel(true))
}
private def presistenceLeftReports(event: ECMClosedEvent): Unit = {
@@ -132,9 +163,76 @@ class DefaultECMHealthService extends ECMHealthService with ECMEventListener {
@Receiver
override def dealNodeHeartbeatRequest(nodeHeartbeatRequest: NodeHeartbeatRequest): NodeHeartbeatMsg = {
- val hearlthReport = getLastEMHealthReport
- transferECMHealthReportToNodeHeartbeatMsg(hearlthReport)
+ val healthReport = getLastEMHealthReport
+ transferECMHealthReportToNodeHeartbeatMsg(healthReport)
}
+ override def getNodeStatus: NodeStatus = {
+ this.status
+ }
+ override def getNodeHealthy: NodeHealthy = {
+ this.healthy
+ }
+
+ override def isSetByManager: Boolean = {
+ this.setByManager
+ }
+
+ override def transitionStatus(toStatus: NodeStatus): Unit = statusLocker synchronized {
+ this.status match {
+ case NodeStatus.Failed | NodeStatus.Success =>
+ logger.warn(s"$toString attempt to change status ${this.status} => $toStatus, ignore it .")
+ return
+ case NodeStatus.ShuttingDown =>
+ toStatus match {
+ case NodeStatus.Failed | NodeStatus.Success =>
+ case _ =>
+ logger.warn(s"$toString attempt to change a Executor from ShuttingDown to $toStatus, ignore it.")
+ return
+ }
+ case _ =>
+
+ }
+ logger.info(s"$toString changed status $status => $toStatus.")
+ this.status = toStatus
+ }
+
+ override def transitionHealthy(toHealthy: NodeHealthy): Unit = healthyLocker synchronized {
+ logger.info(s"nodeHealthy from ${healthy} to ${toHealthy}")
+ this.healthy = toHealthy
+ }
+
+ @Receiver
+ override def dealNodeHealthyRequest(nodeHealthyRequest: NodeHealthyRequest): Unit = {
+ if (NodeHealthy.isAvailable(nodeHealthyRequest.getNodeHealthy)) {
+ this.setByManager = false
+ transitionHealthy(nodeHealthyRequest.getNodeHealthy)
+ } else {
+ this.setByManager = true
+ transitionHealthy(nodeHealthyRequest.getNodeHealthy)
+ }
+ }
+
+ private def checkMachinePerformance(): Unit = {
+ if (!ECMConfiguration.ECM_PROTECTED_LOAD_ENABLED) {
+ return
+ }
+ val cpuLoad = OverloadUtils.getOSBean.getSystemCpuLoad
+ val freeMemory = HardwareUtils.getAvailableMemory()
+ if ((cpuLoad > ECM_PROTECTED_CPU_LOAD && lastCpuLoad > ECM_PROTECTED_CPU_LOAD)
+ || (freeMemory < ECM_PROTECTED_MEMORY && lastFreeMemory < ECM_PROTECTED_MEMORY)) {
+ warn(s"cpuLoad(${cpuLoad}) and freeMemory(${ByteTimeUtils.bytesToString(freeMemory)}) overload prepare to mark ecm to StockAvailable")
+ if (NodeHealthy.isAvailable(getNodeHealthy)) {
+ transitionHealthy(NodeHealthy.StockAvailable)
+ }
+ } else {
+ if (!NodeHealthy.isAvailable(getNodeHealthy) && !isSetByManager) {
+ warn(s"cpuLoad(${cpuLoad}) and freeMemory(${ByteTimeUtils.bytesToString(freeMemory)}) recover prepare to mark ecm to Healthy")
+ transitionHealthy(NodeHealthy.Healthy)
+ }
+ }
+ lastCpuLoad = cpuLoad
+ lastFreeMemory = freeMemory
}
+}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java
index 0e38151..67dd060 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnKillService.java
@@ -20,12 +20,14 @@ package org.apache.linkis.ecm.server.service.impl;
import org.apache.linkis.common.ServiceInstance;
import org.apache.linkis.common.utils.Utils;
import org.apache.linkis.ecm.core.engineconn.EngineConn;
+import org.apache.linkis.ecm.server.conf.ECMConfiguration;
import org.apache.linkis.ecm.server.service.EngineConnKillService;
import org.apache.linkis.ecm.server.service.EngineConnListService;
+import org.apache.linkis.governance.common.utils.GovernanceUtils;
import org.apache.linkis.manager.common.protocol.engine.EngineStopRequest;
import org.apache.linkis.manager.common.protocol.engine.EngineStopResponse;
import org.apache.linkis.manager.common.protocol.engine.EngineSuicideRequest;
-import org.apache.linkis.message.annotation.Receiver;
+import org.apache.linkis.rpc.serializer.annotation.Receiver;
import org.apache.linkis.rpc.Sender;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
@@ -48,6 +50,7 @@ public class DefaultEngineConnKillService implements EngineConnKillService {
@Override
@Receiver
public EngineStopResponse dealEngineConnStop(EngineStopRequest engineStopRequest) {
+ logger.info("received EngineStopRequest " + engineStopRequest);
EngineConn engineConn = getEngineConnByServiceInstance(engineStopRequest.getServiceInstance());
EngineStopResponse response = new EngineStopResponse();
if (null != engineConn) {
@@ -92,22 +95,10 @@ public class DefaultEngineConnKillService implements EngineConnKillService {
private boolean killEngineConnByPid(EngineConn engineConn) {
logger.info("try to kill {} toString with pid({}).", engineConn.getServiceInstance().toString(), engineConn.getPid());
if (StringUtils.isNotBlank(engineConn.getPid())) {
- String k15cmd = "sudo kill " + engineConn.getPid();
- String k9cmd = "sudo kill -9 " + engineConn.getPid();
- int tryNum = 0;
- try {
- while (isProcessAlive(engineConn.getPid()) && tryNum <= 3) {
- logger.info("{} still alive with pid({}), use shell command to kill it. try {}++", engineConn.getServiceInstance().toString(), engineConn.getPid(), tryNum++);
- if (tryNum <= 3) {
- Utils.exec(k15cmd.split(" "), 3000L);
- } else {
- logger.info("{} still alive with pid({}). try {}, use shell command to kill -9 it", engineConn.getServiceInstance().toString(), engineConn.getPid(), tryNum);
- Utils.exec(k9cmd.split(" "), 3000L);
- }
- Thread.sleep(5000);
- }
- } catch (InterruptedException e) {
- logger.error("Interrupted while killing engine {} with pid({})." + engineConn.getServiceInstance().toString(), engineConn.getPid());
+ if (ECMConfiguration.ECM_PROCESS_SCRIPT_KILL()) {
+ GovernanceUtils.killProcess(engineConn.getPid(), engineConn.getServiceInstance().toString(), true);
+ } else {
+ killProcessByKillCmd(engineConn.getPid(), engineConn.getServiceInstance().toString());
}
if (isProcessAlive(engineConn.getPid())) {
return false;
@@ -136,4 +127,23 @@ public class DefaultEngineConnKillService implements EngineConnKillService {
}
}
+ private void killProcessByKillCmd(String pid, String desc ) {
+ String k15cmd = "sudo kill " + pid;
+ String k9cmd = "sudo kill -9 " + pid;
+ int tryNum = 0;
+ try {
+ while (isProcessAlive(pid) && tryNum <= 3) {
+ logger.info("{} still alive with pid({}), use shell command to kill it. try {}++", desc, pid, tryNum++);
+ if (tryNum <= 3) {
+ Utils.exec(k15cmd.split(" "), 3000L);
+ } else {
+ logger.info("{} still alive with pid({}). try {}, use shell command to kill -9 it", desc, pid, tryNum);
+ Utils.exec(k9cmd.split(" "), 3000L);
+ }
+ Thread.sleep(5000);
+ }
+ } catch (InterruptedException e) {
+ logger.error("Interrupted while killing engine {} with pid({})." + desc, pid);
+ }
+ }
}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnPidCallbackService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnPidCallbackService.scala
index f54fc98..ba64d31 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnPidCallbackService.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnPidCallbackService.scala
@@ -22,7 +22,7 @@ import org.apache.linkis.ecm.server.LinkisECMApplication
import org.apache.linkis.ecm.server.listener.EngineConnPidCallbackEvent
import org.apache.linkis.ecm.server.service.EngineConnPidCallbackService
import org.apache.linkis.governance.common.protocol.task.ResponseEngineConnPid
-import org.apache.linkis.message.annotation.Receiver
+import org.apache.linkis.rpc.serializer.annotation.Receiver
class DefaultEngineConnPidCallbackService extends EngineConnPidCallbackService with Logging {
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnStatusCallbackService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnStatusCallbackService.scala
index 10818d6..b19795c 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnStatusCallbackService.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultEngineConnStatusCallbackService.scala
@@ -25,7 +25,7 @@ import org.apache.linkis.ecm.server.service.EngineConnStatusCallbackService
import org.apache.linkis.manager.common.entity.enumeration.NodeStatus
import org.apache.linkis.manager.common.entity.enumeration.NodeStatus.{Failed, Running}
import org.apache.linkis.manager.common.protocol.engine.{EngineConnStatusCallback, EngineConnStatusCallbackToAM}
-import org.apache.linkis.message.annotation.Receiver
+import org.apache.linkis.rpc.serializer.annotation.Receiver
import org.apache.linkis.rpc.Sender
import org.springframework.stereotype.Service
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultLocalDirsHandleService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultLocalDirsHandleService.scala
index 7b6a1d7..5f0c846 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultLocalDirsHandleService.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultLocalDirsHandleService.scala
@@ -23,24 +23,33 @@ import java.nio.file.Paths
import org.apache.linkis.common.io.FsPath
import org.apache.linkis.ecm.server.conf.ECMConfiguration._
import org.apache.linkis.ecm.server.service.LocalDirsHandleService
-
+import org.apache.commons.lang.StringUtils
+import org.apache.commons.lang.time.DateFormatUtils
class DefaultLocalDirsHandleService extends LocalDirsHandleService {
// TODO: 检测当前磁盘的健康状态，如果目录满了，需要上报am
- override def cleanup(): Unit = ???
+ override def cleanup(): Unit = {}
override def getEngineConnManagerHomeDir: String = ECM_HOME_DIR
- override def getEngineConnWorkDir(user: String, ticketId: String): String = new FsPath(Paths.get(ENGINECONN_ROOT_DIR, user, "workDir", ticketId).toFile.getPath).getPath
+ override def getEngineConnWorkDir(user: String, ticketId: String, engineType: String): String = {
+ val prefix = if (StringUtils.isBlank(engineType)) {
+ Paths.get(ENGINECONN_ROOT_DIR, user, DateFormatUtils.format(System.currentTimeMillis(), "yyyyMMdd")).toFile.getPath
+ } else {
+ Paths.get(ENGINECONN_ROOT_DIR, user, DateFormatUtils.format(System.currentTimeMillis(), "yyyyMMdd"), engineType).toFile.getPath
+ }
+ new FsPath(prefix + File.separator + ticketId).getPath
+ }
override def getEngineConnPublicDir: String = ENGINECONN_PUBLIC_DIR
- override def getEngineConnLogDir(user: String, ticketId: String): String = s"${getEngineConnWorkDir(user, ticketId)}${File.separator}logs"
+ override def getEngineConnLogDir(user: String, ticketId: String, engineType: String): String = s"${getEngineConnWorkDir(user, ticketId, engineType)}${File.separator}logs"
+
+ override def getEngineConnTmpDir(user: String, ticketId: String, engineType: String): String = s"${getEngineConnWorkDir(user, ticketId, engineType)}${File.separator}tmp"
- override def getEngineConnTmpDir(user: String, ticketId: String): String = s"${getEngineConnWorkDir(user, ticketId)}${File.separator}tmp"
}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultOperateService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultOperateService.scala
index bae6a26..3af1a7a 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultOperateService.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultOperateService.scala
@@ -22,7 +22,7 @@ import org.apache.linkis.common.utils.{Logging, Utils}
import org.apache.linkis.ecm.server.service.OperateService
import org.apache.linkis.manager.common.operator.OperatorFactory
import org.apache.linkis.manager.common.protocol.em.{ECMOperateRequest, ECMOperateResponse}
-import org.apache.linkis.message.annotation.Receiver
+import org.apache.linkis.rpc.serializer.annotation.Receiver
import org.springframework.stereotype.Service
import scala.collection.JavaConverters.mapAsScalaMapConverter
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultYarnCallbackService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultYarnCallbackService.scala
index 80f151a..c136e75 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultYarnCallbackService.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/DefaultYarnCallbackService.scala
@@ -20,7 +20,7 @@ package org.apache.linkis.ecm.server.service.impl
import org.apache.linkis.ecm.server.LinkisECMApplication
import org.apache.linkis.ecm.server.listener.{YarnAppIdCallbackEvent, YarnInfoCallbackEvent}
import org.apache.linkis.ecm.server.service.YarnCallbackService
-import org.apache.linkis.message.annotation.Receiver
+import org.apache.linkis.rpc.serializer.annotation.Receiver
import org.apache.linkis.protocol.callback.{YarnAPPIdCallbackProtocol, YarnInfoCallbackProtocol}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/LinuxProcessEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/LinuxProcessEngineConnLaunchService.scala
index 49d7666..921a2f7 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/LinuxProcessEngineConnLaunchService.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/LinuxProcessEngineConnLaunchService.scala
@@ -22,24 +22,18 @@ import org.apache.linkis.ecm.linux.launch.LinuxProcessEngineConnLaunch
import org.apache.linkis.ecm.server.conf.ECMConfiguration._
import org.apache.linkis.manager.common.entity.node.EngineNode
import org.apache.linkis.manager.engineplugin.common.launch.entity.{EngineConnBuildRequest, EngineConnLaunchRequest}
-import org.apache.linkis.message.annotation.Receiver
-import org.apache.linkis.message.builder.ServiceMethodContext
-import org.apache.linkis.message.conf.MessageSchedulerConf._
import org.apache.linkis.rpc.Sender
-
-import scala.concurrent.duration.Duration
+import org.apache.linkis.rpc.serializer.annotation.Receiver
class LinuxProcessEngineConnLaunchService extends ProcessEngineConnLaunchService {
@Receiver
- def launchEngineConn(engineConnBuildRequest: EngineConnBuildRequest, smc: ServiceMethodContext): EngineNode = {
+ def launchEngineConn(engineConnBuildRequest: EngineConnBuildRequest, sender: Sender): EngineNode = {
Sender.getSender(ENGINECONN_PLUGIN_SPRING_NAME).ask(engineConnBuildRequest) match {
case request: EngineConnLaunchRequest if ENGINECONN_CREATE_DURATION._1 != 0L =>
launchEngineConn(request, ENGINECONN_CREATE_DURATION._1)
- case request: EngineConnLaunchRequest =>
- launchEngineConn(request, smc.getAttribute(DURATION_KEY).asInstanceOf[Duration]._1)
}
}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala
index 2c30a13..4651bb6 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/service/impl/ProcessEngineConnLaunchService.scala
@@ -18,7 +18,6 @@
package org.apache.linkis.ecm.server.service.impl
import java.util.concurrent.TimeUnit
-
import org.apache.linkis.common.conf.Configuration
import org.apache.linkis.common.utils.Utils
import org.apache.linkis.ecm.core.engineconn.EngineConn
@@ -26,6 +25,7 @@ import org.apache.linkis.ecm.core.launch.ProcessEngineConnLaunch
import org.apache.linkis.ecm.server.LinkisECMApplication
import org.apache.linkis.ecm.server.conf.ECMConfiguration
import org.apache.linkis.ecm.server.conf.ECMConfiguration.MANAGER_SPRING_NAME
+import org.apache.linkis.ecm.core.conf.ECMErrorCode
import org.apache.linkis.ecm.server.errorcode.ECMErrorConstants
import org.apache.linkis.ecm.server.exception.ECMErrorException
import org.apache.linkis.ecm.server.listener.EngineConnStatusChangeEvent
@@ -36,6 +36,8 @@ import org.apache.linkis.manager.engineplugin.common.launch.entity.EngineConnLau
import org.apache.linkis.rpc.Sender
import org.apache.commons.io.IOUtils
import org.apache.commons.lang.exception.ExceptionUtils
+import org.apache.linkis.ecm.server.service.LocalDirsHandleService
+import org.apache.linkis.manager.label.utils.LabelUtil
import scala.concurrent.duration.Duration
import scala.concurrent.{Future, TimeoutException}
@@ -43,18 +45,34 @@ import scala.concurrent.{Future, TimeoutException}
abstract class ProcessEngineConnLaunchService extends AbstractEngineConnLaunchService {
- override def afterLaunch(request: EngineConnLaunchRequest, conn: EngineConn, duration: Long): Unit = {
- super.afterLaunch(request, conn, duration)
+ private var localDirsHandleService: LocalDirsHandleService = _
+
+ def setLocalDirsHandleService(localDirsHandleService: LocalDirsHandleService): Unit = this.localDirsHandleService = localDirsHandleService
+
+
+ override def waitEngineConnStart(request: EngineConnLaunchRequest, conn: EngineConn, duration: Long): Unit = {
conn.getEngineConnLaunchRunner.getEngineConnLaunch match {
- case launch: ProcessEngineConnLaunch => try {
+ case launch: ProcessEngineConnLaunch => Utils.tryCatch{
+ // Set the pid of the shell script before the pid callBack returns
+ launch.getPid().foreach(conn.setPid)
processMonitorThread(conn, launch, duration)
- } catch {
- case e: ECMErrorException =>
- warn(s"Failed to init ${conn.getServiceInstance}, status shutting down")
+ }{
+ case e: Throwable =>
val logPath = Utils.tryCatch(conn.getEngineConnManagerEnv.engineConnLogDirs) { t =>
- ECMConfiguration.ENGINECONN_ROOT_DIR + "/userName/" + conn.getTickedId + "/logs"
+ localDirsHandleService.getEngineConnLogDir(request.user, request.ticketId, LabelUtil.getEngineType(request.labels))
+ }
+ val canRetry = e match {
+ case ecmError: ECMErrorException =>
+ if (ECMErrorCode.EC_START_TIME_OUT == ecmError.getErrCode) {
+ true
+ } else {
+ false
+ }
+ case _ => false
}
- Sender.getSender(MANAGER_SPRING_NAME).send(EngineConnStatusCallbackToAM(conn.getServiceInstance, NodeStatus.ShuttingDown, "Failed to start EngineConn, reason: " + ExceptionUtils.getRootCauseMessage(e) + s"\n You can go to this path($logPath) to find the reason or ask the administrator for help"))
+ logger.warn(s"Failed to init ${conn.getServiceInstance}, status shutting down, canRetry $canRetry, logPath $logPath", e)
+ Sender.getSender(MANAGER_SPRING_NAME).send(EngineConnStatusCallbackToAM(conn.getServiceInstance, NodeStatus.ShuttingDown, "Failed to start EngineConn, reason: " + ExceptionUtils.getRootCauseMessage(e) + s"\n You can go to this path($logPath) to find the reason or ask the administrator for help", canRetry))
+ throw e
}
case _ =>
}
@@ -65,17 +83,17 @@ abstract class ProcessEngineConnLaunchService extends AbstractEngineConnLaunchSe
val tickedId = engineConn.getTickedId
val errorMsg = new StringBuilder
Future {
- val iterator = IOUtils.lineIterator(launch.getProcessInputStream, Configuration.BDP_ENCODING.getValue)
+ val iterator = IOUtils.lineIterator(launch.getProcessInputStream, Configuration.BDP_ENCODING.getValue)
var count = 0
val maxLen = ECMConfiguration.ENGINE_START_ERROR_MSG_MAX_LEN.getValue
while (!isCompleted(engineConn) && iterator.hasNext && count < maxLen) {
val line = iterator.next()
- println(s"${engineConn.getTickedId}:${line}")
errorMsg.append(line).append("\n")
count += 1
}
val exitCode = Option(launch.processWaitFor)
if (exitCode.exists(_ != 0)) {
+ logger.info(s"engine ${tickedId} process exit ")
LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll(EngineConnStatusChangeEvent(tickedId, ShuttingDown))
} else {
LinkisECMApplication.getContext.getECMSyncListenerBus.postToAll(EngineConnStatusChangeEvent(tickedId, Success))
@@ -84,10 +102,10 @@ abstract class ProcessEngineConnLaunchService extends AbstractEngineConnLaunchSe
Utils.tryThrow(Utils.waitUntil(() => engineConn.getStatus != Starting, Duration(timeout, TimeUnit.MILLISECONDS))) {
case e: TimeoutException =>
throw new ECMErrorException(ECMErrorConstants.ECM_ERROR, s"wait for $engineConn initial timeout.")
- case e: InterruptedException => //比如被ms cancel
+ case e: InterruptedException =>
throw new ECMErrorException(ECMErrorConstants.ECM_ERROR, s"wait for $engineConn initial interrupted.")
case t: Throwable =>
- error(s"unexpected error, now shutdown it.")
+ logger.error(s"unexpected error, now shutdown it.")
throw t
}
if (engineConn.getStatus == ShuttingDown) {
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala
index 504a7a6..4ca29a6 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/spring/ECMSpringConfiguration.scala
@@ -77,9 +77,10 @@ class ECMSpringConfiguration {
@Bean
@ConditionalOnMissingBean
- def getLinuxProcessEngineConnLaunchService(resourceLocalizationService: ResourceLocalizationService): EngineConnLaunchService = {
+ def getLinuxProcessEngineConnLaunchService(resourceLocalizationService: ResourceLocalizationService, localDirsHandleService: LocalDirsHandleService): EngineConnLaunchService = {
val service = new LinuxProcessEngineConnLaunchService
service.setResourceLocalizationService(resourceLocalizationService)
+ service.setLocalDirsHandleService(localDirsHandleService)
service
}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/util/ECMUtils.scala b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/util/ECMUtils.scala
index 341bac9..8a75343 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/util/ECMUtils.scala
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/util/ECMUtils.scala
@@ -33,8 +33,11 @@ import scala.collection.JavaConversions._
object ECMUtils {
+ @volatile var bmlClient: BmlClient = _
+ val lock = new Object()
+
private def download(resource: BmlResource, userName: String): util.Map[String, Object] = {
- val client: BmlClient = createBMLClient(userName)
+ val client: BmlClient = getBMLClient(userName)
var response: BmlDownloadResponse = null
if (resource.getVersion == null) {
response = client.downloadShareResource(userName, resource.getResourceId)
@@ -55,11 +58,13 @@ object ECMUtils {
IOUtils.closeQuietly(is)
}
- private def createBMLClient(userName: String): BmlClient = {
- if (userName == null)
- BmlClientFactory.createBmlClient()
- else
- BmlClientFactory.createBmlClient(userName)
+ private def getBMLClient(userName: String): BmlClient = {
+ if (bmlClient == null) {
+ lock.synchronized {
+ if (bmlClient == null) bmlClient = BmlClientFactory.createBmlClient(userName)
+ }
+ }
+ bmlClient
}
diff --git a/linkis-public-enhancements/linkis-publicservice/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/UDFTreeDao.java b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/util/HardwareUtils.scala
similarity index 52%
copy from linkis-public-enhancements/linkis-publicservice/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/UDFTreeDao.java
copy to linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/util/HardwareUtils.scala
index dd02919..9344e58 100644
--- a/linkis-public-enhancements/linkis-publicservice/linkis-udf/linkis-udf-service/src/main/java/org/apache/linkis/udf/dao/UDFTreeDao.java
+++ b/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/util/HardwareUtils.scala
@@ -15,22 +15,37 @@
* limitations under the License.
*/
-package org.apache.linkis.udf.dao;
+package org.apache.linkis.ecm.server.util
+
+import oshi.SystemInfo
+
+object HardwareUtils {
+
+
+ def getAvailableMemory(): Long = {
+ val systemInfo = new SystemInfo
+ val hardware = systemInfo.getHardware
+ val globalMemory = hardware.getMemory
+ globalMemory.getAvailable
+ }
+
+ def getMaxMemory(): Long = {
+ val systemInfo = new SystemInfo
+ val hardware = systemInfo.getHardware
+ val globalMemory = hardware.getMemory
+ globalMemory.getTotal
+ }
+
+ /**
+ * 1 total
+ * 2 available
+ * @return
+ */
+ def getTotalAndAvailableMemory(): (Long, Long) = {
+ val systemInfo = new SystemInfo
+ val hardware = systemInfo.getHardware
+ val globalMemory = hardware.getMemory
+ (globalMemory.getTotal, globalMemory.getAvailable)
+ }
-import org.apache.linkis.udf.entity.UDFTree;
-
-import java.util.List;
-import java.util.Map;
-
-public interface UDFTreeDao {
-
- void addTree(UDFTree udfTree);
-
- void updateTree(UDFTree udfTree);
-
- void deleteTree(Long id, String userName);
-
- UDFTree getTreeById(Long id, String category);
-
- List<UDFTree> getTreesByParentId(Map<String, Object> params);
}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/pom.xml b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/pom.xml
index 70060ce..d1e87d7 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/pom.xml
+++ b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/pom.xml
@@ -68,7 +68,7 @@
<dependency>
<groupId>org.apache.linkis</groupId>
- <artifactId>linkis-message-scheduler</artifactId>
+ <artifactId>linkis-rpc</artifactId>
<version>${linkis.version}</version>
<scope>provided</scope>
</dependency>
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/hook/OnceEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/hook/OnceEngineConnHook.scala
new file mode 100644
index 0000000..e08f29c
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-clustered-engineconn/linkis-once-engineconn/src/main/scala/org/apache/linkis/engineconn/once/executor/hook/OnceEngineConnHook.scala
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.engineconn.once.executor.hook
+
+import org.apache.linkis.engineconn.callback.hook.CallbackEngineConnHook
+import org.apache.linkis.engineconn.common.creation.EngineCreationContext
+import org.apache.linkis.engineconn.common.engineconn.EngineConn
+import org.apache.linkis.engineconn.core.executor.ExecutorManager
+import org.apache.linkis.manager.common.entity.enumeration.NodeStatus
+
+/**
+ * Conn Hook
+ */
+class OnceEngineConnHook extends CallbackEngineConnHook{
+
+ override protected def getNodeStatusOfStartSuccess(engineCreationContext: EngineCreationContext,
+ engineConn: EngineConn): NodeStatus = NodeStatus.Unlock
+
+ override def afterEngineServerStartSuccess(engineCreationContext: EngineCreationContext,
+ engineConn: EngineConn): Unit =
+ {
+ super.afterEngineServerStartSuccess(engineCreationContext, engineConn)
+ ExecutorManager.getInstance.getReportExecutor.tryReady()
+ }
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/pom.xml b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/pom.xml
index a27ef6e..3274276 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/pom.xml
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/pom.xml
@@ -70,7 +70,7 @@
<dependency>
<groupId>org.apache.linkis</groupId>
- <artifactId>linkis-message-scheduler</artifactId>
+ <artifactId>linkis-rpc</artifactId>
<version>${linkis.version}</version>
<scope>provided</scope>
</dependency>
@@ -93,11 +93,11 @@
<version>${linkis.version}</version>
</dependency>
- <dependency>
+ <!-- <dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>${guava.version}</version>
- </dependency>
+ </dependency>-->
</dependencies>
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncConcurrentComputationExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncConcurrentComputationExecutor.scala
index 32bff09..e77da19 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncConcurrentComputationExecutor.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncConcurrentComputationExecutor.scala
@@ -17,9 +17,204 @@
package org.apache.linkis.engineconn.computation.executor.async
-import org.apache.linkis.engineconn.computation.executor.execute.ConcurrentComputationExecutor
+import java.util
+import java.util.concurrent.ConcurrentHashMap
+import org.apache.linkis.DataWorkCloudApplication
+import org.apache.linkis.common.log.LogUtils
+import org.apache.linkis.common.utils.Utils
+import org.apache.linkis.engineconn.acessible.executor.listener.event.TaskResponseErrorEvent
+import org.apache.linkis.engineconn.common.conf.EngineConnConf
+import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask
+import org.apache.linkis.engineconn.computation.executor.execute.{ComputationExecutor, EngineExecutionContext}
+import org.apache.linkis.engineconn.computation.executor.hook.ComputationExecutorHook
+import org.apache.linkis.engineconn.core.EngineConnObject
+import org.apache.linkis.engineconn.executor.entity.ConcurrentExecutor
+import org.apache.linkis.engineconn.executor.listener.{EngineConnSyncListenerBus, ExecutorListenerBusContext}
+import org.apache.linkis.governance.common.entity.ExecutionNodeStatus
+import org.apache.linkis.manager.common.entity.enumeration.NodeStatus
+import org.apache.linkis.protocol.engine.JobProgressInfo
+import org.apache.linkis.scheduler.executer._
+import org.apache.linkis.scheduler.listener.JobListener
+import org.apache.linkis.scheduler.queue.SchedulerEventState._
+import org.apache.linkis.scheduler.queue.{Job, SchedulerEventState}
+import org.apache.commons.lang.StringUtils
+import org.apache.commons.lang.exception.ExceptionUtils
-abstract class AsyncConcurrentComputationExecutor extends ConcurrentComputationExecutor{
+abstract class AsyncConcurrentComputationExecutor(override val outputPrintLimit: Int = 1000) extends ComputationExecutor(outputPrintLimit) with ConcurrentExecutor with JobListener {
+ private lazy val scheduler = AsyncExecuteContext.getAsyncExecuteContext.getOrCreateScheduler(this)
+
+
+ private val taskIdToJobMap: util.Map[String, Job] = new ConcurrentHashMap[String, Job](8)
+
+ override def toExecuteTask(engineConnTask: EngineConnTask, internalExecute: Boolean = false): ExecuteResponse = {
+
+ transformTaskStatus(engineConnTask, ExecutionNodeStatus.Running)
+
+ val job = parserTaskToJob(engineConnTask)
+ job.setJobListener(this)
+ job.setId(engineConnTask.getTaskId)
+ taskIdToJobMap.put(engineConnTask.getTaskId, job)
+ Utils.tryCatch(scheduler.submit(job)) { t =>
+ val msg = s" submit task(${engineConnTask.getTaskId}) failed"
+ getEngineSyncListenerBus.postToAll(TaskResponseErrorEvent(engineConnTask.getTaskId, msg))
+ logger.error(msg, t)
+ onJobCompleted(job)
+ }
+ null
+ }
+
+ protected def parserTaskToJob(engineConnTask: EngineConnTask): Job = {
+ val engineExecutionContext = createEngineExecutionContext(engineConnTask)
+ new AsyncEngineConnJob(engineConnTask, engineExecutionContext)
+ }
+
+ def asyncExecuteTask(engineConnTask: EngineConnTask, engineExecutionContext: EngineExecutionContext): ExecuteResponse = {
+
+ var hookedCode = engineConnTask.getCode
+ Utils.tryCatch {
+ val engineCreationContext = EngineConnObject.getEngineCreationContext
+ ComputationExecutorHook.getComputationExecutorHooks.foreach(hook => {
+ hookedCode = hook.beforeExecutorExecute(engineExecutionContext, engineCreationContext, hookedCode)
+ })
+ }{
+ e =>
+ info("failed to do with hook", e)
+ engineExecutionContext.appendStdout(LogUtils.generateWarn(s"failed execute hook: ${ExceptionUtils.getFullStackTrace(e)}"))
+ }
+ if (hookedCode.length > 100) {
+ info(s"hooked after code: ${hookedCode.substring(0, 100)} ....")
+ } else {
+ info(s"hooked after code: $hookedCode ")
+ }
+ val localPath = EngineConnConf.getLogDir
+ engineExecutionContext.appendStdout(LogUtils.generateInfo(s"EngineConn local log path: ${DataWorkCloudApplication.getServiceInstance.toString} $localPath"))
+
+ var response: ExecuteResponse = Utils.tryCatch(executeLine(engineExecutionContext, hookedCode)
+ ) {
+ t => ErrorExecuteResponse(ExceptionUtils.getRootCauseMessage(t), t)
+ }
+
+ response match {
+ case e: ErrorExecuteResponse =>
+ logger.error("execute code failed!", e.t)
+ engineExecutionContext.appendStdout(LogUtils.generateERROR(s"execute code failed!: ${ExceptionUtils.getFullStackTrace(e.t)}"))
+ case SuccessExecuteResponse() =>
+ logger.info(s"task{${engineConnTask.getTaskId} execute success")
+ case e: OutputExecuteResponse =>
+ val output = if (StringUtils.isNotEmpty(e.getOutput) && e.getOutput.length > outputPrintLimit) {
+ e.getOutput.substring(0, outputPrintLimit)
+ } else e.getOutput
+ engineExecutionContext.appendStdout(output)
+ if (StringUtils.isNotBlank(e.getOutput)) engineExecutionContext.sendResultSet(e)
+ response = SuccessExecuteResponse()
+ case r: AsynReturnExecuteResponse =>
+ val msg = s"task${engineConnTask.getTaskId} received asyncReturnExecuteResponse"
+ logger.info(msg)
+ engineExecutionContext.appendStdout(msg)
+ case _ =>
+ logger.error(s"execute code failed! ${response}")
+ response = ErrorExecuteResponse("Unexpected response", null)
+ }
+ response
+ }
+
+ private def getJobByTaskId(taskId: String): Job = taskIdToJobMap.get(taskId)
+
+ private def removeJob(taskId: String): Job = taskIdToJobMap.remove(taskId)
+
+ override def progress(taskID: String): Float = 0f
+
+ override def getProgressInfo(taskID: String): Array[JobProgressInfo] = null
+
+ override def killTask(taskId: String): Unit = {
+ Utils.tryAndWarn {
+ val task = getTaskById(taskId)
+ if (null != task) {
+ val job = getJobByTaskId(task.getTaskId)
+ if (null != job) {
+ logger.info(s"kill job${job.getId()} task is ${task.getTaskId}")
+ job.kill()
+ }
+ /*task.setStatus(ExecutionNodeStatus.Cancelled)
+ transformTaskStatus(task, ExecutionNodeStatus.Cancelled)*/
+ }
+ }
+ }
+
+ override def killAll(): Unit = {
+ val jobs = taskIdToJobMap.values().iterator()
+ while (jobs.hasNext) {
+ val job = jobs.next()
+ if (null != job) {
+ logger.info(s"kill job${job.getId()} by killAll")
+ job.kill()
+ }
+ }
+ }
+
+ protected override def ensureOp[A](f: => A): A = f
+
+ override def afterExecute(engineConnTask: EngineConnTask, executeResponse: ExecuteResponse): Unit = {
+ }
+
+ private def getEngineSyncListenerBus: EngineConnSyncListenerBus = {
+ ExecutorListenerBusContext.getExecutorListenerBusContext().getEngineConnSyncListenerBus
+ }
+
+ // JobListener
+
+ override def onJobScheduled(job: Job): Unit = {
+ }
+
+ override def onJobInited(job: Job): Unit = {
+ }
+
+ override def onJobWaitForRetry(job: Job): Unit = {
+ }
+
+ override def onJobRunning(job: Job): Unit = {
+ if (isBusy) {
+ error(s"Executor is busy but still got new task ! Running task num : ${getRunningTask}")
+ }
+ if (getRunningTask >= getConcurrentLimit) synchronized {
+ if (getRunningTask >= getConcurrentLimit && NodeStatus.isIdle(getStatus)) {
+ info(s"running task($getRunningTask) > concurrent limit $getConcurrentLimit, now to mark engine to busy ")
+ transition(NodeStatus.Busy)
+ }
+ }
+ runningTasks.increase()
+ }
+
+ override def onJobCompleted(job: Job): Unit = {
+ runningTasks.decrease()
+ job match {
+ case asyncEngineConnJob: AsyncEngineConnJob =>
+ job.getState match {
+ case Succeed =>
+ succeedTasks.increase()
+ transformTaskStatus(asyncEngineConnJob.getEngineConnTask, ExecutionNodeStatus.Succeed)
+ case Failed =>
+ failedTasks.increase()
+ transformTaskStatus(asyncEngineConnJob.getEngineConnTask, ExecutionNodeStatus.Failed)
+ case _ =>
+ }
+ if(!SchedulerEventState.isSucceed(job.getState)) {
+ getEngineSyncListenerBus.postToAll(TaskResponseErrorEvent(asyncEngineConnJob.getEngineConnTask.getTaskId, job.getErrorResponse.message))
+ }
+ removeJob(asyncEngineConnJob.getEngineConnTask.getTaskId)
+ clearTaskCache(asyncEngineConnJob.getEngineConnTask.getTaskId)
+
+ case _ =>
+ }
+
+ if (getStatus == NodeStatus.Busy && getConcurrentLimit > getRunningTask) synchronized {
+ if (getStatus == NodeStatus.Busy && getConcurrentLimit > getRunningTask) {
+ info(s"running task($getRunningTask) < concurrent limit $getConcurrentLimit, now to mark engine to Unlock ")
+ transition(NodeStatus.Unlock)
+ }
+ }
+ }
}
+
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncEngineConnJob.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncEngineConnJob.scala
new file mode 100644
index 0000000..fce9437
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncEngineConnJob.scala
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.engineconn.computation.executor.async
+
+import org.apache.linkis.common.utils.Utils
+import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask
+import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext
+import org.apache.linkis.scheduler.executer.{CompletedExecuteResponse, ErrorExecuteResponse, ExecuteRequest, SuccessExecuteResponse}
+import org.apache.linkis.scheduler.queue.{Job, JobInfo}
+
+class AsyncEngineConnJob(task: EngineConnTask, engineExecutionContext: EngineExecutionContext) extends Job {
+
+
+ override def init(): Unit = {}
+
+ override protected def jobToExecuteRequest: ExecuteRequest = {
+ new AsyncExecuteRequest(task, engineExecutionContext)
+ }
+
+ override def getName: String = getId()
+
+ override def getJobInfo: JobInfo = null
+
+ def getEngineExecutionContext: EngineExecutionContext = this.engineExecutionContext
+
+ def getEngineConnTask: EngineConnTask = task
+
+ override def close(): Unit = {
+
+ }
+
+ override def transitionCompleted(executeCompleted: CompletedExecuteResponse): Unit = {
+ var executeCompletedNew: CompletedExecuteResponse = executeCompleted
+ executeCompleted match {
+ case _: SuccessExecuteResponse =>
+ Utils.tryCatch {
+ logger.info(s"job ${task.getTaskId} execute success, Start to close engineExecutionContext")
+ engineExecutionContext.close()
+ logger.info(s"job ${task.getTaskId} execute success, Finished to close engineExecutionContext")
+ } {
+ t =>
+ executeCompletedNew = ErrorExecuteResponse("send resultSet to entrance failed!", t)
+ }
+ case _ =>
+ Utils.tryQuietly(engineExecutionContext)
+ }
+ super.transitionCompleted(executeCompleted)
+ }
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecuteContext.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecuteContext.scala
new file mode 100644
index 0000000..cf932b5
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecuteContext.scala
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.engineconn.computation.executor.async
+
+import org.apache.linkis.engineconn.computation.executor.conf.ComputationExecutorConf
+import org.apache.linkis.scheduler.queue.parallelqueue.{ParallelScheduler, ParallelSchedulerContextImpl}
+import org.apache.linkis.scheduler.{Scheduler, SchedulerContext}
+
+trait AsyncExecuteContext {
+
+ def getOrCreateScheduler(executor: AsyncConcurrentComputationExecutor): Scheduler
+
+}
+
+class AsyncExecuteContextImpl extends AsyncExecuteContext {
+
+ private var scheduler: Scheduler = _
+
+ override def getOrCreateScheduler(executor: AsyncConcurrentComputationExecutor): Scheduler = {
+ if (null == scheduler) synchronized {
+ if (null == scheduler) {
+ scheduler = new ParallelScheduler(createSchedulerContext(executor))
+ scheduler.init()
+ }
+ }
+ scheduler
+ }
+
+ private def createSchedulerContext(executor: AsyncConcurrentComputationExecutor): SchedulerContext = {
+ val parallelSchedulerContext = new ParallelSchedulerContextImpl(ComputationExecutorConf.ASYNC_EXECUTE_MAX_PARALLELISM.getValue)
+ parallelSchedulerContext.setExecutorManager(new AsyncExecutorManager(executor))
+ parallelSchedulerContext
+ }
+
+}
+
+object AsyncExecuteContext {
+
+ private val CONTEXT: AsyncExecuteContext = new AsyncExecuteContextImpl()
+
+ def getAsyncExecuteContext: AsyncExecuteContext = CONTEXT
+
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecuteExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecuteExecutor.scala
new file mode 100644
index 0000000..a12ce3d
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecuteExecutor.scala
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.engineconn.computation.executor.async
+
+import org.apache.linkis.engineconn.common.exception.EngineConnException
+import org.apache.linkis.engineconn.computation.executor.utlis.ComputationErrorCode
+import org.apache.linkis.scheduler.executer.ExecutorState.ExecutorState
+import org.apache.linkis.scheduler.executer._
+
+class AsyncExecuteExecutor(executor: AsyncConcurrentComputationExecutor) extends Executor {
+
+ override def getId: Long = {
+ 0
+ }
+
+ override def execute(executeRequest: ExecuteRequest): ExecuteResponse = {
+ executeRequest match {
+ case asyncExecuteRequest: AsyncExecuteRequest =>
+ executor.asyncExecuteTask(asyncExecuteRequest.task, asyncExecuteRequest.engineExecutionContext)
+ case _ => throw EngineConnException(ComputationErrorCode.ASYNC_EXECUTOR_ERROR_CODE, "Mismatched execution request")
+ }
+ }
+
+ override def state: ExecutorState = {
+ ExecutorState.Idle
+ }
+
+ override def getExecutorInfo: ExecutorInfo = {
+ null
+ }
+
+ override def close(): Unit = {
+
+ }
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncConcurrentComputationExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecuteRequest.scala
similarity index 73%
copy from linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncConcurrentComputationExecutor.scala
copy to linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecuteRequest.scala
index 32bff09..7a90ade 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncConcurrentComputationExecutor.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecuteRequest.scala
@@ -17,9 +17,10 @@
package org.apache.linkis.engineconn.computation.executor.async
-import org.apache.linkis.engineconn.computation.executor.execute.ConcurrentComputationExecutor
-
-
-abstract class AsyncConcurrentComputationExecutor extends ConcurrentComputationExecutor{
+import org.apache.linkis.engineconn.computation.executor.entity.EngineConnTask
+import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext
+import org.apache.linkis.scheduler.executer.ExecuteRequest
+class AsyncExecuteRequest(val task: EngineConnTask, val engineExecutionContext: EngineExecutionContext) extends ExecuteRequest {
+ override val code: String = task.getCode
}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecutorManager.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecutorManager.scala
new file mode 100644
index 0000000..cc4f1db
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/async/AsyncExecutorManager.scala
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.engineconn.computation.executor.async
+
+import org.apache.linkis.common.utils.Logging
+import org.apache.linkis.scheduler.executer.{Executor, ExecutorManager}
+import org.apache.linkis.scheduler.listener.ExecutorListener
+import org.apache.linkis.scheduler.queue.SchedulerEvent
+
+import scala.concurrent.duration.Duration
+
+class AsyncExecutorManager(executor: AsyncConcurrentComputationExecutor) extends ExecutorManager with Logging {
+
+ private val asyncExecuteExecutor: AsyncExecuteExecutor = new AsyncExecuteExecutor(executor)
+
+ override def setExecutorListener(engineListener: ExecutorListener): Unit = {
+
+ }
+
+ override protected def createExecutor(event: SchedulerEvent): Executor = {
+ asyncExecuteExecutor
+ }
+
+ override def askExecutor(event: SchedulerEvent): Option[Executor] = {
+ Some(createExecutor(event))
+ }
+
+ override def askExecutor(event: SchedulerEvent, wait: Duration): Option[Executor] = {
+ Some(createExecutor(event))
+ }
+
+ override def getById(id: Long): Option[Executor] = {
+ None
+ }
+
+ override def getByGroup(groupName: String): Array[Executor] = {
+ Array.empty[Executor]
+ }
+
+ override def delete(executor: Executor): Unit = {}
+
+ override def shutdown(): Unit = {}
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala
index 91a554d..70b08dc 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/conf/ComputationExecutorConf.scala
@@ -34,6 +34,16 @@ object ComputationExecutorConf {
val UDF_LOAD_FAILED_IGNORE = CommonVars("wds.linkis.engineconn.udf.load.ignore", true)
+ val FUNCTION_LOAD_FAILED_IGNORE = CommonVars("wds.linkis.engineconn.function.load.ignore", true)
+
+ val TASK_IGNORE_UNCOMPLETED_STATUS = CommonVars("wds.linkis.engineconn.task.ignore.uncompleted.status", true).getValue
+
val ENGINE_CONCURRENT_THREAD_NUM = CommonVars("wds.linkis.engineconn.concurrent.thread.num", 20)
+
+ val ASYNC_EXECUTE_MAX_PARALLELISM = CommonVars("wds.linkis.engineconn.max.parallelism", 300)
+
+
+ val DEFAULT_COMPUTATION_EXECUTORMANAGER_CLAZZ = CommonVars("wds.linkis.default.computation.executormanager.clazz", "org.apache.linkis.engineconn.computation.executor.creation.ComputationExecutorManagerImpl")
+
}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationEngineConnExecution.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationEngineConnExecution.scala
index 9a686a3..bd53cf8 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationEngineConnExecution.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationEngineConnExecution.scala
@@ -21,6 +21,7 @@ import org.apache.linkis.DataWorkCloudApplication
import org.apache.linkis.engineconn.common.creation.EngineCreationContext
import org.apache.linkis.engineconn.common.engineconn.EngineConn
import org.apache.linkis.engineconn.common.execution.EngineConnExecution
+import org.apache.linkis.engineconn.computation.executor.conf.ComputationExecutorConf
import org.apache.linkis.engineconn.core.execution.AbstractEngineConnExecution
import org.apache.linkis.engineconn.executor.conf.EngineConnExecutorConfiguration
import org.apache.linkis.engineconn.executor.entity.Executor
@@ -64,8 +65,8 @@ class ComputationExecutorManagerEngineConnExecution extends EngineConnExecution
case _ =>
}
if(shouldSet) DataWorkCloudApplication.setProperty(EngineConnExecutorConfiguration.EXECUTOR_MANAGER_CLASS.key,
- "org.apache.linkis.engineconn.computation.executor.creation.ComputationExecutorManagerImpl")
-}
+ ComputationExecutorConf.DEFAULT_COMPUTATION_EXECUTORMANAGER_CLAZZ.getValue)
+ }
/**
* The smallest got the first execution opportunity.
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala
index 08f6c74..afa0839 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ComputationExecutor.scala
@@ -63,13 +63,13 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) extends Acc
private var codeParser: Option[CodeParser] = None
- private var runningTasks: Count = new Count
+ protected val runningTasks: Count = new Count
- private var pendingTasks: Count = new Count
+ protected val pendingTasks: Count = new Count
- private var succeedTasks: Count = new Count
+ protected val succeedTasks: Count = new Count
- private var failedTasks: Count = new Count
+ protected val failedTasks: Count = new Count
private var lastTask: EngineConnTask = _
@@ -144,11 +144,11 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) extends Acc
}
- def toExecuteTask(engineConnTask: EngineConnTask, internalExecute: Boolean = false): ExecuteResponse = {
+ def toExecuteTask(engineConnTask: EngineConnTask, internalExecute: Boolean = false): ExecuteResponse = {
runningTasks.increase()
this.internalExecute = internalExecute
Utils.tryFinally{
- transformTaskStatus(engineConnTask, ExecutionNodeStatus.Running)
+ transformTaskStatus(engineConnTask, ExecutionNodeStatus.Running)
val engineExecutionContext = createEngineExecutionContext(engineConnTask)
var hookedCode = engineConnTask.getCode
Utils.tryCatch {
@@ -156,7 +156,7 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) extends Acc
ComputationExecutorHook.getComputationExecutorHooks.foreach(hook => {
hookedCode = hook.beforeExecutorExecute(engineExecutionContext, engineCreationContext, hookedCode)
})
- } ( e => info("failed to do with hook", e))
+ } (e => info("failed to do with hook", e))
if (hookedCode.length > 100) {
info(s"hooked after code: ${hookedCode.substring(0, 100)} ....")
} else {
@@ -207,11 +207,11 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) extends Acc
failedTasks.increase()
}
+
if (null == response && codes.isEmpty) {
- error("This code is empty, and the task will be directly marked as successful.")
+ warn("This code is empty, the task will be directly marked as successful")
response = SuccessExecuteResponse()
}
-
response = response match {
case _: OutputExecuteResponse =>
succeedTasks.increase()
@@ -242,11 +242,11 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) extends Acc
toExecuteTask(engineConnTask)
}
- Utils.tryAndWarn(afterExecute(engineConnTask, response))
+ Utils.tryAndWarn(afterExecute(engineConnTask, response))
info(s"Finished to execute task ${engineConnTask.getTaskId}")
- lastTask = null
- response
- }
+ //lastTask = null
+ response
+ }
def setCodeParser(codeParser: CodeParser): Unit = this.codeParser = Some(codeParser)
@@ -285,7 +285,6 @@ abstract class ComputationExecutor(val outputPrintLimit: Int = 1000) extends Acc
Utils.tryAndWarn {
val task = getTaskById(taskId)
if (null != task) {
- task.setStatus(ExecutionNodeStatus.Cancelled)
transformTaskStatus(task, ExecutionNodeStatus.Cancelled)
}
}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ConcurrentComputationExecutor.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ConcurrentComputationExecutor.scala
index a0b30a3..1627f49 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ConcurrentComputationExecutor.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/execute/ConcurrentComputationExecutor.scala
@@ -30,7 +30,7 @@ abstract class ConcurrentComputationExecutor(override val outputPrintLimit: Int
error(s"Executor is busy but still got new task ! Running task num : ${getRunningTask}")
}
if (getRunningTask >= getConcurrentLimit) synchronized {
- if (getRunningTask >= getConcurrentLimit) {
+ if (getRunningTask >= getConcurrentLimit && NodeStatus.isIdle(getStatus)) {
info(s"running task($getRunningTask) > concurrent limit $getConcurrentLimit, now to mark engine to busy ")
transition(NodeStatus.Busy)
}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationExecutorHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationExecutorHook.scala
index cae3289..1d4b184 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationExecutorHook.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationExecutorHook.scala
@@ -54,5 +54,5 @@ object ComputationExecutorHook extends Logging {
hooks.sortWith((a, b) => a.getOrder() <= b.getOrder()).toArray
}
- def getComputationExecutorHooks = computationExecutorHooks
+ def getComputationExecutorHooks: Array[ComputationExecutorHook] = computationExecutorHooks
}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ExecutorLabelsRestHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ExecutorLabelsRestHook.scala
new file mode 100644
index 0000000..66e096e
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ExecutorLabelsRestHook.scala
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.linkis.engineconn.computation.executor.hook
+
+import org.apache.linkis.common.utils.Logging
+import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext
+import org.apache.linkis.manager.label.entity.Label
+import org.apache.linkis.manager.label.entity.engine.{CodeLanguageLabel, RunType}
+import org.apache.linkis.manager.label.utils.LabelUtil
+
+trait ExecutorLabelsRestHook {
+ def getOrder(): Int = 1
+
+ def getHookName(): String
+
+ def restExecutorLabels(labels: Array[Label[_]]): Array[Label[_]]
+}
+
+class CreatorRestLabelsHook extends ExecutorLabelsRestHook with Logging {
+
+ override def getHookName(): String = "CreatorRestLabelsHook"
+
+ override def restExecutorLabels(labels: Array[Label[_]]): Array[Label[_]] = {
+ val codeLabel = LabelUtil.getLabelFromArray[CodeLanguageLabel](labels)
+ if (null != codeLabel && RunType.FUNCTION_MDQ_TYPE.toString.equalsIgnoreCase(codeLabel.getCodeType)) {
+ logger.info(s"code Type Label($codeLabel) reset to scala")
+ val newCodeLabel = LabelBuilderFactoryContext.getLabelBuilderFactory.createLabel(classOf[CodeLanguageLabel])
+ newCodeLabel.setCodeType(RunType.SCALA.toString)
+ labels.filter(!_.isInstanceOf[CodeLanguageLabel]) ++ Array(newCodeLabel)
+ } else {
+ labels
+ }
+ }
+}
+
+object ExecutorLabelsRestHook {
+
+ private val HOOKS = Array[ExecutorLabelsRestHook](new CreatorRestLabelsHook)
+
+ def getExecutorLabelsRestHooks: Array[ExecutorLabelsRestHook] = HOOKS
+
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/InitSQLHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/InitSQLHook.scala
new file mode 100644
index 0000000..ef5afcd
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/InitSQLHook.scala
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.engineconn.computation.executor.hook
+
+import java.io.File
+import java.util
+
+import org.apache.linkis.common.conf.CommonVars
+import org.apache.linkis.common.utils.{Logging, Utils}
+import org.apache.linkis.engineconn.common.creation.EngineCreationContext
+import org.apache.linkis.engineconn.common.engineconn.EngineConn
+import org.apache.linkis.engineconn.common.hook.EngineConnHook
+import org.apache.linkis.engineconn.computation.executor.entity.CommonEngineConnTask
+import org.apache.linkis.engineconn.computation.executor.execute.ComputationExecutor
+import org.apache.linkis.engineconn.core.executor.ExecutorManager
+import org.apache.linkis.governance.common.entity.ExecutionNodeStatus
+import org.apache.linkis.manager.label.entity.Label
+import org.apache.linkis.manager.label.entity.engine.{CodeLanguageLabel, RunType}
+import org.apache.commons.io.FileUtils
+import org.apache.commons.lang.StringUtils
+
+
+abstract class InitSQLHook extends EngineConnHook with Logging {
+
+
+ private val INIT_SQL_DIR = CommonVars("wds.linkis.bdp.hive.init.sql.dir", "/appcom/config/hive-config/init_sql/").getValue
+
+ private val INIT_SQL_ENABLE = CommonVars("wds.linkis.bdp.hive.init.sql.enable", false)
+
+ override def beforeCreateEngineConn(engineCreationContext: EngineCreationContext): Unit = {}
+
+ override def beforeExecutionExecute(engineCreationContext: EngineCreationContext, engineConn: EngineConn): Unit = {}
+
+ protected def getRunType(): String
+
+ override def afterExecutionExecute(engineCreationContext: EngineCreationContext, engineConn: EngineConn): Unit = Utils.tryAndError {
+ val user: String = if (StringUtils.isNotBlank(engineCreationContext.getUser)) engineCreationContext.getUser else {
+ Utils.getJvmUser
+ }
+ if (! INIT_SQL_ENABLE.getValue(engineCreationContext.getOptions)) {
+ info(s"$user engineConn skip execute init_sql")
+ return
+ }
+
+ val initSql = readFile(INIT_SQL_DIR + user + "_hive.sql")
+ if (StringUtils.isBlank(initSql)) {
+ info(s"$user init_sql is empty")
+ return
+ }
+ info(s"$user engineConn begin to run init_sql")
+ val codeLanguageLabel = new CodeLanguageLabel
+ codeLanguageLabel.setCodeType(getRunType())
+ val labels = Array[Label[_]](codeLanguageLabel)
+ val engineConnTask = new CommonEngineConnTask("init_sql")
+ engineConnTask.setLabels(labels)
+ engineConnTask.setCode(initSql)
+ engineConnTask.setProperties(new util.HashMap[String, AnyRef]())
+ engineConnTask.setStatus(ExecutionNodeStatus.Scheduled)
+ ExecutorManager.getInstance.getExecutorByLabels(labels) match {
+ case executor: ComputationExecutor =>
+ executor.toExecuteTask(engineConnTask, internalExecute = true)
+ case _ =>
+ }
+ info(s"$user engineConn finished to run init_sql")
+ }
+
+ protected def readFile(path: String): String = {
+ info("read file: " + path)
+ val file = new File(path)
+ if (file.exists()) {
+ FileUtils.readFileToString(file)
+ } else {
+ info("file: [" + path + "] doesn't exist, ignore it.")
+ ""
+ }
+ }
+}
+
+class SparkInitSQLHook extends InitSQLHook {
+
+ override protected def getRunType(): String = RunType.SQL.toString
+
+}
+
+class HiveInitSQLHook extends InitSQLHook {
+
+ override protected def getRunType(): String = RunType.HIVE.toString
+
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/UDFLoadEngineConnHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/UDFLoadEngineConnHook.scala
index 4700c81..433c18a 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/UDFLoadEngineConnHook.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/UDFLoadEngineConnHook.scala
@@ -17,8 +17,8 @@
package org.apache.linkis.engineconn.computation.executor.hook
-import java.io.File
+import org.apache.linkis.bml.client.{BmlClient, BmlClientFactory}
import org.apache.linkis.common.utils.{Logging, Utils}
import org.apache.linkis.engineconn.common.creation.EngineCreationContext
import org.apache.linkis.engineconn.common.engineconn.EngineConn
@@ -27,75 +27,42 @@ import org.apache.linkis.engineconn.computation.executor.conf.ComputationExecuto
import org.apache.linkis.engineconn.computation.executor.execute.{ComputationExecutor, EngineExecutionContext}
import org.apache.linkis.engineconn.core.executor.ExecutorManager
import org.apache.linkis.manager.label.entity.Label
-import org.apache.linkis.manager.label.entity.engine.{CodeLanguageLabel, EngineTypeLabel}
+import org.apache.linkis.manager.label.entity.engine.{CodeLanguageLabel, EngineTypeLabel, RunType}
import org.apache.linkis.udf.UDFClient
-import org.apache.linkis.udf.entity.UDFInfo
import org.apache.linkis.udf.utils.ConstantVar
-import org.apache.commons.io.FileUtils
+import org.apache.commons.io.{FileUtils, IOUtils}
import org.apache.commons.lang.StringUtils
+import org.apache.linkis.common.conf.Configuration
+import org.apache.linkis.engineconn.core.engineconn.EngineConnManager
+import org.apache.linkis.engineconn.executor.entity.Executor
+import org.apache.linkis.manager.label.entity.engine.RunType.RunType
+import org.apache.linkis.udf.vo.UDFInfoVo
+import java.io.File
import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer
-abstract class UDFLoadEngineConnHook extends EngineConnHook with Logging {
+abstract class UDFLoad extends Logging {
+
protected val udfType: BigInt
protected val category: String
- protected val runType: String
+ protected val runType: RunType
- protected def getRealRunType(engineType: String): String = runType
+ private val bmlClient: BmlClient = BmlClientFactory.createBmlClient()
- protected def constructCode(udfInfo: UDFInfo): String
-
- override def afterExecutionExecute(engineCreationContext: EngineCreationContext, engineConn: EngineConn): Unit = {
- val user = engineCreationContext.getUser
- val codeLanguageLabel = new CodeLanguageLabel
- engineCreationContext.getLabels().find(_.isInstanceOf[EngineTypeLabel]) match {
- case Some(engineTypeLabel) =>
- codeLanguageLabel.setCodeType(getRealRunType(engineTypeLabel.asInstanceOf[EngineTypeLabel].getEngineType))
- case None =>
- codeLanguageLabel.setCodeType(runType)
- warn("no EngineTypeLabel found, use default runType")
- }
- val labels = Array[Label[_]](codeLanguageLabel)
- generateCode(user).foreach {
- case "" =>
- case c: String =>
- info("Submit udf registration to engine, code: " + c)
- ExecutorManager.getInstance.getExecutorByLabels(labels) match {
- case executor: ComputationExecutor =>
-// val task = new CommonEngineConnTask("udf-register-" + UDFLoadEngineConnHook.taskIdGenerator.incrementAndGet(), false)
-// task.setCode(c)
-// task.setStatus(ExecutionNodeStatus.Scheduled)
-// task.setLabels(labels)
-// task.setProperties(Maps.newHashMap())
-// executor.execute(task)
- Utils.tryCatch(executor.executeLine(new EngineExecutionContext(executor), c)){
- case t: Throwable =>
- if (! ComputationExecutorConf.UDF_LOAD_FAILED_IGNORE.getValue) {
- Utils.tryQuietly(executor.close())
- throw t
- }else {
- error("Failed to load udf", t)
- null
- }
- }
- }
- info("executed code: " + c)
- }
- }
+ protected def getRealRunType(engineType: String): RunType = runType
- protected def acceptCodeType(line: String): Boolean = {
- line.startsWith("%" + runType)
- }
+ protected def constructCode(udfInfo: UDFInfoVo): String
protected def generateCode(user: String): Array[String] = {
val codeBuffer = new ArrayBuffer[String]
val statementBuffer = new ArrayBuffer[String]
var accept = true
- getLoadUdfCode(user).split("\n").foreach{
+ getLoadUdfCode(user).split("\n").foreach {
case "" =>
case l if l.startsWith("%") =>
- if(acceptCodeType(l)){
+
+ if (acceptCodeType(l)) {
accept = true
codeBuffer.append(statementBuffer.mkString("\n"))
statementBuffer.clear()
@@ -105,20 +72,27 @@ abstract class UDFLoadEngineConnHook extends EngineConnHook with Logging {
case l if accept => statementBuffer.append(l)
case _ =>
}
- if(statementBuffer.nonEmpty) codeBuffer.append(statementBuffer.mkString("\n"))
- codeBuffer.toArray
+ if (statementBuffer.nonEmpty) codeBuffer.append(statementBuffer.mkString("\n"))
+ codeBuffer.filter(StringUtils.isNotBlank).toArray
+ }
+
+ protected def acceptCodeType(line: String): Boolean = {
+ line.startsWith("%" + runType.toString)
}
+
protected def getLoadUdfCode(user: String): String = {
info("start loading UDFs")
- val udfInfos = UDFClient.getUdfInfos(user, category).filter{ info => info.getUdfType == udfType && info.getExpire == false && info.getLoad == true}
- udfInfos.map(constructCode).mkString("\n")
+ val udfInfos = UDFClient.getUdfInfosByUdfType(user, category, udfType)
+ info("all udfs: ")
+ udfInfos.foreach { l => info("udfName:" + l.getUdfName + " bml_resource_id:" + l.getBmlResourceId + "\n") }
+ udfInfos.filter { info => StringUtils.isNotEmpty(info.getBmlResourceId) }.map(constructCode).mkString("\n")
}
protected def readFile(path: String): String = {
info("read file: " + path)
val file = new File(path)
- if(file.exists()){
+ if (file.exists()) {
FileUtils.readFileToString(file)
} else {
info("udf file: [" + path + "] doesn't exist, ignore it.")
@@ -126,6 +100,101 @@ abstract class UDFLoadEngineConnHook extends EngineConnHook with Logging {
}
}
+ protected def readFile(user: String, resourceId: String, resourceVersion: String): String = {
+ info("begin to download udf from bml.")
+ val downloadResponse = bmlClient.downloadResource(if (user == null) Utils.getJvmUser else user, resourceId, resourceVersion)
+ if (downloadResponse.isSuccess) {
+ Utils.tryFinally {
+ IOUtils.toString(downloadResponse.inputStream, Configuration.BDP_ENCODING.getValue)
+ } {
+ IOUtils.closeQuietly(downloadResponse.inputStream)
+ }
+ } else {
+ info("failed to download udf from bml.")
+ ""
+ }
+ }
+
+ private def getFunctionCode(): Array[String] = {
+ val engineCreationContext = EngineConnManager.getEngineConnManager.getEngineConn.getEngineCreationContext
+ val user = engineCreationContext.getUser
+ Utils.tryCatch(generateCode(user)) {
+ case t: Throwable =>
+ if (!ComputationExecutorConf.UDF_LOAD_FAILED_IGNORE.getValue) {
+ logger.error("Failed to load function, executor close ")
+ throw t
+ } else {
+ logger.error("Failed to load function", t)
+ Array.empty[String]
+ }
+ }
+ }
+
+ private def executeFunctionCode(codes: Array[String], executor: ComputationExecutor): Unit = {
+ if (null == codes || null == executor) {
+ return
+ }
+ codes.foreach { code =>
+ logger.info("Submit function registration to engine, code: " + code)
+ Utils.tryCatch(executor.executeLine(new EngineExecutionContext(executor), code)) {
+ case t: Throwable =>
+ if (!ComputationExecutorConf.UDF_LOAD_FAILED_IGNORE.getValue) {
+ Utils.tryQuietly(executor.close())
+ logger.error("Failed to load function, executor close ")
+ throw t
+ } else {
+ logger.error("Failed to load function", t)
+ null
+ }
+ }
+ }
+ }
+
+ protected def loadFunctions(executor: Executor): Unit = {
+
+ val codes = getFunctionCode()
+ if (null != codes && codes.nonEmpty) {
+ executor match {
+ case computationExecutor: ComputationExecutor =>
+ executeFunctionCode(codes, computationExecutor)
+ case _ =>
+ }
+ }
+ logger.info(s"Successful to execute function code ${runType}, type : ${udfType}")
+ }
+
+ protected def loadUDF(labels : Array[Label[_]]): Unit = {
+
+ val codes = getFunctionCode()
+ if (null != codes && codes.nonEmpty) {
+ val executor = ExecutorManager.getInstance.getExecutorByLabels(labels)
+ executor match {
+ case computationExecutor: ComputationExecutor =>
+ executeFunctionCode(codes, computationExecutor)
+ case _ =>
+ }
+ }
+ logger.info(s"Successful to execute code ${runType}, type : ${udfType}")
+ }
+
+}
+
+abstract class UDFLoadEngineConnHook extends UDFLoad with EngineConnHook with Logging {
+
+
+ override def afterExecutionExecute(engineCreationContext: EngineCreationContext, engineConn: EngineConn): Unit = {
+ val codeLanguageLabel = new CodeLanguageLabel
+ engineCreationContext.getLabels().find(_.isInstanceOf[EngineTypeLabel]) match {
+ case Some(engineTypeLabel) =>
+ codeLanguageLabel.setCodeType(getRealRunType(engineTypeLabel.asInstanceOf[EngineTypeLabel].getEngineType).toString)
+ case None =>
+ codeLanguageLabel.setCodeType(runType.toString)
+ warn("no EngineTypeLabel found, use default runType")
+ }
+ val labels = Array[Label[_]](codeLanguageLabel)
+ loadUDF(labels)
+ }
+
override def afterEngineServerStartFailed(engineCreationContext: EngineCreationContext, throwable: Throwable): Unit = {}
override def beforeCreateEngineConn(engineCreationContext: EngineCreationContext): Unit = {}
@@ -137,59 +206,40 @@ abstract class UDFLoadEngineConnHook extends EngineConnHook with Logging {
class JarUdfEngineHook extends UDFLoadEngineConnHook {
override val udfType: BigInt = ConstantVar.UDF_JAR
override val category: String = ConstantVar.UDF
- override val runType = "sql"
+ override val runType = RunType.SQL
- override protected def constructCode(udfInfo: UDFInfo): String = {
+ override protected def constructCode(udfInfo: UDFInfoVo): String = {
"%sql\n" + udfInfo.getRegisterFormat
}
- override protected def getRealRunType(engineType: String): String = {
- if(engineType.equals("hive")){
- return "hql"
+ override protected def getRealRunType(engineType: String): RunType = {
+ if (engineType.equals("hive")) {
+ return RunType.HIVE
}
runType
}
}
-class PyUdfEngineHook extends UDFLoadEngineConnHook{
+
+class PyUdfEngineHook extends UDFLoadEngineConnHook {
override val udfType: BigInt = ConstantVar.UDF_PY
override val category: String = ConstantVar.UDF
- override val runType = "py"
- override protected def constructCode(udfInfo: UDFInfo): String = {
- "%py\n" + readFile(udfInfo.getPath) + "\n" + (if(StringUtils.isNotBlank(udfInfo.getRegisterFormat)) udfInfo.getRegisterFormat else "")
+ override val runType = RunType.PYSPARK
+
+ override protected def constructCode(udfInfo: UDFInfoVo): String = {
+ "%py\n" + readFile(udfInfo.getCreateUser, udfInfo.getBmlResourceId, udfInfo.getBmlResourceVersion) + "\n" +
+ (if (StringUtils.isNotBlank(udfInfo.getRegisterFormat)) udfInfo.getRegisterFormat else "")
}
}
-class ScalaUdfEngineHook extends UDFLoadEngineConnHook{
+
+class ScalaUdfEngineHook extends UDFLoadEngineConnHook {
override val udfType: BigInt = ConstantVar.UDF_SCALA
override val category: String = ConstantVar.UDF
- override val runType = "scala"
- override protected def constructCode(udfInfo: UDFInfo): String = {
- "%scala\n" + readFile(udfInfo.getPath) + "\n" + (if(StringUtils.isNotBlank(udfInfo.getRegisterFormat)) udfInfo.getRegisterFormat else "")
- }
-}
-class PyFunctionEngineHook extends UDFLoadEngineConnHook{
- override val udfType: BigInt = ConstantVar.FUNCTION_PY
- override val category: String = ConstantVar.FUNCTION
- override val runType = "py"
- override protected def constructCode(udfInfo: UDFInfo): String = {
- "%py\n" + readFile(udfInfo.getPath)
- }
+ override val runType = RunType.SCALA
- override protected def getRealRunType(engineType: String): String = {
- if(engineType.equals("python")){
- return "python"
- }
- runType
- }
-}
-class ScalaFunctionEngineHook extends UDFLoadEngineConnHook{
- override val udfType: BigInt = ConstantVar.FUNCTION_SCALA
- override val category: String = ConstantVar.FUNCTION
- override val runType = "scala"
- override protected def constructCode(udfInfo: UDFInfo): String = {
- "%scala\n" + readFile(udfInfo.getPath)
+ override protected def constructCode(udfInfo: UDFInfoVo): String = {
+ "%scala\n" + readFile(udfInfo.getCreateUser, udfInfo.getBmlResourceId, udfInfo.getBmlResourceVersion) + "\n" +
+ (if (StringUtils.isNotBlank(udfInfo.getRegisterFormat)) udfInfo.getRegisterFormat else "")
}
+
}
-//object UDFLoadEngineConnHook {
-// val taskIdGenerator = new AtomicInteger(0)
-//}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/UseDatabaseEngineHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/UseDatabaseEngineHook.scala
new file mode 100644
index 0000000..3f8cf5d
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/UseDatabaseEngineHook.scala
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.engineconn.computation.executor.hook
+
+import org.apache.linkis.common.conf.CommonVars
+import org.apache.linkis.common.utils.{Logging, Utils}
+import org.apache.linkis.engineconn.common.creation.EngineCreationContext
+import org.apache.linkis.engineconn.common.engineconn.EngineConn
+import org.apache.linkis.engineconn.common.hook.EngineConnHook
+import org.apache.linkis.engineconn.computation.executor.execute.{ComputationExecutor, EngineExecutionContext}
+import org.apache.linkis.engineconn.core.executor.ExecutorManager
+import org.apache.linkis.manager.label.entity.Label
+import org.apache.linkis.manager.label.entity.engine.{CodeLanguageLabel, RunType}
+import org.apache.commons.lang.StringUtils
+
+abstract class UseDatabaseEngineHook extends EngineConnHook with Logging {
+
+
+ private val USE_DEFAULT_DB_ENABLE = CommonVars("wds.linkis.bdp.use.default.db.enable", true)
+
+ override def beforeCreateEngineConn(engineCreationContext: EngineCreationContext): Unit = {}
+
+ override def beforeExecutionExecute(engineCreationContext: EngineCreationContext, engineConn: EngineConn): Unit = {}
+
+ protected def getRunType(): String
+
+ override def afterExecutionExecute(engineCreationContext: EngineCreationContext, engineConn: EngineConn): Unit = Utils.tryAndError {
+ val user: String = if (StringUtils.isNotBlank(engineCreationContext.getUser)) engineCreationContext.getUser else {
+ Utils.getJvmUser
+ }
+ if (! USE_DEFAULT_DB_ENABLE.getValue(engineCreationContext.getOptions)) {
+ info(s"$user engineConn skip execute use default db")
+ return
+ }
+ val database = if (StringUtils.isNotEmpty(user)) {
+ user + "_ind"
+ } else {
+ "default"
+ }
+ val useDataBaseSql = "use " + database
+ info(s"$user begin to run init_code $useDataBaseSql")
+ val codeLanguageLabel = new CodeLanguageLabel
+ codeLanguageLabel.setCodeType(getRunType)
+ val labels = Array[Label[_]](codeLanguageLabel)
+ ExecutorManager.getInstance.getExecutorByLabels(labels) match {
+ case executor: ComputationExecutor =>
+ executor.executeLine(new EngineExecutionContext(executor), useDataBaseSql)
+ case _ =>
+ }
+ }
+
+}
+
+class SparkUseDatabaseEngineHook extends UseDatabaseEngineHook {
+
+ override protected def getRunType(): String = RunType.SQL.toString
+
+}
+
+class HiveUseDatabaseEngineHook extends UseDatabaseEngineHook {
+
+ override protected def getRunType(): String = RunType.HIVE.toString
+
+}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/executor/FunctionExecutorHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/executor/FunctionExecutorHook.scala
new file mode 100644
index 0000000..15d126d
--- /dev/null
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/executor/FunctionExecutorHook.scala
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.engineconn.computation.executor.hook.executor
+
+import org.apache.linkis.engineconn.computation.executor.hook.UDFLoad
+import org.apache.linkis.engineconn.core.hook.ExecutorHook
+import org.apache.linkis.engineconn.executor.entity.Executor
+import org.apache.linkis.manager.label.entity.engine.RunType
+import org.apache.linkis.manager.label.entity.engine.RunType.RunType
+import org.apache.linkis.udf.utils.ConstantVar
+import org.apache.linkis.udf.vo.UDFInfoVo
+
+class PyFunctionEngineHook extends UDFLoad with ExecutorHook {
+ override val udfType: BigInt = ConstantVar.FUNCTION_PY
+ override val category: String = ConstantVar.FUNCTION
+ override val runType = RunType.PYSPARK
+
+ override protected def constructCode(udfInfo: UDFInfoVo): String = {
+ "%py\n" + readFile(udfInfo.getCreateUser, udfInfo.getBmlResourceId, udfInfo.getBmlResourceVersion)
+ }
+
+ override protected def getRealRunType(runTypeIn: String): RunType = {
+ if (runTypeIn.equals("python")) {
+ return RunType.PYTHON
+ }
+ runType
+ }
+
+ override def getHookName(): String = "PyFunctionEngineHook"
+
+ override def afterExecutorInit(executor: Executor): Unit = {
+ logger.info("start to register python function ")
+ loadFunctions(executor)
+ logger.info("Finished to register python function ")
+ }
+
+ override def isAccepted(codeType: String): Boolean = runType.toString.equalsIgnoreCase(codeType) || RunType.PYTHON.toString.equalsIgnoreCase(codeType)
+}
+
+
+class ScalaFunctionEngineHook extends UDFLoad with ExecutorHook {
+ override val udfType: BigInt = ConstantVar.FUNCTION_SCALA
+ override val category: String = ConstantVar.FUNCTION
+ override val runType = RunType.SCALA
+
+ override protected def constructCode(udfInfo: UDFInfoVo): String = {
+ "%scala\n" + readFile(udfInfo.getCreateUser, udfInfo.getBmlResourceId, udfInfo.getBmlResourceVersion)
+ }
+
+ override def getHookName(): String = "ScalaFunctionEngineHook"
+
+ override def afterExecutorInit(executor: Executor): Unit = {
+ logger.info("start to register scala function ")
+ loadFunctions(executor)
+ logger.info("Finished to register scala function ")
+ }
+
+ override def isAccepted(codeType: String): Boolean = runType.toString.equalsIgnoreCase(codeType)
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/listener/TaskStatusListener.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/listener/TaskStatusListener.scala
index a893cc6..1619324 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/listener/TaskStatusListener.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/listener/TaskStatusListener.scala
@@ -17,7 +17,7 @@
package org.apache.linkis.engineconn.computation.executor.listener
-import org.apache.linkis.engineconn.acessible.executor.listener.event.TaskStatusChangedEvent
+import org.apache.linkis.engineconn.acessible.executor.listener.event.{TaskResponseErrorEvent, TaskStatusChangedEvent}
import org.apache.linkis.engineconn.executor.listener.EngineConnSyncListener
@@ -25,4 +25,6 @@ trait TaskStatusListener extends EngineConnSyncListener {
def onTaskStatusChanged(taskStatusChangedEvent: TaskStatusChangedEvent): Unit
+ def onTaskResponseErrorEvent(taskResponseErrorEvent: TaskResponseErrorEvent): Unit
+
}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionService.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionService.scala
index 159404c..7e62f54 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionService.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionService.scala
@@ -18,13 +18,13 @@
package org.apache.linkis.engineconn.computation.executor.service
import org.apache.linkis.governance.common.protocol.task._
-import org.apache.linkis.message.builder.ServiceMethodContext
+import org.apache.linkis.rpc.Sender
import org.apache.linkis.scheduler.executer.ExecuteResponse
trait TaskExecutionService {
- def execute(requestTask: RequestTask, smc: ServiceMethodContext): ExecuteResponse
+ def execute(requestTask: RequestTask, sender: Sender): ExecuteResponse
// def taskStatus(taskID: String): ResponseTaskStatus
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala
index 2589c20..29b21f4 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/service/TaskExecutionServiceImpl.scala
@@ -17,10 +17,9 @@
package org.apache.linkis.engineconn.computation.executor.service
-import java.util.concurrent.atomic.AtomicInteger
-import java.util.concurrent._
-
import com.google.common.cache.{Cache, CacheBuilder}
+import org.apache.commons.lang.StringUtils
+import org.apache.commons.lang.exception.ExceptionUtils
import org.apache.linkis.common.listener.Event
import org.apache.linkis.common.utils.{Logging, Utils}
import org.apache.linkis.engineconn.acessible.executor.listener.LogListener
@@ -28,30 +27,35 @@ import org.apache.linkis.engineconn.acessible.executor.listener.event._
import org.apache.linkis.engineconn.acessible.executor.log.LogHelper
import org.apache.linkis.engineconn.acessible.executor.service.LockService
import org.apache.linkis.engineconn.common.conf.{EngineConnConf, EngineConnConstant}
+import org.apache.linkis.engineconn.computation.executor.async.AsyncConcurrentComputationExecutor
import org.apache.linkis.engineconn.computation.executor.conf.ComputationExecutorConf
import org.apache.linkis.engineconn.computation.executor.entity.{CommonEngineConnTask, EngineConnTask}
import org.apache.linkis.engineconn.computation.executor.execute.{ComputationExecutor, ConcurrentComputationExecutor}
+import org.apache.linkis.engineconn.computation.executor.hook.ExecutorLabelsRestHook
import org.apache.linkis.engineconn.computation.executor.listener.{ResultSetListener, TaskProgressListener, TaskStatusListener}
import org.apache.linkis.engineconn.computation.executor.utlis.{ComputationEngineConstant, ComputationEngineUtils}
import org.apache.linkis.engineconn.core.executor.ExecutorManager
+import org.apache.linkis.engineconn.executor.entity.ResourceFetchExecutor
import org.apache.linkis.engineconn.executor.listener.ExecutorListenerBusContext
import org.apache.linkis.engineconn.executor.listener.event.EngineConnSyncEvent
import org.apache.linkis.governance.common.entity.ExecutionNodeStatus
import org.apache.linkis.governance.common.exception.engineconn.{EngineConnExecutorErrorCode, EngineConnExecutorErrorException}
import org.apache.linkis.governance.common.protocol.task._
import org.apache.linkis.manager.common.entity.enumeration.NodeStatus
-import org.apache.linkis.message.annotation.Receiver
-import org.apache.linkis.message.builder.ServiceMethodContext
+import org.apache.linkis.manager.common.protocol.resource.ResponseTaskYarnResource
+import org.apache.linkis.manager.label.entity.Label
import org.apache.linkis.protocol.message.RequestProtocol
import org.apache.linkis.rpc.Sender
+import org.apache.linkis.rpc.serializer.annotation.Receiver
import org.apache.linkis.rpc.utils.RPCUtils
import org.apache.linkis.scheduler.executer.{ErrorExecuteResponse, ExecuteResponse, IncompleteExecuteResponse, SubmitResponse}
import org.apache.linkis.server.BDPJettyServerHelper
-import javax.annotation.PostConstruct
-import org.apache.commons.lang.StringUtils
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Component
+import java.util.concurrent._
+import java.util.concurrent.atomic.AtomicInteger
+import javax.annotation.PostConstruct
import scala.collection.JavaConverters._
@@ -102,24 +106,23 @@ class TaskExecutionServiceImpl extends TaskExecutionService with Logging with Re
}
@Receiver
- override def execute(requestTask: RequestTask, smc: ServiceMethodContext): ExecuteResponse = {
- // smc // todo get sender
+ override def execute(requestTask: RequestTask, sender: Sender): ExecuteResponse = {
+
// check lock
info("Received a new task, task content is " + requestTask)
if (StringUtils.isBlank(requestTask.getLock)) {
error(s"Invalid lock : ${requestTask.getLock} , requestTask : " + requestTask)
- return ErrorExecuteResponse(s"Invalid lock : ${requestTask.getLock}.", new EngineConnExecutorErrorException(EngineConnExecutorErrorCode.INVALID_PARAMS, "Invalid lock or code(请获取到锁后再提交任务.)"))
+ return ErrorExecuteResponse(s"Invalid lock : ${requestTask.getLock}.", new EngineConnExecutorErrorException(EngineConnExecutorErrorCode.INVALID_PARAMS, "Invalid lock or code"))
}
if (!lockService.isLockExist(requestTask.getLock)) {
error(s"Lock ${requestTask.getLock} not exist, cannot execute.")
- return ErrorExecuteResponse("Lock not exixt", new EngineConnExecutorErrorException(EngineConnExecutorErrorCode.INVALID_LOCK, "Lock : " + requestTask.getLock + " not exist(您的锁无效，请重新获取后再提交)."))
+ return ErrorExecuteResponse("Lock not exixt", new EngineConnExecutorErrorException(EngineConnExecutorErrorCode.INVALID_LOCK, "Lock : " + requestTask.getLock + " not exist."))
}
if (StringUtils.isBlank(requestTask.getCode)) {
- return IncompleteExecuteResponse("Your code is incomplete, it may be that only comments are selected for execution(您的代码不完整，可能是仅仅选中了注释进行执行)")
+ return IncompleteExecuteResponse("Your code is incomplete, it may be that only comments are selected for execution")
}
- // 获取任务类型和任务代码，运行任务
val taskId: Int = taskExecutedNum.incrementAndGet()
val retryAble: Boolean = {
val retry = requestTask.getProperties.getOrDefault(ComputationEngineConstant.RETRYABLE_TYPE_NAME, null)
@@ -133,8 +136,34 @@ class TaskExecutionServiceImpl extends TaskExecutionService with Logging with Re
task.setStatus(ExecutionNodeStatus.Scheduled)
val labels = requestTask.getLabels.asScala.toArray
task.setLabels(labels)
- val entranceServerInstance = RPCUtils.getServiceInstanceFromSender(smc.getSender)
+ val entranceServerInstance = RPCUtils.getServiceInstanceFromSender(sender)
task.setCallbackServiceInstance(entranceServerInstance)
+ if (executorManager.containExecutor(labels)) {
+ submitTaskToExecutor(task, labels)
+ } else {
+ logger.info(s"task $taskId need to create executor")
+ val runnable = new Runnable {
+ override def run(): Unit = Utils.tryCatch {
+ submitTaskToExecutor(task, labels) match {
+ case ErrorExecuteResponse(message, throwable) =>
+ sendToEntrance(task, ResponseTaskError(task.getTaskId, message))
+ logger.error(message, throwable)
+ sendToEntrance(task, ResponseTaskStatus(task.getTaskId, ExecutionNodeStatus.Failed))
+ case _ =>
+ }
+ } { t =>
+ logger.warn("Failed to submit task ", t)
+ sendToEntrance(task, ResponseTaskError(task.getTaskId, ExceptionUtils.getRootCauseMessage(t)))
+ sendToEntrance(task, ResponseTaskStatus(task.getTaskId, ExecutionNodeStatus.Failed))
+ }
+ }
+ val submitTaskToExecutorFuture = Utils.defaultScheduler.submit(runnable)
+ SubmitResponse(task.getTaskId)
+ }
+ }
+
+ private def submitTaskToExecutor(task: CommonEngineConnTask, labels: Array[Label[_]]): ExecuteResponse = {
+ val newLabels = restExecutorLabels(labels)
val executor = executorManager.getExecutorByLabels(labels)
executor match {
case computationExecutor: ComputationExecutor =>
@@ -142,14 +171,18 @@ class TaskExecutionServiceImpl extends TaskExecutionService with Logging with Re
submitTask(task, computationExecutor)
case o =>
val labelsStr = if (labels != null) labels.filter(_ != null).map(_.getStringValue).mkString(",") else ""
- val msg = "Invalid computationExecutor : " + o.getClass.getName + ", labels : " + labelsStr + ", requestTask : " + requestTask
+ val msg = "Invalid computationExecutor : " + o.getClass.getName + ", labels : " + labelsStr + ", requestTask : " + task.getTaskId
error(msg)
- ErrorExecuteResponse("Invalid computationExecutor(生成无效的计算引擎，请联系管理员).",
+ ErrorExecuteResponse("Invalid computationExecutor.",
new EngineConnExecutorErrorException(EngineConnExecutorErrorCode.INVALID_ENGINE_TYPE, msg))
}
-
}
+ private def restExecutorLabels(labels: Array[Label[_]]): Array[Label[_]] = {
+ var newLabels = labels
+ ExecutorLabelsRestHook.getExecutorLabelsRestHooks.foreach(hooke => newLabels = hooke.restExecutorLabels(newLabels))
+ newLabels
+ }
// override def taskStatus(taskID: String): ResponseTaskStatus = {
// val task = taskIdCache.get(taskID)
@@ -159,6 +192,8 @@ class TaskExecutionServiceImpl extends TaskExecutionService with Logging with Re
private def submitTask(task: CommonEngineConnTask, computationExecutor: ComputationExecutor): ExecuteResponse = {
info(s"Task ${task.getTaskId} was submited.")
computationExecutor match {
+ case executor: AsyncConcurrentComputationExecutor =>
+ submitASyncTask(task, executor)
case concurrentComputationExecutor: ConcurrentComputationExecutor =>
submitConcurrentTask(task, concurrentComputationExecutor)
case _ =>
@@ -166,6 +201,19 @@ class TaskExecutionServiceImpl extends TaskExecutionService with Logging with Re
}
}
+ private def submitASyncTask(task: CommonEngineConnTask, computationExecutor: ComputationExecutor): ExecuteResponse = {
+ var response: ExecuteResponse = SubmitResponse(task.getTaskId)
+ Utils.tryCatch {
+ computationExecutor.execute(task)
+ } { t =>
+ error(s"Failed to submit task${task.getTaskId} ", t)
+ response = ErrorExecuteResponse("Failed to submit task", t)
+ null
+ }
+ response
+ }
+
+
private def submitSyncTask(task: CommonEngineConnTask, computationExecutor: ComputationExecutor): ExecuteResponse = {
val runTask = new Runnable {
override def run(): Unit = Utils.tryAndWarn {
@@ -264,6 +312,14 @@ class TaskExecutionServiceImpl extends TaskExecutionService with Logging with Re
while (null != taskFuture && !taskFuture.isDone) {
if (ExecutionNodeStatus.isCompleted(task.getStatus) || ExecutionNodeStatus.isRunning(task.getStatus)) {
sendToEntrance(task, taskProgress(task.getTaskId))
+ taskYarnResource(task.getTaskId) match {
+ case responseTaskYarnResource: ResponseTaskYarnResource => {
+ if (responseTaskYarnResource.resourceMap != null && !responseTaskYarnResource.resourceMap.isEmpty) {
+ sendToEntrance(task, responseTaskYarnResource)
+ }
+ }
+ case _ =>
+ }
Thread.sleep(TimeUnit.MILLISECONDS.convert(sleepInterval, TimeUnit.SECONDS))
}
}
@@ -271,6 +327,16 @@ class TaskExecutionServiceImpl extends TaskExecutionService with Logging with Re
})
}
+ private def taskYarnResource(taskID: String): ResponseTaskYarnResource = {
+ val executor = taskIdCache.getIfPresent(taskID)
+ executor match {
+ case executor: ResourceFetchExecutor =>
+ val resourceWithApplicationId = executor.FetchResource
+ ResponseTaskYarnResource(taskID, resourceWithApplicationId)
+ case _ => null
+ }
+ }
+
override def taskProgress(taskID: String): ResponseTaskProgress = {
var response = ResponseTaskProgress(taskID, 0, null)
if (StringUtils.isBlank(taskID)) return response
@@ -365,6 +431,7 @@ class TaskExecutionServiceImpl extends TaskExecutionService with Logging with Re
case logUpdateEvent: TaskLogUpdateEvent => onLogUpdate(logUpdateEvent)
case taskResultCreateEvent: TaskResultCreateEvent => onResultSetCreated(taskResultCreateEvent)
case taskResultSizeCreatedEvent: TaskResultSizeCreatedEvent => onResultSizeCreated(taskResultSizeCreatedEvent)
+ case taskResponseErrorEvent: TaskResponseErrorEvent => onTaskResponseErrorEvent(taskResponseErrorEvent)
case _ =>
warn("Unknown event : " + BDPJettyServerHelper.gson.toJson(event))
}
@@ -403,8 +470,12 @@ class TaskExecutionServiceImpl extends TaskExecutionService with Logging with Re
LogHelper.pushAllRemainLogs()
}
val toStatus = taskStatusChangedEvent.toStatus
- logger.info(s"send task ${task.getTaskId} status $toStatus to entrance")
- sendToEntrance(task, ResponseTaskStatus(taskStatusChangedEvent.taskId, taskStatusChangedEvent.toStatus))
+ if (!ComputationExecutorConf.TASK_IGNORE_UNCOMPLETED_STATUS || ExecutionNodeStatus.isCompleted(toStatus)) {
+ logger.info(s"send task ${task.getTaskId} status $toStatus to entrance")
+ sendToEntrance(task, ResponseTaskStatus(taskStatusChangedEvent.taskId, taskStatusChangedEvent.toStatus))
+ } else {
+ logger.info(s"task ${task.getTaskId} status $toStatus will not be send to entrance")
+ }
} else {
error("Task cannot null! taskStatusChangedEvent: " + ComputationEngineUtils.GSON.toJson(taskStatusChangedEvent))
}
@@ -473,4 +544,11 @@ class TaskExecutionServiceImpl extends TaskExecutionService with Logging with Re
}
}
}
+
+ override def onTaskResponseErrorEvent(taskResponseErrorEvent: TaskResponseErrorEvent): Unit = {
+ val task = getTaskByTaskId(taskResponseErrorEvent.taskId)
+ if (null != task) {
+ sendToEntrance(task, ResponseTaskError(task.getTaskId, taskResponseErrorEvent.errorMsg))
+ }
+ }
}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorCode.scala b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/utlis/ComputationErrorCode.scala
similarity index 80%
copy from linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorCode.scala
copy to linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/utlis/ComputationErrorCode.scala
index 8302e45..08f9ddc 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorCode.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/utlis/ComputationErrorCode.scala
@@ -15,16 +15,14 @@
* limitations under the License.
*/
-package org.apache.linkis.ecm.server.exception
+package org.apache.linkis.engineconn.computation.executor.utlis
-object ECMErrorCode {
+object ComputationErrorCode {
- val EC_START_TIME_OUT = 11100
- val EC_INTERRUPT_TIME_OUT = 11101
+ val ASYNC_EXECUTOR_ERROR_CODE = 11301
- val EC_START_FAILED = 11102
+ val UDF_LOAD_ERROR_CODE = 11302
- val EC_FETCH_LOG_FAILED = 11110
}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/executor/ExecutorManager.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/executor/ExecutorManager.scala
index bb6872e..4e019a6 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/executor/ExecutorManager.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/executor/ExecutorManager.scala
@@ -20,12 +20,12 @@ package org.apache.linkis.engineconn.core.executor
import java.util
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.atomic.AtomicInteger
-
import org.apache.linkis.common.utils.{Logging, Utils}
import org.apache.linkis.engineconn.common.creation.EngineCreationContext
import org.apache.linkis.engineconn.common.engineconn.EngineConn
import org.apache.linkis.engineconn.core.EngineConnObject
import org.apache.linkis.engineconn.core.engineconn.EngineConnManager
+import org.apache.linkis.engineconn.core.hook.ExecutorHook
import org.apache.linkis.engineconn.core.util.EngineConnUtils
import org.apache.linkis.engineconn.executor.conf.EngineConnExecutorConfiguration
import org.apache.linkis.engineconn.executor.entity.{Executor, LabelExecutor, SensibleExecutor}
@@ -33,6 +33,7 @@ import org.apache.linkis.manager.engineplugin.common.EngineConnPlugin
import org.apache.linkis.manager.engineplugin.common.creation._
import org.apache.linkis.manager.engineplugin.common.exception.{EngineConnPluginErrorCode, EngineConnPluginErrorException}
import org.apache.linkis.manager.label.entity.Label
+import org.apache.linkis.manager.label.utils.LabelUtil
import scala.collection.JavaConverters._
@@ -56,6 +57,8 @@ trait LabelExecutorManager extends ExecutorManager {
def removeExecutor(labels: Array[Label[_]]): LabelExecutor
+ def containExecutor(labels: Array[Label[_]]): Boolean
+
}
class LabelExecutorManagerImpl extends LabelExecutorManager with Logging {
@@ -64,6 +67,7 @@ class LabelExecutorManagerImpl extends LabelExecutorManager with Logging {
protected val GSON = EngineConnUtils.GSON
private val idCreator = new AtomicInteger()
+
protected lazy val engineConn: EngineConn = EngineConnManager.getEngineConnManager.getEngineConn
protected val engineConnPlugin: EngineConnPlugin = EngineConnObject.getEngineConnPlugin
@@ -97,8 +101,11 @@ class LabelExecutorManagerImpl extends LabelExecutorManager with Logging {
info("No LabelExecutorFactory matched, use DefaultExecutorFactory to create executor.")
defaultFactory.createExecutor(engineCreationContext, engineConn).asInstanceOf[LabelExecutor]
}
+ val codeType = LabelUtil.getCodeType(labelExecutor.getExecutorLabels())
info(s"Finished to create ${labelExecutor.getClass.getSimpleName}(${labelExecutor.getId}) with labels $labelStr.")
+ ExecutorHook.getAllExecutorHooks().filter(_.isAccepted(codeType)).foreach(_.beforeExecutorInit(labelExecutor))
labelExecutor.init()
+ ExecutorHook.getAllExecutorHooks().filter(_.isAccepted(codeType)).foreach(_.afterExecutorInit(labelExecutor))
info(s"Finished to init ${labelExecutor.getClass.getSimpleName}(${labelExecutor.getId}).")
labelExecutor
}
@@ -108,24 +115,23 @@ class LabelExecutorManagerImpl extends LabelExecutorManager with Logging {
protected def createExecutor(engineCreationContext: EngineCreationContext,
labels: Array[Label[_]] = null): LabelExecutor = {
if (null == labels || labels.isEmpty) {
- defaultFactory match {
- case labelExecutorFactory: CodeLanguageLabelExecutorFactory =>
+ defaultFactory match {
+ case labelExecutorFactory: CodeLanguageLabelExecutorFactory =>
val createExecutorLabels = Array[Label[_]](labelExecutorFactory.getDefaultCodeLanguageLabel)
createLabelExecutor(engineCreationContext, createExecutorLabels)
- case _ =>
- val executor = tryCreateExecutor(engineCreationContext, null)
- executors.put(executor.getId, executor)
- executor
+ case _ =>
+ val executor = tryCreateExecutor(engineCreationContext, null)
+ executors.put(executor.getId, executor)
+ executor
//throw new EngineConnPluginErrorException(EngineConnPluginErrorCode.INVALID_LABELS, "no labels and defaultFactory is not CodeLanguageLabelExecutorFactory")
- }
+ }
} else {
createLabelExecutor(engineCreationContext, labels)
- }
-
+ }
}
private def createLabelExecutor(engineCreationContext: EngineCreationContext,
- labels: Array[Label[_]]): LabelExecutor = {
+ labels: Array[Label[_]]): LabelExecutor = {
val labelKey = getLabelKey(labels)
if (null == labelKey) {
val msg = "Cannot get label key. labels : " + GSON.toJson(labels)
@@ -134,8 +140,8 @@ class LabelExecutorManagerImpl extends LabelExecutorManager with Logging {
if (!executors.containsKey(labelKey)) executors synchronized {
if (!executors.containsKey(labelKey)) {
- val executor = tryCreateExecutor(engineCreationContext, labels)
- executors.put(labelKey, executor)
+ val executor = tryCreateExecutor(engineCreationContext, labels)
+ executors.put(labelKey, executor)
}
}
executors.get(labelKey)
@@ -147,26 +153,33 @@ class LabelExecutorManagerImpl extends LabelExecutorManager with Logging {
val labelKey = getLabelKey(labels)
if (null == labelKey) return null
if (!executors.containsKey(labelKey)) executors synchronized {
- if (!executors.containsKey(labelKey)) {
- createExecutor(engineConn.getEngineCreationContext, labels)
- }
+ if (!executors.containsKey(labelKey)) {
+ createExecutor(engineConn.getEngineCreationContext, labels)
+ }
}
executors.get(labelKey)
}
+ override def containExecutor(labels: Array[Label[_]]): Boolean = {
+ val labelKey = getLabelKey(labels)
+ null != labelKey && executors.containsKey(labelKey)
+ }
+
override def getExecutor(id: String): Executor = executors.values().asScala.find(_.getId == id).orNull
override def getExecutors: Array[Executor] = executors.values().asScala.toArray
- override def removeExecutor(labels: Array[Label[_]]): LabelExecutor = {
+ override def removeExecutor(labels: Array[Label[_]]): LabelExecutor = executors.synchronized {
val labelKey = getLabelKey(labels)
if (labelKey != null && executors.containsKey(labelKey)) executors.remove(labelKey)
else null
}
- override def removeExecutor(id: String): Executor = executors.asScala.find(_._2.getId == id).map{
- case (k, _) => executors.remove(k)
- }.orNull
+ override def removeExecutor(id: String): Executor = executors.synchronized {
+ executors.asScala.find(_._2.getId == id).map {
+ case (k, _) => executors.remove(k)
+ }.orNull
+ }
override def getReportExecutor: Executor = if (getExecutors.isEmpty) {
val labels = defaultFactory match {
@@ -180,9 +193,11 @@ class LabelExecutorManagerImpl extends LabelExecutorManager with Logging {
} else {
getExecutors.maxBy {
case executor: SensibleExecutor => executor.getStatus.ordinal()
- case executor: Executor => executor.getId.hashCode
+ case executor: Executor => executor.getId.hashCode
+ }
}
-}
+
+
}
object ExecutorManager extends Logging {
@@ -196,8 +211,8 @@ object ExecutorManager extends Logging {
}
def getInstance: LabelExecutorManager = {
- if(executorManager == null) synchronized {
- if(executorManager == null) init()
+ if (executorManager == null) synchronized {
+ if (executorManager == null) init()
}
executorManager
}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationExecutorHook.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/hook/ExecutorHook.scala
similarity index 58%
copy from linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationExecutorHook.scala
copy to linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/hook/ExecutorHook.scala
index cae3289..f87d64e 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-computation-engineconn/src/main/scala/org/apache/linkis/engineconn/computation/executor/hook/ComputationExecutorHook.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/core/hook/ExecutorHook.scala
@@ -5,47 +5,55 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
-package org.apache.linkis.engineconn.computation.executor.hook
+
+package org.apache.linkis.engineconn.core.hook
import org.apache.linkis.common.utils.{ClassUtils, Logging, Utils}
-import org.apache.linkis.engineconn.common.creation.EngineCreationContext
-import org.apache.linkis.engineconn.computation.executor.execute.EngineExecutionContext
+import org.apache.linkis.engineconn.executor.entity.Executor
import scala.collection.JavaConverters.asScalaSetConverter
import scala.collection.mutable.ArrayBuffer
-
-trait ComputationExecutorHook {
+trait ExecutorHook extends Logging {
def getOrder(): Int = 1
- def getHookName(): String
+ def getHookName(): String = getClass.getName
+
+ def isAccepted(codeType: String): Boolean
- def beforeExecutorExecute(engineExecutionContext: EngineExecutionContext, engineCreationContext: EngineCreationContext, codeBeforeHook: String): String = codeBeforeHook
+ /*
+ Will be called before executor inited, and only be called once.
+ */
+ def beforeExecutorInit(executor: Executor): Unit = {}
+
+ /*
+ Will be called after executor inited, and only be called once.
+ */
+ def afterExecutorInit(executor: Executor): Unit = {}
}
-object ComputationExecutorHook extends Logging {
+object ExecutorHook extends Logging {
- private lazy val computationExecutorHooks: Array[ComputationExecutorHook] = initComputationExecutorHook
+ private lazy val executorHooks: Array[ExecutorHook] = initExecutorHooks
- private def initComputationExecutorHook: Array[ComputationExecutorHook] = {
- val hooks = new ArrayBuffer[ComputationExecutorHook]
+ private def initExecutorHooks(): Array[ExecutorHook] = {
+ val hooks = new ArrayBuffer[ExecutorHook]
Utils.tryCatch {
val reflections = ClassUtils.reflections
- val allSubClass = reflections.getSubTypesOf(classOf[ComputationExecutorHook])
+ val allSubClass = reflections.getSubTypesOf(classOf[ExecutorHook])
allSubClass.asScala.filter(!ClassUtils.isInterfaceOrAbstract(_)).foreach(l => {
- hooks += l.newInstance
+ hooks += l.newInstance()
})
} {
t: Throwable =>
@@ -54,5 +62,6 @@ object ComputationExecutorHook extends Logging {
hooks.sortWith((a, b) => a.getOrder() <= b.getOrder()).toArray
}
- def getComputationExecutorHooks = computationExecutorHooks
-}
\ No newline at end of file
+ def getAllExecutorHooks(): Array[ExecutorHook] = executorHooks
+
+}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-launch/src/main/scala/org/apache/linkis/engineconn/launch/EngineConnServer.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/launch/EngineConnServer.scala
similarity index 90%
rename from linkis-computation-governance/linkis-engineconn/linkis-engineconn-launch/src/main/scala/org/apache/linkis/engineconn/launch/EngineConnServer.scala
rename to linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/launch/EngineConnServer.scala
index fd256e3..108adb1 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-launch/src/main/scala/org/apache/linkis/engineconn/launch/EngineConnServer.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-core/src/main/scala/org/apache/linkis/engineconn/launch/EngineConnServer.scala
@@ -5,18 +5,19 @@
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
+
package org.apache.linkis.engineconn.launch
+import org.apache.commons.lang.exception.ExceptionUtils
import org.apache.linkis.common.ServiceInstance
import org.apache.linkis.common.conf.{CommonVars, Configuration}
import org.apache.linkis.common.utils.{Logging, Utils}
@@ -34,12 +35,10 @@ import org.apache.linkis.governance.common.utils.EngineConnArgumentsParser
import org.apache.linkis.manager.engineplugin.common.launch.process.Environment
import org.apache.linkis.manager.label.builder.factory.{LabelBuilderFactory, LabelBuilderFactoryContext}
import org.apache.linkis.manager.label.entity.Label
-import org.apache.commons.lang.exception.ExceptionUtils
import scala.collection.JavaConversions._
import scala.collection.mutable.ArrayBuffer
-
object EngineConnServer extends Logging {
@@ -51,23 +50,23 @@ object EngineConnServer extends Logging {
info("<<---------------------EngineConnServer Start --------------------->>")
try {
- // 1. 封装EngineCreationContext
+ // 1. build EngineCreationContext
init(args)
val isTestMode = Configuration.IS_TEST_MODE.getValue(engineCreationContext.getOptions)
- if(isTestMode) {
+ if (isTestMode) {
info(s"Step into test mode, pause 30s if debug is required. If you want to disable test mode, please set ${Configuration.IS_TEST_MODE.key} = false.")
Utils.sleepQuietly(30000)
}
info("Finished to create EngineCreationContext, EngineCreationContext content: " + EngineConnUtils.GSON.toJson(engineCreationContext))
EngineConnHook.getEngineConnHooks.foreach(_.beforeCreateEngineConn(getEngineCreationContext))
info("Finished to execute hook of beforeCreateEngineConn.")
- //2. 创建EngineConn
+ //2. cresate EngineConn
val engineConn = getEngineConnManager.createEngineConn(getEngineCreationContext)
info(s"Finished to create ${engineConn.getEngineConnType}EngineConn.")
EngineConnHook.getEngineConnHooks.foreach(_.beforeExecutionExecute(getEngineCreationContext, engineConn))
info("Finished to execute all hooks of beforeExecutionExecute.")
- //3. 注册的executions 执行
- Utils.tryThrow(executeEngineConn(engineConn)){ t =>
+ //3. register executions
+ Utils.tryThrow(executeEngineConn(engineConn)) { t =>
error(s"Init executors error. Reason: ${ExceptionUtils.getRootCauseMessage(t)}", t)
throw new EngineConnExecutorErrorException(EngineConnExecutorErrorCode.INIT_EXECUTOR_FAILED, "Init executors failed. ", t)
}
@@ -77,22 +76,19 @@ object EngineConnServer extends Logging {
info("Finished to execute hook of afterExecutionExecute")
EngineConnHook.getEngineConnHooks.foreach(_.afterEngineServerStartSuccess(getEngineCreationContext, engineConn))
} catch {
- case t: Throwable =>
- error("EngineConnServer Start Failed", t)
- EngineConnHook.getEngineConnHooks.foreach(_.afterEngineServerStartFailed(getEngineCreationContext, t))
- System.exit(1)
+ case t: Throwable =>
+ error("EngineConnServer Start Failed.", t)
+ EngineConnHook.getEngineConnHooks.foreach(_.afterEngineServerStartFailed(getEngineCreationContext, t))
+ System.exit(1)
}
- //4. 等待Executions执行完毕
+ //4. wait Executions execute
ShutdownHook.getShutdownHook.waitForStopOrError()
info("<<---------------------EngineConnServer Exit --------------------->>")
System.exit(ShutdownHook.getShutdownHook.getExitCode())
}
- /**
- *
- * @param args main函数入参
- */
+
private def init(args: Array[String]): Unit = {
val arguments = EngineConnArgumentsParser.getEngineConnArgumentsParser.parseToObj(args)
val engineConf = arguments.getEngineConnConfMap
@@ -118,14 +114,15 @@ object EngineConnServer extends Logging {
}
private def executeEngineConn(engineConn: EngineConn): Unit = {
- EngineConnExecution.getEngineConnExecutions.foreach{
+ EngineConnExecution.getEngineConnExecutions.foreach {
case execution: AbstractEngineConnExecution =>
info(s"Ready to execute ${execution.getClass.getSimpleName}.")
execution.execute(getEngineCreationContext, engineConn)
- if(execution.returnAfterMeExecuted(getEngineCreationContext, engineConn)) return
+ if (execution.returnAfterMeExecuted(getEngineCreationContext, engineConn)) return
case execution =>
info(s"Ready to execute ${execution.getClass.getSimpleName}.")
- execution.execute(getEngineCreationContext, engineConn)}
+ execution.execute(getEngineCreationContext, engineConn)
+ }
}
def getEngineCreationContext: EngineCreationContext = this.engineCreationContext
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/pom.xml b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/pom.xml
index f03333f..c69c54a 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/pom.xml
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/pom.xml
@@ -38,18 +38,18 @@
<dependency>
<groupId>org.apache.linkis</groupId>
- <artifactId>linkis-engineconn-launch</artifactId>
+ <artifactId>linkis-engineconn-core</artifactId>
<version>${linkis.version}</version>
</dependency>
- <!--<dependency>
+ <dependency>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-scheduler</artifactId>
<version>${linkis.version}</version>
- </dependency>-->
+ </dependency>
<dependency>
<groupId>org.apache.linkis</groupId>
- <artifactId>linkis-message-scheduler</artifactId>
+ <artifactId>linkis-rpc</artifactId>
<version>${linkis.version}</version>
<exclusions>
<exclusion>
@@ -57,7 +57,9 @@
<groupId>org.reflections</groupId>
</exclusion>
</exclusions>
+ <scope>provided</scope>
</dependency>
+
</dependencies>
<build>
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala
index feed3d1..4bea680 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/conf/AccessibleExecutorConfiguration.scala
@@ -32,7 +32,7 @@ object AccessibleExecutorConfiguration {
val ENGINECONN_LOG_SEND_TIME_INTERVAL = CommonVars("wds.linkis.engineconn.log.send.time.interval", 200)
- val ENGINECONN_LOG_SEND_SIZE = CommonVars[Int]("wds.linkis.engineconn.log.send.cache.size", 5)
+ val ENGINECONN_LOG_SEND_SIZE = CommonVars[Int]("wds.linkis.engineconn.log.send.cache.size", 300)
val ENGINECONN_MAX_FREE_TIME = CommonVars("wds.linkis.engineconn.max.free.time", new TimeType("0.5h"))
@@ -43,6 +43,11 @@ object AccessibleExecutorConfiguration {
val ENGINECONN_SUPPORT_PARALLELISM = CommonVars("wds.linkis.engineconn.support.parallelism", false)
- val ENGINECONN_HEARTBEAT_TIME = CommonVars("wds.linkis.engineconn.heartbeat.time", new TimeType("3m"))
+ val ENGINECONN_HEARTBEAT_TIME = CommonVars("wds.linkis.engineconn.heartbeat.time", new TimeType("2m"))
+
+
+ val ENABLE_MAINTAIN = CommonVars("wds.linkis.engineconn.maintain.enable", false)
+
+ val ENABLE_MAINTAIN_CREATORS = CommonVars("wds.linkis.engineconn.maintain.cretors", "IDE")
}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/execution/AccessibleEngineConnExecution.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/execution/AccessibleEngineConnExecution.scala
index 63a303e..057088c 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/execution/AccessibleEngineConnExecution.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/execution/AccessibleEngineConnExecution.scala
@@ -18,7 +18,6 @@
package org.apache.linkis.engineconn.acessible.executor.execution
import java.util.concurrent.TimeUnit
-
import org.apache.linkis.common.utils.{Logging, Utils}
import org.apache.linkis.engineconn.acessible.executor.conf.AccessibleExecutorConfiguration
import org.apache.linkis.engineconn.acessible.executor.entity.AccessibleExecutor
@@ -32,10 +31,11 @@ import org.apache.linkis.engineconn.executor.entity.{Executor, LabelExecutor, Re
import org.apache.linkis.engineconn.executor.listener.ExecutorListenerBusContext
import org.apache.linkis.engineconn.executor.service.ManagerService
import org.apache.linkis.manager.common.entity.enumeration.NodeStatus
-import org.apache.linkis.manager.common.protocol.engine.EngineConnReleaseRequest
+import org.apache.linkis.manager.common.protocol.engine.{ECCanKillRequest, EngineConnReleaseRequest}
import org.apache.linkis.manager.common.protocol.resource.ResourceUsedProtocol
import org.apache.linkis.rpc.Sender
import org.apache.commons.lang.exception.ExceptionUtils
+import org.apache.linkis.manager.label.utils.LabelUtil
class AccessibleEngineConnExecution extends EngineConnExecution with Logging {
@@ -74,37 +74,32 @@ class AccessibleEngineConnExecution extends EngineConnExecution with Logging {
val maxFreeTimeVar = AccessibleExecutorConfiguration.ENGINECONN_MAX_FREE_TIME.getValue(context.getOptions)
val maxFreeTimeStr = maxFreeTimeVar.toString
val maxFreeTime = maxFreeTimeVar.toLong
- info("executorStatusChecker created， maxFreeTimeMills is " + maxFreeTime)
+ logger.info("executorStatusChecker created， maxFreeTimeMills is " + maxFreeTime)
Utils.defaultScheduler.scheduleAtFixedRate(new Runnable {
override def run(): Unit = Utils.tryAndWarn {
val accessibleExecutor = ExecutorManager.getInstance.getReportExecutor match {
case executor: AccessibleExecutor => executor
case executor: Executor =>
- warn(s"Executor(${executor.getId}) is not a AccessibleExecutor, do noting when reached max free time .")
+ logger.warn(s"Executor(${executor.getId}) is not a AccessibleExecutor, do noting when reached max free time .")
return
}
if (NodeStatus.isCompleted(accessibleExecutor.getStatus)) {
error(s"${accessibleExecutor.getId} has completed with status ${accessibleExecutor.getStatus}, now stop it.")
ShutdownHook.getShutdownHook.notifyStop()
} else if (accessibleExecutor.getStatus == NodeStatus.ShuttingDown) {
- warn(s"${accessibleExecutor.getId} is ShuttingDown...")
+ logger.warn(s"${accessibleExecutor.getId} is ShuttingDown...")
ShutdownHook.getShutdownHook.notifyStop()
- } else if (maxFreeTime > 0 && (NodeStatus.Unlock.equals(accessibleExecutor.getStatus) || NodeStatus.Idle.equals(accessibleExecutor.getStatus) )
+ } else if (maxFreeTime > 0 && (NodeStatus.Unlock.equals(accessibleExecutor.getStatus) || NodeStatus.Idle.equals(accessibleExecutor.getStatus))
&& System.currentTimeMillis - accessibleExecutor.getLastActivityTime > maxFreeTime) {
- warn(s"${accessibleExecutor.getId} has not been used for $maxFreeTimeStr, now try to shutdown it.")
- ShutdownHook.getShutdownHook.notifyStop()
- requestManagerReleaseExecutor(" idle release")
- Utils.defaultScheduler.scheduleWithFixedDelay(new Runnable {
- override def run(): Unit = {
- Utils.tryCatch {
- warn(s"Now exit with code ${ShutdownHook.getShutdownHook.getExitCode()}")
- System.exit(ShutdownHook.getShutdownHook.getExitCode())
- } { t =>
- error(s"Exit error : ${ExceptionUtils.getRootCauseMessage(t)}.", t)
- System.exit(-1)
- }
- }
- }, 3000,1000*10, TimeUnit.MILLISECONDS)
+ if (ifECCanMaintain()) {
+ logger.info("ec will not be killed at this time")
+ accessibleExecutor.updateLastActivityTime()
+ } else {
+ logger.warn(s"${accessibleExecutor.getId} has not been used for $maxFreeTimeStr, now try to shutdown it.")
+ requestManagerReleaseExecutor(" idle release")
+ ShutdownHook.getShutdownHook.notifyStop()
+ }
+
}
}
}, 3 * 60 * 1000, AccessibleExecutorConfiguration.ENGINECONN_HEARTBEAT_TIME.getValue.toLong, TimeUnit.MILLISECONDS)
@@ -115,6 +110,37 @@ class AccessibleEngineConnExecution extends EngineConnExecution with Logging {
ManagerService.getManagerService.requestReleaseEngineConn(engineReleaseRequest)
}
+ private def isMaintainSupported(): Boolean = {
+ if (! AccessibleExecutorConfiguration.ENABLE_MAINTAIN.getValue) return false
+ val userCreator = LabelUtil.getUserCreatorLabel(EngineConnObject.getEngineCreationContext.getLabels())
+ if (null != userCreator && AccessibleExecutorConfiguration.ENABLE_MAINTAIN_CREATORS.getValue.contains(userCreator.getCreator)) {
+ logger.info(s"${userCreator.getStringValue} maintain enabled")
+ true
+ } else {
+ false
+ }
+ }
+
+ private def ifECCanMaintain(): Boolean = {
+ if (! isMaintainSupported()) return false
+ val engineTypeLabel = LabelUtil.getEngineTypeLabel(EngineConnObject.getEngineCreationContext.getLabels())
+ val userCreator = LabelUtil.getUserCreatorLabel(EngineConnObject.getEngineCreationContext.getLabels())
+ if (null == engineTypeLabel) return false
+ val ecCanKillRequest = new ECCanKillRequest
+ ecCanKillRequest.setEngineConnInstance(Sender.getThisServiceInstance)
+ ecCanKillRequest.setUser(userCreator.getUser)
+ ecCanKillRequest.setUserCreatorLabel(userCreator)
+ ecCanKillRequest.setEngineTypeLabel(engineTypeLabel)
+ Utils.tryCatch{
+ val response = ManagerService.getManagerService.ecCanKillRequest(ecCanKillRequest)
+ logger.info(s"From manager get $response")
+ ! response.getFlag
+ } { throwable: Throwable =>
+ logger.warn("Failed to ecCanKillRequestManager, will be default exit", throwable)
+ false
+ }
+ }
+
protected def reportUsedResource(executor: Executor, engineCreationContext: EngineCreationContext): Unit = executor match {
case resourceExecutor: ResourceExecutor =>
ManagerService.getManagerService
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/listener/event/TaskEvent.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/listener/event/TaskEvent.scala
index 6be4815..f73203a 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/listener/event/TaskEvent.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/listener/event/TaskEvent.scala
@@ -33,4 +33,6 @@ case class TaskStatusChangedEvent(taskId: String, fromStatus: ExecutionNodeStatu
case class TaskResultCreateEvent(taskId: String, resStr: String, alias: String) extends TaskEvent
-case class TaskResultSizeCreatedEvent(taskId: String, resultSize: Int) extends TaskEvent
\ No newline at end of file
+case class TaskResultSizeCreatedEvent(taskId: String, resultSize: Int) extends TaskEvent
+
+case class TaskResponseErrorEvent(taskId: String, errorMsg: String) extends TaskEvent
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/AccessibleService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/AccessibleService.scala
index 64b6da5..71beaf4 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/AccessibleService.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/AccessibleService.scala
@@ -20,7 +20,7 @@ package org.apache.linkis.engineconn.acessible.executor.service
import org.apache.linkis.engineconn.acessible.executor.listener.ExecutorStatusListener
import org.apache.linkis.manager.common.protocol.engine.EngineSuicideRequest
import org.apache.linkis.manager.common.protocol.node.{RequestNodeStatus, ResponseNodeStatus}
-import org.apache.linkis.message.builder.ServiceMethodContext
+import org.apache.linkis.rpc.Sender
trait AccessibleService extends ExecutorStatusListener {
@@ -30,7 +30,7 @@ trait AccessibleService extends ExecutorStatusListener {
def reStartExecutor: Boolean
- def dealEngineStopRequest(engineSuicideRequest: EngineSuicideRequest, smc: ServiceMethodContext): Unit
+ def dealEngineStopRequest(engineSuicideRequest: EngineSuicideRequest, sender: Sender): Unit
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala
index 993cb9f..600e0d2 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultAccessibleService.scala
@@ -30,15 +30,14 @@ import org.apache.linkis.engineconn.executor.service.ManagerService
import org.apache.linkis.manager.common.entity.enumeration.NodeStatus
import org.apache.linkis.manager.common.protocol.engine.{EngineConnReleaseRequest, EngineSuicideRequest}
import org.apache.linkis.manager.common.protocol.node.{RequestNodeStatus, ResponseNodeStatus}
-import org.apache.linkis.message.annotation.Receiver
-import org.apache.linkis.message.builder.ServiceMethodContext
import org.apache.linkis.rpc.Sender
-
-import javax.annotation.PostConstruct
+import org.apache.linkis.rpc.serializer.annotation.Receiver
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.context.event.{ContextClosedEvent, EventListener}
import org.springframework.stereotype.Service
+import javax.annotation.PostConstruct
+
@Service
class DefaultAccessibleService extends AccessibleService with Logging {
@@ -48,8 +47,10 @@ class DefaultAccessibleService extends AccessibleService with Logging {
private val asyncListenerBusContext = ExecutorListenerBusContext.getExecutorListenerBusContext().getEngineConnAsyncListenerBus
+ private var shutDownHooked: Boolean = false
+
@Receiver
- override def dealEngineStopRequest(engineSuicideRequest: EngineSuicideRequest, smc: ServiceMethodContext): Unit = {
+ override def dealEngineStopRequest(engineSuicideRequest: EngineSuicideRequest, sender: Sender): Unit = {
// todo check user
if (DataWorkCloudApplication.getServiceInstance.equals(engineSuicideRequest.getServiceInstance)) {
stopEngine()
@@ -64,6 +65,7 @@ class DefaultAccessibleService extends AccessibleService with Logging {
}
}
+
@EventListener
def executorShutDownHook(event: ContextClosedEvent): Unit = {
info("executorShutDownHook start to execute.")
@@ -71,13 +73,16 @@ class DefaultAccessibleService extends AccessibleService with Logging {
warn("EngineConn not ready, do not shutdown")
return
}
+ if (shutDownHooked) {
+ warn("had stop, do not shutdown")
+ return
+ }
var executor: Executor = ExecutorManager.getInstance.getReportExecutor
- if (null != executor){
+ if (null != executor) {
Utils.tryAndWarn{
executor.tryShutdown()
}
warn(s"Engine : ${Sender.getThisInstance} with state has stopped successfully.")
-
} else {
executor = SensibleExecutor.getDefaultErrorSensibleExecutor
}
@@ -87,13 +92,16 @@ class DefaultAccessibleService extends AccessibleService with Logging {
}
executorHeartbeatService.reportHeartBeatMsg(executor)
info("Reported status shuttingDown to manager.")
- Thread.sleep(2000)
+ Utils.tryQuietly(Thread.sleep(2000))
+ shutDownHooked = true
+ ShutdownHook.getShutdownHook.notifyStop()
}
override def stopExecutor: Unit = {
// todo
}
+
override def pauseExecutor: Unit = {
}
@@ -114,9 +122,7 @@ class DefaultAccessibleService extends AccessibleService with Logging {
}
}
- /**
- * service 需要加定时任务判断Executor是否空闲很久，然后调用该方法进行释放
- */
+
override def requestManagerReleaseExecutor(msg: String): Unit = {
val engineReleaseRequest = new EngineConnReleaseRequest(Sender.getThisServiceInstance, Utils.getJvmUser, msg, EngineConnObject.getEngineCreationContext.getTicketId)
@@ -127,10 +133,10 @@ class DefaultAccessibleService extends AccessibleService with Logging {
override def dealRequestNodeStatus(requestNodeStatus: RequestNodeStatus): ResponseNodeStatus = {
val status = if (EngineConnObject.isReady) {
ExecutorManager.getInstance.getReportExecutor match {
- case executor: SensibleExecutor =>
- executor.getStatus
- case _ => NodeStatus.Starting
- }
+ case executor: SensibleExecutor =>
+ executor.getStatus
+ case _ => NodeStatus.Starting
+ }
} else {
NodeStatus.Starting
}
@@ -162,4 +168,3 @@ class DefaultAccessibleService extends AccessibleService with Logging {
}
}
-
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultExecutorHeartbeatService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultExecutorHeartbeatService.scala
index ade5afc..6b4205b 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultExecutorHeartbeatService.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultExecutorHeartbeatService.scala
@@ -31,7 +31,7 @@ import org.apache.linkis.engineconn.executor.listener.ExecutorListenerBusContext
import org.apache.linkis.engineconn.executor.service.ManagerService
import org.apache.linkis.manager.common.entity.enumeration.NodeStatus
import org.apache.linkis.manager.common.protocol.node.{NodeHeartbeatMsg, NodeHeartbeatRequest}
-import org.apache.linkis.message.annotation.Receiver
+import org.apache.linkis.rpc.serializer.annotation.Receiver
import org.apache.linkis.rpc.Sender
import javax.annotation.PostConstruct
import org.springframework.beans.factory.annotation.Autowired
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultManagerService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultManagerService.scala
index 3039cd0..3e280d0 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultManagerService.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultManagerService.scala
@@ -18,12 +18,11 @@
package org.apache.linkis.engineconn.acessible.executor.service
import java.util
-
import org.apache.linkis.common.utils.Logging
import org.apache.linkis.engineconn.executor.service.ManagerService
import org.apache.linkis.governance.common.conf.GovernanceCommonConf
import org.apache.linkis.manager.common.entity.enumeration.NodeStatus
-import org.apache.linkis.manager.common.protocol.engine.EngineConnReleaseRequest
+import org.apache.linkis.manager.common.protocol.engine.{ECCanKillRequest, ECCanKillResponse, EngineConnReleaseRequest}
import org.apache.linkis.manager.common.protocol.label.LabelReportRequest
import org.apache.linkis.manager.common.protocol.node.{NodeHeartbeatMsg, ResponseNodeStatus}
import org.apache.linkis.manager.common.protocol.resource.ResourceUsedProtocol
@@ -79,4 +78,10 @@ class DefaultManagerService extends ManagerService with Logging{
getManagerSender.send(resourceUsedProtocol)
}
+ override def ecCanKillRequest(ecCanKillRequest: ECCanKillRequest): ECCanKillResponse = {
+ getManagerSender.ask(ecCanKillRequest) match {
+ case ecCanKillResponse: ECCanKillResponse => ecCanKillResponse
+ case _ => new ECCanKillResponse()
+ }
+ }
}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala
index 35e723c..2537b04 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/DefaultOperateService.scala
@@ -20,7 +20,7 @@ import org.apache.commons.lang.exception.ExceptionUtils
import org.apache.linkis.common.utils.{Logging, Utils}
import org.apache.linkis.manager.common.operator.OperatorFactory
import org.apache.linkis.manager.common.protocol.engine.{EngineOperateRequest, EngineOperateResponse}
-import org.apache.linkis.message.annotation.Receiver
+import org.apache.linkis.rpc.serializer.annotation.Receiver
import org.springframework.stereotype.Service
import scala.collection.JavaConverters.mapAsScalaMapConverter
@@ -31,7 +31,7 @@ class DefaultOperateService extends OperateService with Logging {
@Receiver
override def executeOperation(engineOperateRequest: EngineOperateRequest): EngineOperateResponse = {
val parameters = engineOperateRequest.parameters.asScala.toMap
- val operator = Utils.tryCatch(OperatorFactory().getOperatorRequest(parameters)){ t =>
+ val operator = Utils.tryCatch(OperatorFactory().getOperatorRequest(parameters)) { t =>
error(s"Get operator failed, parameters is ${engineOperateRequest.parameters}.", t)
return EngineOperateResponse(Map.empty, true, ExceptionUtils.getRootCauseMessage(t))
}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala
index 5d13cbc..02d5b3a 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/accessible-executor/src/main/scala/org/apache/linkis/engineconn/acessible/executor/service/EngineConnTimedLockService.scala
@@ -28,7 +28,7 @@ import org.apache.linkis.governance.common.exception.engineconn.{EngineConnExecu
import org.apache.linkis.manager.common.entity.enumeration.NodeStatus
import org.apache.linkis.manager.common.protocol.engine.EngineLockType
import org.apache.linkis.manager.common.protocol.{RequestEngineLock, RequestEngineUnlock, ResponseEngineLock, ResponseEngineUnlock}
-import org.apache.linkis.message.annotation.Receiver
+import org.apache.linkis.rpc.serializer.annotation.Receiver
import org.apache.linkis.server.BDPJettyServerHelper
import org.apache.commons.lang.StringUtils
@@ -109,7 +109,7 @@ class EngineConnTimedLockService extends LockService with Logging {
})
} else {
error("No valid executors while adding lock.")
- accessibleExecutor.transition(NodeStatus.Idle)
+ accessibleExecutor.transition(NodeStatus.Idle)
}
ExecutorListenerBusContext.getExecutorListenerBusContext().getEngineConnAsyncListenerBus.post(ExecutorLockEvent(accessibleExecutor, lockString))
Some(lockString)
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/pom.xml b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/pom.xml
index 5c42d53..ef9c4f6 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/pom.xml
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/pom.xml
@@ -37,8 +37,9 @@
<dependency>
<groupId>org.apache.linkis</groupId>
- <artifactId>linkis-message-scheduler</artifactId>
+ <artifactId>linkis-rpc</artifactId>
<version>${linkis.version}</version>
+ <scope>provided</scope>
</dependency>
<dependency>
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala
index d07e6ba..6cfd255 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/callback-service/src/main/scala/org/apache/linkis/engineconn/callback/service/EngineConnCallback.scala
@@ -42,13 +42,13 @@ abstract class AbstractEngineConnStartUpCallback(emInstance: ServiceInstance) ex
def callback(protocol: RequestProtocol): Unit = {
protocol match {
case protocol: EngineConnStatusCallback => {
- if(protocol.status.equals(NodeStatus.Failed)){
- error(s"protocol will send to em: ${protocol}")
- }else{
- info(s"protocol will send to em: ${protocol}")
+ if (protocol.status.equals(NodeStatus.Failed)) {
+ logger.error(s"protocol will send to em: ${protocol}")
+ } else {
+ logger.info(s"protocol will send to em: ${protocol}")
}
}
- case _=>
+ case _ =>
}
getEMSender.send(protocol)
}
diff --git a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorCode.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/entity/ResourceFetchExecutor.scala
similarity index 77%
copy from linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorCode.scala
copy to linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/entity/ResourceFetchExecutor.scala
index 8302e45..41641ad 100644
--- a/linkis-computation-governance/linkis-engineconn-manager/linkis-engineconn-manager-server/src/main/scala/org/apache/linkis/ecm/server/exception/ECMErrorCode.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/entity/ResourceFetchExecutor.scala
@@ -15,16 +15,14 @@
* limitations under the License.
*/
-package org.apache.linkis.ecm.server.exception
+package org.apache.linkis.engineconn.executor.entity
-object ECMErrorCode {
+import org.apache.linkis.manager.common.protocol.resource.ResourceWithStatus
- val EC_START_TIME_OUT = 11100
+import java.util
- val EC_INTERRUPT_TIME_OUT = 11101
+trait ResourceFetchExecutor {
- val EC_START_FAILED = 11102
-
- val EC_FETCH_LOG_FAILED = 11110
+ def FetchResource: util.HashMap[String, ResourceWithStatus]
}
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/service/ManagerService.scala b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/service/ManagerService.scala
index 00b1563..e0312ef 100644
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/service/ManagerService.scala
+++ b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-executor/executor-core/src/main/scala/org/apache/linkis/engineconn/executor/service/ManagerService.scala
@@ -20,7 +20,7 @@ package org.apache.linkis.engineconn.executor.service
import org.apache.linkis.common.utils.Utils
import org.apache.linkis.engineconn.executor.conf.EngineConnExecutorConfiguration
import org.apache.linkis.manager.common.entity.enumeration.NodeStatus
-import org.apache.linkis.manager.common.protocol.engine.EngineConnReleaseRequest
+import org.apache.linkis.manager.common.protocol.engine.{ECCanKillRequest, ECCanKillResponse, EngineConnReleaseRequest}
import org.apache.linkis.manager.common.protocol.node.NodeHeartbeatMsg
import org.apache.linkis.manager.common.protocol.resource.ResourceUsedProtocol
import org.apache.linkis.manager.label.entity.Label
@@ -37,6 +37,8 @@ trait ManagerService {
def reportUsedResource(resourceUsedProtocol: ResourceUsedProtocol): Unit
+ def ecCanKillRequest(ecCanKillRequest: ECCanKillRequest): ECCanKillResponse
+
}
object ManagerService {
diff --git a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-launch/pom.xml b/linkis-computation-governance/linkis-engineconn/linkis-engineconn-launch/pom.xml
deleted file mode 100644
index 5447138..0000000
--- a/linkis-computation-governance/linkis-engineconn/linkis-engineconn-launch/pom.xml
+++ /dev/null
@@ -1,73 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<project xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xmlns="http://maven.apache.org/POM/4.0.0"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <parent>
- <artifactId>linkis</artifactId>
- <groupId>org.apache.linkis</groupId>
- <version>1.0.3</version>
- <relativePath>../../../pom.xml</relativePath>
- </parent>
- <modelVersion>4.0.0</modelVersion>
-
- <artifactId>linkis-engineconn-launch</artifactId>
-
- <dependencies>
-
- <dependency>
- <groupId>org.apache.linkis</groupId>
- <artifactId>linkis-engineconn-core</artifactId>
- <version>${linkis.version}</version>
- </dependency>
-
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-deploy-plugin</artifactId>
- </plugin>
-
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <!--<excludes>-->
- <!--<exclude>**/*.yml</exclude>-->
- <!--<exclude>**/*.properties</exclude>-->
- <!--<exclude>**/*.sh</exclude>-->
- <!--</excludes>-->
- </configuration>
- </plugin>
- </plugins>
- <resources>
- <resource>
- <directory>src/main/resources</directory>
- </resource>
- </resources>
- <finalName>${project.artifactId}-${project.version}</finalName>
- </build>
-
-</project>
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-engineconn/pom.xml b/linkis-computation-governance/linkis-engineconn/pom.xml
index e2ba47a..b6fa1e7 100644
--- a/linkis-computation-governance/linkis-engineconn/pom.xml
+++ b/linkis-computation-governance/linkis-engineconn/pom.xml
@@ -32,7 +32,6 @@
<module>linkis-engineconn-common</module>
<module>linkis-engineconn-core</module>
<module>linkis-engineconn-executor/executor-core</module>
- <module>linkis-engineconn-launch</module>
<module>linkis-engineconn-executor/accessible-executor</module>
<module>linkis-engineconn-executor/resource-executor</module>
<module>linkis-engineconn-executor/callback-service</module>
diff --git a/linkis-computation-governance/linkis-entrance-client/pom.xml b/linkis-computation-governance/linkis-entrance-client/pom.xml
deleted file mode 100644
index 54aa4a4..0000000
--- a/linkis-computation-governance/linkis-entrance-client/pom.xml
+++ /dev/null
@@ -1,77 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <artifactId>linkis</artifactId>
- <groupId>org.apache.linkis</groupId>
- <version>1.0.3</version>
- </parent>
-
- <artifactId>linkis-entrance-client</artifactId>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.linkis</groupId>
- <artifactId>linkis-entrance</artifactId>
- <version>${linkis.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.apache.linkis</groupId>
- <artifactId>linkis-io-entrance</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.linkis</groupId>
- <artifactId>linkis-rpc</artifactId>
- <version>${linkis.version}</version>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-deploy-plugin</artifactId>
- </plugin>
-
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- </plugin>
- </plugins>
- <resources>
- <resource>
- <directory>${basedir}/src/main/resources</directory>
- </resource>
- </resources>
- <finalName>${project.artifactId}-${project.version}</finalName>
- </build>
-
-
-</project>
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/ClientEngineBuilderBeanAnnotation.java b/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/ClientEngineBuilderBeanAnnotation.java
deleted file mode 100644
index 276b4b0..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/ClientEngineBuilderBeanAnnotation.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient.annotation;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.annotation.Bean;
-import org.springframework.core.annotation.AliasFor;
-import org.springframework.stereotype.Component;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Target({ElementType.METHOD, ElementType.TYPE})
-@Retention(RetentionPolicy.RUNTIME)
-@Bean(value = ClientEngineBuilderBeanAnnotation.BEAN_NAME)
-@Component(value = ClientEngineBuilderBeanAnnotation.BEAN_NAME)
-public @interface ClientEngineBuilderBeanAnnotation {
- String BEAN_NAME = "clientEngineBuilder";
- @AliasFor(annotation = Component.class)
- String value() default BEAN_NAME;
-
- @Target({ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE, ElementType.PARAMETER})
- @Retention(RetentionPolicy.RUNTIME)
- @Qualifier(BEAN_NAME)
- @Autowired
- @interface ClientEngineBuilderAutowiredAnnotation {
- @AliasFor(annotation = Qualifier.class)
- String value() default BEAN_NAME;
- @AliasFor(annotation = Autowired.class)
- boolean required() default true;
- }
-}
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/ClientEngineRequesterBeanAnnotation.java b/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/ClientEngineRequesterBeanAnnotation.java
deleted file mode 100644
index 789ed22..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/ClientEngineRequesterBeanAnnotation.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient.annotation;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.annotation.Bean;
-import org.springframework.core.annotation.AliasFor;
-import org.springframework.stereotype.Component;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Target({ElementType.METHOD, ElementType.TYPE})
-@Retention(RetentionPolicy.RUNTIME)
-@Bean(value = ClientEngineRequesterBeanAnnotation.BEAN_NAME)
-@Component(value = ClientEngineRequesterBeanAnnotation.BEAN_NAME)
-public @interface ClientEngineRequesterBeanAnnotation {
- String BEAN_NAME = "clientEngineRequester";
- @AliasFor(annotation = Component.class)
- String value() default BEAN_NAME;
-
- @Target({ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE, ElementType.PARAMETER})
- @Retention(RetentionPolicy.RUNTIME)
- @Qualifier(BEAN_NAME)
- @Autowired
- @interface ClientEngineRequesterAutowiredAnnotation {
- @AliasFor(annotation = Qualifier.class)
- String value() default BEAN_NAME;
- @AliasFor(annotation = Autowired.class)
- boolean required() default true;
- }
-}
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/ClientEngineSelectorBeanAnnotation.java b/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/ClientEngineSelectorBeanAnnotation.java
deleted file mode 100644
index 97460a5..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/ClientEngineSelectorBeanAnnotation.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient.annotation;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.annotation.Bean;
-import org.springframework.core.annotation.AliasFor;
-import org.springframework.stereotype.Component;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Target({ElementType.METHOD, ElementType.TYPE})
-@Retention(RetentionPolicy.RUNTIME)
-@Bean(value = ClientEngineSelectorBeanAnnotation.BEAN_NAME)
-@Component(value = ClientEngineSelectorBeanAnnotation.BEAN_NAME)
-public @interface ClientEngineSelectorBeanAnnotation {
- String BEAN_NAME = "clientEngineSelector";
- @AliasFor(annotation = Component.class)
- String value() default BEAN_NAME;
-
- @Target({ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE, ElementType.PARAMETER})
- @Retention(RetentionPolicy.RUNTIME)
- @Qualifier(BEAN_NAME)
- @Autowired
- @interface ClientEngineSelectorAutowiredAnnotation {
- @AliasFor(annotation = Qualifier.class)
- String value() default BEAN_NAME;
- @AliasFor(annotation = Autowired.class)
- boolean required() default true;
- }
-}
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/ClientEntranceParserBeanAnnotation.java b/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/ClientEntranceParserBeanAnnotation.java
deleted file mode 100644
index 885c808..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/ClientEntranceParserBeanAnnotation.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient.annotation;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.annotation.Bean;
-import org.springframework.core.annotation.AliasFor;
-import org.springframework.stereotype.Component;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Target({ElementType.METHOD, ElementType.TYPE})
-@Retention(RetentionPolicy.RUNTIME)
-@Bean(value = ClientEntranceParserBeanAnnotation.BEAN_NAME)
-@Component(value = ClientEntranceParserBeanAnnotation.BEAN_NAME)
-public @interface ClientEntranceParserBeanAnnotation {
- String BEAN_NAME = "clientEntranceParser";
- @AliasFor(annotation = Component.class)
- String value() default BEAN_NAME;
-
- @Target({ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE, ElementType.PARAMETER})
- @Retention(RetentionPolicy.RUNTIME)
- @Qualifier(BEAN_NAME)
- @Autowired
- @interface ClientEntranceParserAutowiredAnnotation {
- @AliasFor(annotation = Qualifier.class)
- String value() default BEAN_NAME;
- @AliasFor(annotation = Autowired.class)
- boolean required() default true;
- }
-}
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/ClientGroupFactoryBeanAnnotation.java b/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/ClientGroupFactoryBeanAnnotation.java
deleted file mode 100644
index 854cce3..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/ClientGroupFactoryBeanAnnotation.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient.annotation;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.annotation.Bean;
-import org.springframework.core.annotation.AliasFor;
-import org.springframework.stereotype.Component;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Target({ElementType.METHOD, ElementType.TYPE})
-@Retention(RetentionPolicy.RUNTIME)
-@Bean(value = ClientGroupFactoryBeanAnnotation.BEAN_NAME)
-@Component(value = ClientGroupFactoryBeanAnnotation.BEAN_NAME)
-public @interface ClientGroupFactoryBeanAnnotation {
- String BEAN_NAME = "clientGroupFactory";
- @AliasFor(annotation = Component.class)
- String value() default BEAN_NAME;
-
- @Target({ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE, ElementType.PARAMETER})
- @Retention(RetentionPolicy.RUNTIME)
- @Qualifier(BEAN_NAME)
- @Autowired
- @interface ClientGroupFactoryAutowiredAnnotation {
- @AliasFor(annotation = Qualifier.class)
- String value() default BEAN_NAME;
- @AliasFor(annotation = Autowired.class)
- boolean required() default true;
- }
-}
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/ClientInterceptorsBeanAnnotation.java b/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/ClientInterceptorsBeanAnnotation.java
deleted file mode 100644
index fbd575e..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/ClientInterceptorsBeanAnnotation.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient.annotation;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.annotation.Bean;
-import org.springframework.core.annotation.AliasFor;
-import org.springframework.stereotype.Component;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Target({ElementType.METHOD, ElementType.TYPE})
-@Retention(RetentionPolicy.RUNTIME)
-@Bean(value = ClientInterceptorsBeanAnnotation.BEAN_NAME)
-@Component(value = ClientInterceptorsBeanAnnotation.BEAN_NAME)
-public @interface ClientInterceptorsBeanAnnotation {
- String BEAN_NAME = "clientInterceptors";
- @AliasFor(annotation = Component.class)
- String value() default BEAN_NAME;
-
- @Target({ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE, ElementType.PARAMETER})
- @Retention(RetentionPolicy.RUNTIME)
- @Qualifier(BEAN_NAME)
- @Autowired
- @interface ClientInterceptorsAutowiredAnnotation {
- @AliasFor(annotation = Qualifier.class)
- String value() default BEAN_NAME;
- @AliasFor(annotation = Autowired.class)
- boolean required() default true;
- }
-}
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/DefaultEntranceClientBeanAnnotation.java b/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/DefaultEntranceClientBeanAnnotation.java
deleted file mode 100644
index 85353f3..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/annotation/DefaultEntranceClientBeanAnnotation.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient.annotation;
-
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.beans.factory.annotation.Qualifier;
-import org.springframework.context.annotation.Bean;
-import org.springframework.core.annotation.AliasFor;
-import org.springframework.stereotype.Component;
-
-import java.lang.annotation.ElementType;
-import java.lang.annotation.Retention;
-import java.lang.annotation.RetentionPolicy;
-import java.lang.annotation.Target;
-
-@Target({ElementType.METHOD, ElementType.TYPE})
-@Retention(RetentionPolicy.RUNTIME)
-@Bean(value = DefaultEntranceClientBeanAnnotation.BEAN_NAME)
-@Component(value = DefaultEntranceClientBeanAnnotation.BEAN_NAME)
-public @interface DefaultEntranceClientBeanAnnotation {
- String BEAN_NAME = "entranceClient";
- @AliasFor(annotation = Component.class)
- String value() default BEAN_NAME;
-
- @Target({ElementType.CONSTRUCTOR, ElementType.FIELD, ElementType.METHOD, ElementType.TYPE, ElementType.PARAMETER})
- @Retention(RetentionPolicy.RUNTIME)
- @Qualifier(BEAN_NAME)
- @Autowired
- @interface DefaultEntranceClientAutowiredAnnotation {
- @AliasFor(annotation = Qualifier.class)
- String value() default BEAN_NAME;
- @AliasFor(annotation = Autowired.class)
- boolean required() default true;
- }
-}
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/conf/ClientSpringConfiguration.java b/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/conf/ClientSpringConfiguration.java
deleted file mode 100644
index 8088e9d..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/java/org/apache/linkis/entranceclient/conf/ClientSpringConfiguration.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient.conf;
-
-import org.apache.linkis.DataWorkCloudApplication;
-import org.apache.linkis.common.conf.CommonVars;
-import org.apache.linkis.entrance.conf.EntranceSpringConfiguration;
-import org.apache.linkis.entrance.interceptor.EntranceInterceptor;
-import org.apache.linkis.entrance.scheduler.EntranceSchedulerContext;
-import org.apache.linkis.entranceclient.EntranceClient;
-import org.apache.linkis.entranceclient.EntranceClientImpl;
-import org.apache.linkis.entranceclient.annotation.ClientEntranceParserBeanAnnotation;
-import org.apache.linkis.entranceclient.annotation.ClientInterceptorsBeanAnnotation;
-import org.apache.linkis.entranceclient.annotation.DefaultEntranceClientBeanAnnotation;
-import org.apache.linkis.entranceclient.context.ClientEntranceParser;
-import org.apache.linkis.rpc.RPCReceiveRestful;
-import org.apache.linkis.rpc.conf.RPCConfiguration;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.autoconfigure.AutoConfigureBefore;
-import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
-import org.springframework.context.annotation.Conditional;
-import org.springframework.context.annotation.Configuration;
-
-@Configuration
-@AutoConfigureBefore({EntranceSpringConfiguration.class, RPCReceiveRestful.class})
-class ClientSpringConfiguration {
-
- private Logger logger = LoggerFactory.getLogger(getClass());
-
- {
- logger.info("start a multi-entrance application...");
- }
-
- @ClientEntranceParserBeanAnnotation
- @ConditionalOnMissingBean(name = {ClientEntranceParserBeanAnnotation.BEAN_NAME})
- public ClientEntranceParser generateClientEntranceParser() {
- return new ClientEntranceParser();
- }
-
-
- @ClientInterceptorsBeanAnnotation
- @ConditionalOnMissingBean(name = {ClientInterceptorsBeanAnnotation.BEAN_NAME})
- public EntranceInterceptor[] generateEntranceInterceptor() {
- return new EntranceInterceptor[0];
- }
-
-
- @DefaultEntranceClientBeanAnnotation
- @Conditional(MultiEntranceCondition.class)
- public EntranceClient generateEntranceClient(@ClientEntranceParserBeanAnnotation.ClientEntranceParserAutowiredAnnotation ClientEntranceParser clientEntranceParser,
- @ClientInterceptorsBeanAnnotation.ClientInterceptorsAutowiredAnnotation EntranceInterceptor[] clientInterceptors,
- @Autowired EntranceSchedulerContext entranceSchedulerContext) {
- logger.warn("try to check the rpc receiver consumer threadPool...");
- setClientValue(RPCConfiguration.BDP_RPC_RECEIVER_ASYN_QUEUE_CAPACITY(), ClientConfiguration.BDP_RPC_RECEIVER_ASYN_QUEUE_CAPACITY_FOR_CLIENT());
- setClientValue(RPCConfiguration.BDP_RPC_RECEIVER_ASYN_CONSUMER_THREAD_MAX(), ClientConfiguration.BDP_RPC_RECEIVER_ASYN_CONSUMER_THREAD_MAX_FOR_CLIENT());
- EntranceClientImpl client = EntranceClientImpl.apply(ClientConfiguration.CLIENT_DEFAULT_NAME());
- logger.warn("Multi-entrance application is ready to initial EntranceClient " + client.getEntranceClientName());
- client.init(clientEntranceParser, entranceSchedulerContext, clientInterceptors, ClientConfiguration.CLIENT_DEFAULT_PARALLELISM_USERS().getValue());
- return client;
- }
-
-// @EventListener
-// public void onApplicationStarting(ApplicationStartingEvent applicationStartingEvent) {
-// if(MultiEntranceCondition.isMultiEntranceApplication()) {
-// logger.warn("start a multi-entrance application, now try to check the rpc receiver consumer threadPool...");
-// setClientValue(RPCConfiguration.BDP_RPC_RECEIVER_ASYN_QUEUE_CAPACITY(), ClientConfiguration.BDP_RPC_RECEIVER_ASYN_QUEUE_CAPACITY_FOR_CLIENT());
-// setClientValue(RPCConfiguration.BDP_RPC_RECEIVER_ASYN_CONSUMER_THREAD_MAX(), ClientConfiguration.BDP_RPC_RECEIVER_ASYN_CONSUMER_THREAD_MAX_FOR_CLIENT());
-// setClientValue(EntranceConfiguration.CONCURRENT_ENGINE_MAX_PARALLELISM(), ClientConfiguration.CONCURRENT_ENGINE_MAX_PARALLELISM_FOR_CLIENT());
-// }
-// }
-
- private void setClientValue(CommonVars<?> fromConf, CommonVars<Integer> toConf) {
- int queueSize = Integer.parseInt(fromConf.getValue().toString());
- int queueSizeValue = toConf.getValue();
- if (queueSize < queueSizeValue) {
- String key = fromConf.key();
- DataWorkCloudApplication.setProperty(key, String.valueOf(queueSizeValue));
- logger.warn("Multi-entrance application set " + key + "=" + queueSizeValue);
- }
- }
-}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/resources/META-INF/spring.factories b/linkis-computation-governance/linkis-entrance-client/src/main/resources/META-INF/spring.factories
deleted file mode 100644
index 1607e90..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/resources/META-INF/spring.factories
+++ /dev/null
@@ -1,20 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
-org.apache.linkis.entranceclient.conf.ClientForEntranceSpringConfiguration,\
-org.apache.linkis.entranceclient.conf.ClientSpringConfiguration
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/EngineApplicationNameFactory.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/EngineApplicationNameFactory.scala
deleted file mode 100644
index e399ffb..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/EngineApplicationNameFactory.scala
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient
-
-import org.apache.linkis.entranceclient.conf.ClientConfiguration.{CLIENT_ENGINE_MANAGER_SPRING_APPLICATION_NAME, CLIENT_ENGINE_SPRING_APPLICATION_NAME}
-
-trait EngineApplicationNameFactory {
- private var engineApplicationName: String = CLIENT_ENGINE_SPRING_APPLICATION_NAME.getValue
-
- def setEngineApplicationName(engineApplicationName: String): Unit = this.engineApplicationName = engineApplicationName
- def getEngineApplicationName: String = engineApplicationName
-
- def isEngineApplicationNameEmpty: Boolean = engineApplicationName == CLIENT_ENGINE_SPRING_APPLICATION_NAME.getValue
-
-}
-trait EngineManagerApplicationNameFactory {
- private var engineManagerApplicationName: String = CLIENT_ENGINE_MANAGER_SPRING_APPLICATION_NAME.getValue
- def setEngineManagerApplicationName(engineManagerApplicationName: String): Unit = this.engineManagerApplicationName = engineManagerApplicationName
- def getEngineManagerApplicationName: String = engineManagerApplicationName
- def isEngineManagerApplicationNameEmpty: Boolean = engineManagerApplicationName == CLIENT_ENGINE_MANAGER_SPRING_APPLICATION_NAME.getValue
-}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/EntranceClient.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/EntranceClient.scala
deleted file mode 100644
index 344b2f5..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/EntranceClient.scala
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient
-
-import org.apache.linkis.entranceclient.execute.ClientJob
-
-trait EntranceClient {
-
- def getEntranceClientName: String
-
- def execute(code: String, user: String, creator: String): Boolean
-
- def execute(code: String, user: String, creator: String,
- params: java.util.Map[String, Any]): Boolean
-
- def executeJob(code: String, user: String, creator: String): String
-
- def executeJob(code: String, user: String, creator: String,
- params: java.util.Map[String, Any]): String
-
- def getJob(jobId: String): Option[ClientJob]
-
- def executeResult(code: String, user: String, creator: String): Array[String]
-
- def executeResult(code: String, user: String, creator: String,
- params: java.util.Map[String, Any]): Array[String]
-
-}
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/EntranceClientImpl.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/EntranceClientImpl.scala
deleted file mode 100644
index f29a2b0..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/EntranceClientImpl.scala
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient
-
-import java.util
-
-import org.apache.linkis.common.utils.Logging
-import org.apache.linkis.entrance.context.DefaultEntranceContext
-import org.apache.linkis.entrance.exception.EntranceErrorException
-import org.apache.linkis.entrance.interceptor.EntranceInterceptor
-import org.apache.linkis.entrance.scheduler.EntranceSchedulerContext
-import org.apache.linkis.entrance.server.DefaultEntranceServer
-import org.apache.linkis.entrance.{EntranceContext, EntranceServer}
-import org.apache.linkis.entranceclient.execute._
-import org.apache.linkis.protocol.constants.TaskConstant
-import org.apache.linkis.scheduler.queue.parallelqueue.ParallelScheduler
-import org.apache.linkis.server.JMap
-import org.apache.commons.lang.StringUtils
-
-class EntranceClientImpl private() extends EntranceClient with EngineApplicationNameFactory
- with EngineManagerApplicationNameFactory with Logging {
-
- private var entranceServer: EntranceServer = _
- private var clientName: String = _
-
-
- def setClientName(clientName: String): Unit = this.clientName = clientName
-
-
- def init(entranceServer: EntranceServer): Unit = {
- this.entranceServer = entranceServer
- }
-
- def init(entranceContext: EntranceContext): Unit = init(new DefaultEntranceServer(entranceContext))
-
- def init(clientEntranceParser: context.ClientEntranceParser, schedulerContext: EntranceSchedulerContext, interceptors: Array[EntranceInterceptor],
- maxParallelismUsers: Int): Unit = if (entranceServer == null) synchronized {
- if (entranceServer == null) {
- val scheduler = new ParallelScheduler(schedulerContext)
- scheduler.init()
- scheduler.start()
- val entranceContext = new DefaultEntranceContext(clientEntranceParser, new context.ClientPersistenceManager, new context.ClientLogManager,
- scheduler, interceptors, null, Array.empty)
- init(entranceContext)
- }
- }
-
- private def executeUntil[T](code: String, user: String, creator: String,
- params: java.util.Map[String, Any], op: ClientJob => T): T = {
- val execId = executeJob(code, user, creator, params)
- entranceServer.getJob(execId).foreach {
- case job: ClientJob =>
- job.waitForComplete()
- return op(job)
- }
- throw new EntranceErrorException(50003, s"execute failed, cannot find the job $execId.")
- }
-
- override def getEntranceClientName: String = if(StringUtils.isNotEmpty(clientName)) clientName else entranceServer.getName
-
- override def execute(code: String, user: String, creator: String): Boolean = execute(code, user, creator, null)
-
- override def execute(code: String, user: String, creator: String,
- params: java.util.Map[String, Any]): Boolean =
- executeUntil(code, user, creator, params, job =>
- if (job.isSucceed) true else false)
-
- override def executeJob(code: String, user: String, creator: String): String = executeJob(code, user, creator, null)
-
- override def executeJob(code: String, user: String, creator: String, params: util.Map[String, Any]): String = {
- val requestMap = new JMap[String, Any]
- requestMap.put(TaskConstant.EXECUTIONCODE, code)
- requestMap.put(TaskConstant.UMUSER, user)
- requestMap.put(TaskConstant.REQUESTAPPLICATIONNAME, creator)
- if(params != null && !params.isEmpty) {
- if(params.containsKey(EntranceServer.DO_NOT_PRINT_PARAMS_LOG)) {
- requestMap.put(EntranceServer.DO_NOT_PRINT_PARAMS_LOG, params.get(EntranceServer.DO_NOT_PRINT_PARAMS_LOG))
- params.remove(EntranceServer.DO_NOT_PRINT_PARAMS_LOG)
- }
- if(!params.isEmpty) requestMap.put(TaskConstant.PARAMS, params)
- }
- entranceServer.execute(requestMap)
- }
-
- override def getJob(jobId: String): Option[ClientJob] = entranceServer.getJob(jobId).map { case job: ClientJob => job }
-
- override def executeResult(code: String, user: String, creator: String): Array[String] =
- executeResult(code, user, creator, null)
-
- override def executeResult(code: String, user: String, creator: String,
- params: java.util.Map[String, Any]): Array[String] =
- executeUntil(code, user, creator, params, job =>
- if(job.isSucceed) job.getResultSets
- else if(job.getErrorResponse != null) {
- val exception = new EntranceErrorException(22001, job.getErrorResponse.message)
- if(job.getErrorResponse.t != null) exception.initCause(job.getErrorResponse.t)
- throw exception
- } else throw new EntranceErrorException(22002, "execute failed, unknown reason."))
-
-}
-object EntranceClientImpl {
-
- private val clientNameToEntranceClient = new JMap[String, EntranceClientImpl]
-
- def apply(clientName: String): EntranceClientImpl = {
- if(!clientNameToEntranceClient.containsKey(clientName)) synchronized {
- if(!clientNameToEntranceClient.containsKey(clientName)) {
- val client = new EntranceClientImpl
- client.setClientName(clientName)
- clientNameToEntranceClient.put(clientName, client)
- }
- }
- clientNameToEntranceClient.get(clientName)
- }
-
- def getClientNames = clientNameToEntranceClient.keySet()
-}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/conf/ClientConfiguration.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/conf/ClientConfiguration.scala
deleted file mode 100644
index 2524a6e..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/conf/ClientConfiguration.scala
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient.conf
-
-import org.apache.linkis.common.conf.CommonVars
-
-object ClientConfiguration {
-
- val CLIENT_ENGINE_MANAGER_SPRING_APPLICATION_NAME = CommonVars("wds.linkis.client.enginemanager.application.name.default", "IOEngineManager")
- val CLIENT_ENGINE_SPRING_APPLICATION_NAME = CommonVars("wds.linkis.client.engine.application.name.default", "IOEngine")
-
- val CLIENT_DEFAULT_NAME: String = CommonVars("wds.linkis.client.name.default", "storageClient").getValue
- val CLIENT_DEFAULT_PARALLELISM_USERS = CommonVars("wds.linkis.client.parallelism.users.max", new Integer(100))
-
- val BDP_RPC_RECEIVER_ASYN_QUEUE_CAPACITY_FOR_CLIENT = CommonVars("wds.linkis.client.rpc.receiver.asyn.queue.size.max", new Integer(3000))
- val BDP_RPC_RECEIVER_ASYN_CONSUMER_THREAD_MAX_FOR_CLIENT = CommonVars("wds.linkis.client.rpc.receiver.asyn.consumer.thread.max", new Integer(120))
- val CONCURRENT_ENGINE_MAX_PARALLELISM_FOR_CLIENT = CommonVars("wds.linkis.client.engine.maxParallelismJobs", new Integer(25))
-}
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/conf/ClientForEntranceSpringConfiguration.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/conf/ClientForEntranceSpringConfiguration.scala
deleted file mode 100644
index 77fe250..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/conf/ClientForEntranceSpringConfiguration.scala
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient.conf
-
-import org.apache.linkis.common.utils.Logging
-//import org.apache.linkis.entrance.annotation.EntranceServerBeanAnnotation.EntranceServerAutowiredAnnotation
-//import org.apache.linkis.entrance.annotation._
-//import org.apache.linkis.entrance.conf.EntranceSpringConfiguration
-//import org.apache.linkis.entrance.execute._
-//import org.apache.linkis.entrance.interceptor.EntranceInterceptor
-//import org.apache.linkis.entrance.log.LogManager
-//import org.apache.linkis.entrance.persistence.PersistenceManager
-//import org.apache.linkis.entrance.{EntranceParser, EntranceServer}
-//import org.apache.linkis.entranceclient.annotation.ClientEngineSelectorBeanAnnotation.ClientEngineSelectorAutowiredAnnotation
-//import org.apache.linkis.entranceclient.annotation.ClientEntranceParserBeanAnnotation.ClientEntranceParserAutowiredAnnotation
-//import org.apache.linkis.entranceclient.annotation.ClientInterceptorsBeanAnnotation.ClientInterceptorsAutowiredAnnotation
-//import org.apache.linkis.entranceclient.annotation.DefaultEntranceClientBeanAnnotation
-//import org.apache.linkis.entranceclient.{EntranceClient, EntranceClientImpl, _}
-//import org.apache.linkis.rpc.{RPCMessageEvent, Receiver, ReceiverChooser}
-//import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean
-//import org.springframework.boot.autoconfigure.{AutoConfigureAfter, AutoConfigureBefore}
-//import org.springframework.context.annotation.{Conditional, Configuration}
-
-/*@Configuration
-@AutoConfigureBefore(Array(classOf[EntranceSpringConfiguration]))
-@AutoConfigureAfter(Array(classOf[ClientSpringConfiguration]))
-@Conditional(Array(classOf[SingleEntranceCondition]))*/
-class ClientForEntranceSpringConfiguration extends Logging {
-
- warn(s"start a single-entrance application...")
-
- /*@DefaultEntranceClientBeanAnnotation
- def generateEntranceClient(@EntranceServerAutowiredAnnotation entranceServer: EntranceServer): EntranceClient = {
- val client = EntranceClientImpl(ClientConfiguration.CLIENT_DEFAULT_NAME)
- warn(s"ready to initial EntranceClient ${client.getEntranceClientName}...")
- client.init(entranceServer)
- client
- }*/
-/*
-
- @ReceiverChooserBeanAnnotation
- def generateEntranceReceiverChooser(): ReceiverChooser = new ReceiverChooser{
- override def chooseReceiver(event: RPCMessageEvent): Option[Receiver] = None
- }
-
- @PersistenceManagerBeanAnnotation
- @ConditionalOnMissingBean(value = Array(classOf[context.ClientPersistenceManager]))
- def generatePersistenceManager(): PersistenceManager = new context.ClientPersistenceManager
-
- @EntranceParserBeanAnnotation
- def generateEntranceParser(@ClientEntranceParserAutowiredAnnotation clientEntranceParser: context.ClientEntranceParser): EntranceParser = clientEntranceParser
-
- @LogManagerBeanAnnotation
- @ConditionalOnMissingBean(value = Array(classOf[context.ClientLogManager]))
- def generateLogManager(): LogManager = new context.ClientLogManager
-
-
- @EntranceInterceptorBeanAnnotation
- def generateEntranceInterceptor(@ClientInterceptorsAutowiredAnnotation clientInterceptors: Array[EntranceInterceptor]): Array[EntranceInterceptor] = clientInterceptors
-*/
-
-
-// @EngineSelectorBeanAnnotation
-// def generateEngineSelector(@ClientEngineSelectorAutowiredAnnotation clientEngineSelector: EngineSelector): EngineSelector = clientEngineSelector
-
-
-}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/conf/MultiEntranceCondition.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/conf/MultiEntranceCondition.scala
deleted file mode 100644
index 4e0a5bb..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/conf/MultiEntranceCondition.scala
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient.conf
-
-import org.apache.linkis.entrance.conf.EntranceConfiguration
-import org.apache.linkis.entranceclient
-import org.springframework.context.annotation.{Condition, ConditionContext}
-import org.springframework.core.`type`.AnnotatedTypeMetadata
-
-class MultiEntranceCondition extends Condition {
- override def matches(conditionContext: ConditionContext, annotatedTypeMetadata: AnnotatedTypeMetadata): Boolean =
- entranceclient.conf.MultiEntranceCondition.isMultiEntranceApplication
-}
-class SingleEntranceCondition extends Condition {
- override def matches(conditionContext: ConditionContext, annotatedTypeMetadata: AnnotatedTypeMetadata): Boolean =
- !entranceclient.conf.MultiEntranceCondition.isMultiEntranceApplication
-}
-object MultiEntranceCondition {
- def isMultiEntranceApplication: Boolean = EntranceConfiguration.MULTI_ENTRANCE_CONDITION.getValue
-}
\ No newline at end of file
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/context/ClientEntranceParser.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/context/ClientEntranceParser.scala
deleted file mode 100644
index 6c845f9..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/context/ClientEntranceParser.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient.context
-
-import java.util
-
-import org.apache.linkis.entrance.parser.CommonEntranceParser
-import org.apache.linkis.entranceclient.execute
-import org.apache.linkis.governance.common.entity.task.RequestPersistTask
-import org.apache.linkis.protocol.task.Task
-import org.apache.linkis.scheduler.queue.Job
-
-class ClientEntranceParser extends CommonEntranceParser {
-
- override def parseToTask(job: Job): Task = job match {
- case j: execute.ClientJob => j.getTask
- }
-
- override def parseToJob(task: Task): Job = {
- val job = new execute.ClientJob
- task match {
- case t: RequestPersistTask =>
- job.setTask(t)
- job.setCreator(t.getRequestApplicationName)
- job.setUser(t.getUmUser)
- if (t.getParams != null) job.setParams(t.getParams.asInstanceOf[util.Map[String, Any]])
- }
- job
- }
-}
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/context/ClientLogManager.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/context/ClientLogManager.scala
deleted file mode 100644
index 5f0cc2f..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/context/ClientLogManager.scala
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient.context
-
-import java.io.{InputStream, OutputStream}
-
-import org.apache.linkis.entrance.log._
-import org.apache.linkis.scheduler.queue.Job
-import org.apache.linkis.server.conf.ServerConfiguration
-import org.apache.commons.io.input.NullInputStream
-import org.apache.commons.io.output.NullOutputStream
-
-class ClientLogManager extends CacheLogManager {
- override def getLogReader(execId: String): LogReader = {
- new CacheLogReader("", ServerConfiguration.BDP_SERVER_ENCODING.getValue,
- new Cache(10), "") {
- override def getInputStream: InputStream = new NullInputStream(0)
- }
- }
-
- override def createLogWriter(job: Job): LogWriter = {
- new NullCacheLogWriter(ServerConfiguration.BDP_SERVER_ENCODING.getValue,
- new Cache(20))
- }
- class NullCacheLogWriter(charset:String,
- sharedCache:Cache,
- override protected val outputStream: OutputStream = new NullOutputStream)
- extends CacheLogWriter("", charset, sharedCache, "")
-}
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/context/ClientPersistenceEngine.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/context/ClientPersistenceEngine.scala
deleted file mode 100644
index dbd1515..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/context/ClientPersistenceEngine.scala
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient.context
-
-import java.lang
-
-import org.apache.linkis.entrance.persistence.PersistenceEngine
-import org.apache.linkis.protocol.task.Task
-
-class ClientPersistenceEngine extends PersistenceEngine {
- override def persist(task: Task): Unit = {}
-
- /**
- * If a task's progress, status, logs, and result set are updated, this method is updated <br>
- * 如果一个任务的进度、状态、日志和结果集发生更新，由此方法进行更新<br>
- *
- * @param task
- */
- override def updateIfNeeded(task: Task): Unit = {}
-
- /**
- * Used to hang up a unified import task through this method, and continue to do the processing.
- * 用于如果某一个统一入口挂掉了，通过这个方法，将挂掉的统一入口tasks读取过来，继续做处理
- *
- * @param instance
- * @return
- */
-override def readAll(instance: String): Array[Task] = Array.empty
-
- /**
- * Return a task information in the database through a taskID, such as query log storage address, etc.
- * 通过一个taskID，返回数据库中的一条task信息，这种情况包括如 查询日志存放地址等
- *
- * @param taskID
- * @return
- */
- override def retrieve(taskID: lang.Long): Task = null
-
- override def close(): Unit = {}
-
- override def flush(): Unit = {}
-}
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/context/ClientPersistenceManager.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/context/ClientPersistenceManager.scala
deleted file mode 100644
index 0bdd575..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/context/ClientPersistenceManager.scala
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient.context
-
-import org.apache.linkis.entrance.EntranceContext
-import org.apache.linkis.entrance.persistence.{PersistenceEngine, PersistenceManager, ResultSetEngine}
-import org.apache.linkis.entranceclient.execute
-import org.apache.linkis.protocol.engine.JobProgressInfo
-import org.apache.linkis.scheduler.executer.OutputExecuteResponse
-import org.apache.linkis.scheduler.queue.Job
-
-class ClientPersistenceManager extends PersistenceManager {
- private val persistenceEngine = new ClientPersistenceEngine
- override def getEntranceContext: EntranceContext = null
-
- override def setEntranceContext(entranceContext: EntranceContext): Unit = null
-
- override def createPersistenceEngine(): PersistenceEngine = persistenceEngine
-
- override def createResultSetEngine(): ResultSetEngine = null
-
- override def onProgressUpdate(job: Job, progress: Float, progressInfo: Array[JobProgressInfo]): Unit = {}
-
- override def onJobScheduled(job: Job): Unit = {}
-
- override def onJobInited(job: Job): Unit = {}
-
- override def onJobRunning(job: Job): Unit = {}
-
- override def onJobCompleted(job: Job): Unit = {
-
- }
-
- override def onResultSetCreated(job: Job, response: OutputExecuteResponse): Unit = job match {
- case j: execute.ClientJob =>
- j.addResultSet(response.getOutput)
- j.incrementResultSetPersisted()
- }
-
-
- override def onResultSizeCreated(job: Job, resultSize: Int): Unit = job match {
- case j: execute.ClientJob => j.setResultSize(resultSize)
- }
-
- override def onJobWaitForRetry(job: Job): Unit = {}
-}
diff --git a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/execute/ClientJob.scala b/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/execute/ClientJob.scala
deleted file mode 100644
index 09faec5..0000000
--- a/linkis-computation-governance/linkis-entrance-client/src/main/scala/org/apache/linkis/entranceclient/execute/ClientJob.scala
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.linkis.entranceclient.execute
-
-import org.apache.linkis.entrance.execute.EntranceJob
-import org.apache.linkis.governance.common.entity.task.RequestPersistTask
-import org.apache.linkis.scheduler.executer.{ExecuteRequest, JobExecuteRequest}
-import org.apache.linkis.scheduler.queue.SchedulerEventState.SchedulerEventState
-import org.apache.linkis.scheduler.queue.{JobInfo, SchedulerEventState}
-
-import scala.collection.mutable.ArrayBuffer
-
-class ClientJob extends EntranceJob {
-
- private val resultSets = ArrayBuffer[String]()
-
- def addResultSet(resultSet: String): Unit = resultSets += resultSet
-
- def getResultSets: Array[String] = resultSets.toArray
-
- def waitForComplete(): Unit = {
- if (SchedulerEventState.isCompleted(this.getState)) return
- resultSets synchronized {
- while (!SchedulerEventState.isCompleted(this.getState)) resultSets.wait()
- }
- }
-
- override protected def isWaitForPersistedTimeout(startWaitForPersistedTime: Long): Boolean = System.currentTimeMillis() - startWaitForPersistedTime >= 3000
-
- override def afterStateChanged(fromState: SchedulerEventState, toState: SchedulerEventState): Unit = {
- super.afterStateChanged(fromState, toState)
- if(SchedulerEventState.isCompleted(this.getState)) resultSets synchronized resultSets.notify()
- }
-
- override def init(): Unit = {}
-
- override protected def jobToExecuteRequest: ExecuteRequest = new ExecuteRequest with JobExecuteRequest {
- override val code: String = getTask.asInstanceOf[RequestPersistTask].getCode
- override val jobId: String = getId
- }
-
- override def getName: String = getTask.asInstanceOf[RequestPersistTask].getRequestApplicationName + "_" + getTask.asInstanceOf[RequestPersistTask].getUmUser + "_" + getId
-
- override def getJobInfo: JobInfo = null
-
- override def close(): Unit = {}
-}
diff --git a/linkis-computation-governance/linkis-entrance/pom.xml b/linkis-computation-governance/linkis-entrance/pom.xml
index 99d8cd5..a91d16c 100644
--- a/linkis-computation-governance/linkis-entrance/pom.xml
+++ b/linkis-computation-governance/linkis-entrance/pom.xml
@@ -58,7 +58,7 @@
<dependency>
<groupId>org.apache.linkis</groupId>
- <artifactId>linkis-message-scheduler</artifactId>
+ <artifactId>linkis-rpc</artifactId>
<version>${linkis.version}</version>
<scope>provided</scope>
</dependency>
diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java
index eecc472..91519f5 100644
--- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java
+++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/conf/EntranceSpringConfiguration.java
@@ -59,7 +59,7 @@ public class EntranceSpringConfiguration {
private Logger logger = LoggerFactory.getLogger(getClass());
{
- logger.info("load the ujes-entrance spring configuration.");
+ logger.info("load the linkis-cg-entrance spring configuration.");
}
@PersistenceEngineBeanAnnotation
@@ -118,8 +118,8 @@ public class EntranceSpringConfiguration {
new OnceJobInterceptor(),
new CSEntranceInterceptor(),
new ShellDangerousGrammerInterceptor(),
- new PythonCodeCheckInterceptor(),
- new DBInfoCompleteInterceptor(),
+ // new PythonCodeCheckInterceptor(),
+ // new DBInfoCompleteInterceptor(),
new SparkCodeCheckInterceptor(),
new SQLCodeCheckInterceptor(),
new LabelCheckInterceptor(),
@@ -193,38 +193,6 @@ public class EntranceSpringConfiguration {
return new EntranceSchedulerContext(groupFactory, consumerManager, executorManager);
}
- /* @EngineRequesterBeanAnnotation
- @ConditionalOnMissingBean(name = {EngineRequesterBeanAnnotation.BEAN_NAME})
- public EngineRequester generateEngineRequester(){
- return new EngineRequesterImpl();
- }
- */
- /* @EngineSelectorBeanAnnotation
- @ConditionalOnMissingBean(name = {EngineSelectorBeanAnnotation.BEAN_NAME})
- public EngineSelector generateEngineSelector(@EntranceListenerBusBeanAnnotation.EntranceListenerBusAutowiredAnnotation
- EntranceEventListenerBus<EntranceEventListener, EntranceEvent> entranceEventListenerBus) {
- SingleEngineSelector singleEngineSelector = new SingleEngineSelector();
- singleEngineSelector.setEntranceEventListenerBus(entranceEventListenerBus);
- return singleEngineSelector;
- }
- */
- /*@EngineBuilderBeanAnnotation
- @ConditionalOnMissingBean(name = {EngineBuilderBeanAnnotation.BEAN_NAME})
- public EngineBuilder generateEngineBuilder(@GroupFactoryBeanAnnotation.GroupFactoryAutowiredAnnotation GroupFactory groupFactory) {
- return new AbstractEngineBuilder(groupFactory) {
- @Override
- public EntranceEngine createEngine(long id) {
- return new SingleEntranceEngine(id);
- }
- };
- }*/
-
- /* @EngineManagerBeanAnnotation
- @ConditionalOnMissingBean(name = {EngineManagerBeanAnnotation.BEAN_NAME})
- public EngineManager generateEngineManager() {
- return new EngineManagerImpl();
- }*/
-
@EntranceExecutorManagerBeanAnnotation
@ConditionalOnMissingBean(name = {EntranceExecutorManagerBeanAnnotation.BEAN_NAME})
public ExecutorManager generateExecutorManager(
@@ -245,22 +213,4 @@ public class EntranceSpringConfiguration {
scheduler.start();
return scheduler;
}
-
- /* @NewEngineBroadcastListenerBeanAnnotation
- @ConditionalOnMissingBean(name = {NewEngineBroadcastListenerBeanAnnotation.BEAN_NAME})
- public NewEngineBroadcastListener generateNewEngineBroadcastListener(@EntranceExecutorManagerBeanAnnotation.EntranceExecutorManagerAutowiredAnnotation
- EntranceExecutorManager entranceExecutorManager) {
- NewEngineBroadcastListener newEngineBroadcastListener = new NewEngineBroadcastListener();
- newEngineBroadcastListener.setEntranceExecutorManager(entranceExecutorManager);
- return newEngineBroadcastListener;
- }
-
- @ResponseEngineStatusChangedBroadcastListenerBeanAnnotation
- @ConditionalOnMissingBean(name = {ResponseEngineStatusChangedBroadcastListenerBeanAnnotation.BEAN_NAME})
- public ResponseEngineStatusChangedBroadcastListener generateResponseEngineStatusChangedBroadcastListener(@EntranceExecutorManagerBeanAnnotation.EntranceExecutorManagerAutowiredAnnotation
- EntranceExecutorManager entranceExecutorManager) {
- ResponseEngineStatusChangedBroadcastListener broadcastListener = new ResponseEngineStatusChangedBroadcastListener();
- broadcastListener.setEntranceExecutorManager(entranceExecutorManager);
- return broadcastListener;
- }*/
}
diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java
index cddd368..f16ddbd 100644
--- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java
+++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/persistence/QueryPersistenceManager.java
@@ -22,6 +22,7 @@ import org.apache.linkis.common.io.FsPath;
import org.apache.linkis.entrance.EntranceContext;
import org.apache.linkis.entrance.cs.CSEntranceHelper;
import org.apache.linkis.entrance.execute.EntranceJob;
+import org.apache.linkis.entrance.log.FlexibleErrorCodeManager;
import org.apache.linkis.governance.common.entity.job.JobRequest;
import org.apache.linkis.governance.common.entity.job.SubJobInfo;
import org.apache.linkis.protocol.engine.JobProgressInfo;
@@ -34,6 +35,9 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import scala.Option;
+import scala.Tuple2;
+
public class QueryPersistenceManager extends PersistenceManager {
private EntranceContext entranceContext;
@@ -133,11 +137,20 @@ public class QueryPersistenceManager extends PersistenceManager {
// todo check
updatedProgress = -1 * progress;
}
- if (job.getProgress() >= 0 && job.getProgress() == updatedProgress) {
+ if (Double.isNaN(updatedProgress)) {
return;
}
- job.setProgress(updatedProgress);
EntranceJob entranceJob = (EntranceJob) job;
+ float persistedProgress = 0.0f;
+ try {
+ persistedProgress = Float.parseFloat(entranceJob.getJobRequest().getProgress());
+ } catch (Exception e) {
+ logger.warn("Invalid progress : " + entranceJob.getJobRequest().getProgress(), e);
+ }
+ if (job.getProgress() >= 0 && persistedProgress >= updatedProgress) {
+ return;
+ }
+ job.setProgress(updatedProgress);
entranceJob.getJobRequest().setProgress(String.valueOf(updatedProgress));
updateJobStatus(job);
}
@@ -168,6 +181,19 @@ public class QueryPersistenceManager extends PersistenceManager {
try {
if (job.isSucceed()) {
CSEntranceHelper.registerCSRSData(job);
+ } else {
+ JobRequest jobRequest =
+ this.entranceContext.getOrCreateEntranceParser().parseToJobRequest(job);
+ if (null == jobRequest.getErrorCode() || jobRequest.getErrorCode() == 0) {
+ Option<Tuple2<String, String>> tuple2Option =
+ FlexibleErrorCodeManager.errorMatch(jobRequest.getErrorDesc());
+ if (tuple2Option.isDefined()) {
+ logger.info(jobRequest.getId() + " to reset errorCode by errorMsg");
+ Tuple2<String, String> errorCodeContent = tuple2Option.get();
+ jobRequest.setErrorCode(Integer.parseInt(errorCodeContent._1));
+ jobRequest.setErrorDesc(errorCodeContent._2);
+ }
+ }
}
} catch (Throwable e) {
logger.error("Failed to register cs rs data ", e);
@@ -185,7 +211,7 @@ public class QueryPersistenceManager extends PersistenceManager {
if (job.isSucceed()) {
// 如果是job是成功的，那么需要将task的错误描述等都要设置为null
jobRequest.setErrorCode(0);
- jobRequest.setErrorDesc(null);
+ jobRequest.setErrorDesc("");
}
} catch (Exception e) {
entranceContext.getOrCreateLogManager().onLogUpdate(job, e.getMessage());
diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java
index a6351b3..867b6ee 100644
--- a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java
+++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/restful/EntranceRestfulApi.java
@@ -72,7 +72,6 @@ public class EntranceRestfulApi implements EntranceRestfulRemote {
@RequestMapping(path = "/execute", method = RequestMethod.POST)
public Message execute(HttpServletRequest req, @RequestBody Map<String, Object> json) {
Message message = null;
- // try{
logger.info("Begin to get an execID");
json.put(TaskConstant.UMUSER, SecurityFilter.getLoginUsername(req));
HashMap<String, String> map = (HashMap) json.get(TaskConstant.SOURCE);
@@ -117,11 +116,6 @@ public class EntranceRestfulApi implements EntranceRestfulRemote {
message.data("execID", execID);
message.data("taskID", taskID);
logger.info("End to get an an execID: {}, taskID: {}", execID, taskID);
- // }catch(ErrorException e){
- // message = Message.error(e.getDesc());
- // message.setStatus(1);
- // message.setMethod("/api/entrance/execute");
- // }
return message;
}
@@ -231,7 +225,7 @@ public class EntranceRestfulApi implements EntranceRestfulRemote {
}
message = Message.ok();
message.setMethod("/api/entrance/" + id + "/progress");
- // TODO 去掉绝对值判断
+
message.data("progress", Math.abs(job.get().getProgress()))
.data("execID", id)
.data("progressInfo", list);
@@ -374,7 +368,6 @@ public class EntranceRestfulApi implements EntranceRestfulRemote {
String id = idNode.get(i).asText();
Long taskID = taskIDNode.get(i).asLong();
String realId = ZuulEntranceUtils.parseExecID(id)[3];
- // 通过jobid获取job,可能会由于job找不到而导致有looparray的报错,一旦报错的话，就可以将该任务直接置为Cancenlled
Option<Job> job = Option.apply(null);
try {
job = entranceServer.getJob(realId);
@@ -453,7 +446,6 @@ public class EntranceRestfulApi implements EntranceRestfulRemote {
@PathVariable("id") String id,
@RequestParam(value = "taskID", required = false) long taskID) {
String realId = ZuulEntranceUtils.parseExecID(id)[3];
- // 通过jobid获取job,可能会由于job找不到而导致有looparray的报错,一旦报错的话，就可以将该任务直接置为Cancenlled
Option<Job> job = Option.apply(null);
try {
job = entranceServer.getJob(realId);
diff --git a/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/vo/YarnResourceWithStatusVo.java b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/vo/YarnResourceWithStatusVo.java
new file mode 100644
index 0000000..6b50a88
--- /dev/null
+++ b/linkis-computation-governance/linkis-entrance/src/main/java/org/apache/linkis/entrance/vo/YarnResourceWithStatusVo.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.entrance.vo;
+
+import org.apache.linkis.manager.common.protocol.resource.ResourceWithStatus;
+
+import java.io.Serializable;
+
+public class YarnResourceWithStatusVo implements Serializable {
+
+ public Long queueMemory;
+
+ public Integer queueCores;
+
+ public Integer queueInstances;
+
+ public String jobStatus;
+
+ public String applicationId;
+
+ public String queue;
+
+ public YarnResourceWithStatusVo(String applicationId, ResourceWithStatus resourceWithStatus) {
+ this.queueMemory = resourceWithStatus.queueMemory();
+ this.queueCores = resourceWithStatus.queueCores();
+ this.queueInstances = resourceWithStatus.queueInstances();
+ this.jobStatus = resourceWithStatus.jobStatus();
+ this.applicationId = applicationId;
+ this.queue = resourceWithStatus.queue();
+ }
+
+ public YarnResourceWithStatusVo(
+ String applicationId,
+ Long queueMemory,
+ Integer queueCores,
+ Integer queueInstances,
+ String jobStatus,
+ String queue) {
+ this.applicationId = applicationId;
+ this.queueMemory = queueMemory;
+ this.queueCores = queueCores;
+ this.queueInstances = queueInstances;
+ this.jobStatus = jobStatus;
+ this.queue = queue;
+ }
+
+ public String getApplicationId() {
+ return applicationId;
+ }
+
+ public void setApplicationId(String applicationId) {
+ this.applicationId = applicationId;
+ }
+
+ public Long getQueueMemory() {
+ return queueMemory;
+ }
+
+ public void setQueueMemory(Long queueMemory) {
+ this.queueMemory = queueMemory;
+ }
+
+ public Integer getQueueCores() {
+ return queueCores;
+ }
+
+ public void setQueueCores(Integer queueCores) {
+ this.queueCores = queueCores;
+ }
+
+ public Integer getQueueInstances() {
+ return queueInstances;
+ }
+
+ public void setQueueInstances(Integer queueInstances) {
+ this.queueInstances = queueInstances;
+ }
+
+ public String getJobStatus() {
+ return jobStatus;
+ }
+
+ public void setJobStatus(String jobStatus) {
+ this.jobStatus = jobStatus;
+ }
+
+ public String getQueue() {
+ return queue;
+ }
+
+ public void setQueue(String queue) {
+ this.queue = queue;
+ }
+}
diff --git a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala
index 9c28890..a69320a 100644
--- a/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala
+++ b/linkis-computation-governance/linkis-entrance/src/main/scala/org/apache/linkis/entrance/conf/EntranceConfiguration.scala
@@ -127,7 +127,7 @@ object EntranceConfiguration {
val SKIP_AUTH = CommonVars("wds.linkis.entrance.skip.auth", true)
- val PROGRESS_PUSH = CommonVars[String]("wds.linkis.entrance.push.progress", "false")
+ val PROGRESS_PUSH = CommonVars[String]("wds.linkis.entrance.push.progress", "true")
val CONCURRENT_FACTORY_MAX_CAPACITY = CommonVars("wds.linkis.concurrent.group.factory.capacity", 1000)
diff --git a/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelKeyConstant.java b/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelKeyConstant.java
index 3e53235..9a951bf 100644
--- a/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelKeyConstant.java
+++ b/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/constant/LabelKeyConstant.java
@@ -51,6 +51,8 @@ public class LabelKeyConstant {
public static final String JOB_RUNNING_TIMEOUT_KEY = "jobRunningTimeout";
+ public static final String RETRY_TIMEOUT_KEY = "jobRetryTimeout";
+
public static final String EXECUTE_ONCE_KEY = "executeOnce";
public static final String LOAD_BALANCE_KEY = "loadBalance";
diff --git a/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/SerializableLabel.java b/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/SerializableLabel.java
index 581cf5b..02c7552 100644
--- a/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/SerializableLabel.java
+++ b/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/SerializableLabel.java
@@ -108,9 +108,6 @@ public abstract class SerializableLabel<T> implements Label<T> {
CACHE.put(this.getClass(), orderedValueNames);
}
if (stringValueArray.length != orderedValueNames.size()) {
- // Warn
- // LOG.info("str_value: [" + stringValue + "] doesn't match ordered_value_name: [" +
- // StringUtils.join(orderedValueNames, ",") + "]");
return;
}
for (int i = 0; i < orderedValueNames.size(); i++) {
diff --git a/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/TenantLabel.java b/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/TenantLabel.java
index 9c85a9f..6496183 100644
--- a/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/TenantLabel.java
+++ b/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/TenantLabel.java
@@ -64,7 +64,8 @@ public class TenantLabel extends GenericLabel
if (stringValue.split(SerializableLabel.VALUE_SEPARATOR).length != 1) {
throw new LabelErrorException(
LabelConstant.LABEL_BUILDER_ERROR_CODE,
- "标签route的值设置错误，只能设置1个值，并且不能使用符号" + VALUE_SEPARATOR);
+ "The value of the label is set incorrectly, only one value can be set, and the symbol cannot be used"
+ + VALUE_SEPARATOR);
}
}
}
diff --git a/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala b/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
index 515cb1a..3ff83ba 100644
--- a/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
+++ b/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
@@ -31,6 +31,7 @@ object RunType extends Enumeration {
val SHELL = Value("shell")
val IO_FILE = Value("io_file")
val IO_HDFS = Value("io_hdfs")
+ val FPS = Value("fps")
val PIPELINE = Value("pipeline")
val JDBC = Value("jdbc")
val PRESTO_SQL = Value("psql")
diff --git a/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/UserCreatorLabel.java b/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/UserCreatorLabel.java
index cf89dac..e91da1a 100644
--- a/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/UserCreatorLabel.java
+++ b/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/UserCreatorLabel.java
@@ -79,7 +79,9 @@ public class UserCreatorLabel extends GenericLabel implements EngineNodeLabel, U
if (stringValue.split(SerializableLabel.VALUE_SEPARATOR).length != 2) {
throw new LabelErrorException(
LabelConstant.LABEL_BUILDER_ERROR_CODE,
- "标签usercreator的值设置错误，需要2个值，并且需要使用" + VALUE_SEPARATOR + "隔开");
+ "The value of the label is set incorrectly, only one value can be set, and the symbol cannot be used"
+ + VALUE_SEPARATOR
+ + "隔开");
}
}
}
diff --git a/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/entrance/RetryWaitTimeOutLabel.java b/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/entrance/RetryWaitTimeOutLabel.java
new file mode 100644
index 0000000..2570fa9
--- /dev/null
+++ b/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/entrance/RetryWaitTimeOutLabel.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.manager.label.entity.entrance;
+
+import org.apache.linkis.manager.label.constant.LabelKeyConstant;
+import org.apache.linkis.manager.label.entity.GenericLabel;
+import org.apache.linkis.manager.label.entity.annon.ValueSerialNum;
+
+import java.util.HashMap;
+
+public class RetryWaitTimeOutLabel extends GenericLabel implements JobStrategyLabel {
+
+ public RetryWaitTimeOutLabel() {
+ setLabelKey(LabelKeyConstant.RETRY_TIMEOUT_KEY);
+ }
+
+ public Long getJobRetryTimeout() {
+ if (null == getValue()) {
+ return -1L;
+ }
+ return Long.parseLong(getValue().getOrDefault(LabelKeyConstant.RETRY_TIMEOUT_KEY, "-1"));
+ }
+
+ @ValueSerialNum(0)
+ public void setJobRetryTimeout(String timeout) {
+ if (null == getValue()) {
+ setValue(new HashMap<>());
+ }
+ getValue().put(LabelKeyConstant.RETRY_TIMEOUT_KEY, timeout);
+ }
+}
diff --git a/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/route/RouteLabel.java b/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/route/RouteLabel.java
index 5bd2f12..c611735 100644
--- a/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/route/RouteLabel.java
+++ b/linkis-computation-governance/linkis-manager/label-common/src/main/java/org/apache/linkis/manager/label/entity/route/RouteLabel.java
@@ -49,7 +49,8 @@ public class RouteLabel extends InheritableLabel<String> implements UserModifiab
if (stringValue.split(SerializableLabel.VALUE_SEPARATOR).length != 1) {
throw new LabelErrorException(
LabelConstant.LABEL_BUILDER_ERROR_CODE,
- "标签route的值设置错误，只能设置1个值，并且不能使用符号" + VALUE_SEPARATOR);
+ "The value of the label is set incorrectly, only 1 value can be set, and the symbol cannot be used"
+ + VALUE_SEPARATOR);
}
}
}
diff --git a/linkis-computation-governance/linkis-manager/label-manager/pom.xml b/linkis-computation-governance/linkis-manager/label-manager/pom.xml
index 75bf906..ac3f07d 100644
--- a/linkis-computation-governance/linkis-manager/label-manager/pom.xml
+++ b/linkis-computation-governance/linkis-manager/label-manager/pom.xml
@@ -45,8 +45,9 @@
<dependency>
<groupId>org.apache.linkis</groupId>
- <artifactId>linkis-message-scheduler</artifactId>
+ <artifactId>linkis-rpc</artifactId>
<version>${linkis.version}</version>
+ <scope>provided</scope>
</dependency>
</dependencies>
diff --git a/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/ResourceLabelService.scala b/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/ResourceLabelService.scala
index ef9eb9c..ea1f59e 100644
--- a/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/ResourceLabelService.scala
+++ b/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/ResourceLabelService.scala
@@ -17,6 +17,7 @@
package org.apache.linkis.manager.label.service
+import org.apache.linkis.manager.common.entity.persistence.PersistenceResource
import org.apache.linkis.manager.common.entity.resource.NodeResource
import org.apache.linkis.manager.label.entity.Label
@@ -65,5 +66,7 @@ trait ResourceLabelService {
def removeResourceByLabels(labels: java.util.List[Label[_]]): Unit
+ def getPersistenceResourceByLabel(label: Label[_]): PersistenceResource
+
}
diff --git a/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelAddService.scala b/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelAddService.scala
index 8f6d034..95221ff 100644
--- a/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelAddService.scala
+++ b/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelAddService.scala
@@ -24,7 +24,7 @@ import org.apache.linkis.manager.common.protocol.label.LabelReportRequest
import org.apache.linkis.manager.label.builder.factory.LabelBuilderFactoryContext
import org.apache.linkis.manager.label.entity.Label
import org.apache.linkis.manager.label.service.{NodeLabelAddService, NodeLabelService}
-import org.apache.linkis.message.annotation.Receiver
+import org.apache.linkis.rpc.serializer.annotation.Receiver
import org.apache.linkis.protocol.label.NodeLabelAddRequest
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
diff --git a/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelRemoveService.scala b/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelRemoveService.scala
index 14ed9d6..87a39ff 100644
--- a/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelRemoveService.scala
+++ b/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelRemoveService.scala
@@ -24,9 +24,10 @@ import org.apache.linkis.manager.label.entity.em.EMInstanceLabel
import org.apache.linkis.manager.label.entity.engine.EngineInstanceLabel
import org.apache.linkis.manager.label.service.{NodeLabelRemoveService, NodeLabelService}
import org.apache.linkis.manager.persistence.LabelManagerPersistence
-import org.apache.linkis.message.annotation.Receiver
+import org.apache.linkis.rpc.serializer.annotation.Receiver
import org.apache.linkis.protocol.label.NodeLabelRemoveRequest
import org.apache.linkis.manager.label.conf.LabelCommonConfig
+import org.apache.linkis.manager.label.exception.LabelRuntimeException
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.stereotype.Service
@@ -45,6 +46,9 @@ class DefaultNodeLabelRemoveService extends NodeLabelRemoveService with Logging
@Receiver
override def removeNodeLabel(nodeLabelRemoveRequest: NodeLabelRemoveRequest): Unit = {
info(s"Start to remove labels from node ${nodeLabelRemoveRequest.getServiceInstance}")
+ if(nodeLabelRemoveRequest.getServiceInstance == null) {
+ throw new LabelRuntimeException(130001, "ServiceInstance in request is null, please check label remove request!")
+ }
nodeLabelService.removeLabelsFromNode(nodeLabelRemoveRequest.getServiceInstance)
val persistenceLabel = if (nodeLabelRemoveRequest.isEngine) {
val engineLabel = labelFactory.createLabel(classOf[EngineInstanceLabel])
diff --git a/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala b/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala
index 85014ab..eeaf1f6 100644
--- a/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala
+++ b/linkis-computation-governance/linkis-manager/label-manager/src/main/scala/org/apache/linkis/manager/label/service/impl/DefaultNodeLabelService.scala
@@ -114,9 +114,9 @@ class DefaultNodeLabelService extends NodeLabelService with Logging {
val willBeAdd = newKeyList.diff(oldKeyList)
val willBeUpdate = oldKeyList.diff(willBeDelete)
val modifiableKeyList = LabelUtils.listAllUserModifiableLabel()
- if(!CollectionUtils.isEmpty(willBeDelete)){
- nodeLabels.foreach(nodeLabel => {
- if(modifiableKeyList.contains(nodeLabel.getLabelKey) && willBeDelete.contains(nodeLabel.getLabelKey)){
+ if (!CollectionUtils.isEmpty(willBeDelete)) {
+ nodeLabels.foreach(nodeLabel => {
+ if (modifiableKeyList.contains(nodeLabel.getLabelKey) && willBeDelete.contains(nodeLabel.getLabelKey)) {
... 16969 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org