You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by ca...@apache.org on 2022/08/31 14:17:59 UTC
[incubator-linkis] branch dev-1.3.1 updated: [Feature] datasource manager support mulit env (#2961)
This is an automated email from the ASF dual-hosted git repository.
casion pushed a commit to branch dev-1.3.1
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
The following commit(s) were added to refs/heads/dev-1.3.1 by this push:
new ff8b14018 [Feature] datasource manager support mulit env (#2961)
ff8b14018 is described below
commit ff8b1401868819e8c7c58ddec1dbd0ab8421a5f1
Author: chenmutime <ch...@outlook.com>
AuthorDate: Wed Aug 31 22:17:53 2022 +0800
[Feature] datasource manager support mulit env (#2961)
* feat: support multi env
* get type key by dataSourceTypeId
* add unique key on env table; optimal code
* add hook when save connectParams of DataSource
Co-authored-by: v_minminghe <v_...@webank.com>
Co-authored-by: Casion <ca...@gmail.com>
---
.../db/upgrade/1.3.1_schema/mysql/linkis_ddl.sql | 22 ++++
.../core/dao/DataSourceEnvDao.java | 10 ++
.../core/dao/mapper/DataSouceMapper.xml | 1 +
.../core/dao/mapper/DataSourceEnvMapper.xml | 10 ++
.../core/restful/DataSourceAdminRestfulApi.java | 129 ++++++++++++++++++-
.../core/restful/DataSourceCoreRestfulApi.java | 14 +++
.../core/service/DataSourceInfoService.java | 37 +++++-
.../service/impl/DataSourceInfoServiceImpl.java | 136 ++++++++++++++-------
8 files changed, 306 insertions(+), 53 deletions(-)
diff --git a/linkis-dist/package/db/upgrade/1.3.1_schema/mysql/linkis_ddl.sql b/linkis-dist/package/db/upgrade/1.3.1_schema/mysql/linkis_ddl.sql
new file mode 100644
index 000000000..14a04e30d
--- /dev/null
+++ b/linkis-dist/package/db/upgrade/1.3.1_schema/mysql/linkis_ddl.sql
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+-- ----------------------------
+-- add UNIQUE KEY with env_name field for linkis_ps_dm_datasource_env
+-- ----------------------------
+
+ALTER TABLE `linkis_ps_dm_datasource_env` ADD CONSTRAINT `uk_env_name` UNIQUE (`env_name`);
\ No newline at end of file
diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/dao/DataSourceEnvDao.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/dao/DataSourceEnvDao.java
index 9fe9b5c3c..503c8718d 100644
--- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/dao/DataSourceEnvDao.java
+++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/dao/DataSourceEnvDao.java
@@ -20,6 +20,8 @@ package org.apache.linkis.datasourcemanager.core.dao;
import org.apache.linkis.datasourcemanager.common.domain.DataSourceEnv;
import org.apache.linkis.datasourcemanager.core.vo.DataSourceEnvVo;
+import org.apache.ibatis.annotations.Param;
+
import java.util.List;
/** Data source dao */
@@ -70,4 +72,12 @@ public interface DataSourceEnvDao {
* @return
*/
List<DataSourceEnv> selectByPageVo(DataSourceEnvVo dataSourceEnvVo);
+
+ /**
+ * get DataSourceEnv by envName
+ *
+ * @param envName
+ * @return
+ */
+ DataSourceEnv selectOneByName(@Param("envName") String envName);
}
diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/dao/mapper/DataSouceMapper.xml b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/dao/mapper/DataSouceMapper.xml
index 4ac1ca07c..35aa42466 100644
--- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/dao/mapper/DataSouceMapper.xml
+++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/dao/mapper/DataSouceMapper.xml
@@ -187,6 +187,7 @@
<if test="dataSourceTypeId != null and dataSourceTypeId > 0">
<![CDATA[`datasource_type_id` = #{dataSourceTypeId},]]>
</if>
+ <![CDATA[`parameter` = #{parameter},]]>
<![CDATA[`create_identify` = #{createIdentify,jdbcType=VARCHAR},]]>
<![CDATA[`create_system` = #{createSystem,jdbcType=VARCHAR},]]>
<choose>
diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/dao/mapper/DataSourceEnvMapper.xml b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/dao/mapper/DataSourceEnvMapper.xml
index 3fa9b675d..c174e0131 100644
--- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/dao/mapper/DataSourceEnvMapper.xml
+++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/dao/mapper/DataSourceEnvMapper.xml
@@ -123,4 +123,14 @@
`id` = #{id}
</where>
</update>
+
+ <select id="selectOneByName" resultMap="dataSourceEnvMap">
+ <![CDATA[SELECT ]]>
+ <include refid="data_source_env_detail"/>
+ <![CDATA[ FROM `linkis_ps_dm_datasource_env` ]]>
+ <where>
+ `env_name` = #{envName}
+ </where>
+ <![CDATA[ LIMIT 1; ]]>
+ </select>
</mapper>
\ No newline at end of file
diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceAdminRestfulApi.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceAdminRestfulApi.java
index abeeb0646..e88e752af 100644
--- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceAdminRestfulApi.java
+++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceAdminRestfulApi.java
@@ -17,6 +17,7 @@
package org.apache.linkis.datasourcemanager.core.restful;
+import com.google.common.collect.Lists;
import org.apache.linkis.common.exception.ErrorException;
import org.apache.linkis.datasourcemanager.common.domain.DataSourceEnv;
import org.apache.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
@@ -45,10 +46,7 @@ import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
-import java.util.Calendar;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
@Api(tags = "data source admin restful api")
@RestController
@@ -57,6 +55,7 @@ import java.util.Set;
produces = {"application/json"})
public class DataSourceAdminRestfulApi {
+ private final List<String> permitSystemList = Arrays.asList("Qualitis");
@Autowired private DataSourceInfoService dataSourceInfoService;
@Autowired private DataSourceRelateService dataSourceRelateService;
@@ -89,6 +88,14 @@ public class DataSourceAdminRestfulApi {
if (result.size() > 0) {
throw new ConstraintViolationException(result);
}
+ if (dataSourceInfoService.existDataSourceEnv(dataSourceEnv.getEnvName())) {
+ return Message.error(
+ "The data source env named: "
+ + dataSourceEnv.getEnvName()
+ + " has been existed [数据源环境: "
+ + dataSourceEnv.getEnvName()
+ + " 已经存在]");
+ }
dataSourceEnv.setCreateUser(userName);
insertDataSourceEnv(dataSourceEnv);
return Message.ok().data("insertId", dataSourceEnv.getId());
@@ -96,6 +103,111 @@ public class DataSourceAdminRestfulApi {
"Fail to insert data source environment[新增数据源环境失败]");
}
+ @ApiOperation(value = "insertJsonEnvBatch", notes = "insert batch json env", response = Message.class)
+ @ApiOperationSupport(ignoreParameters = {"dataSourceEnvList", "system"})
+ @RequestMapping(value = "/env/json/batch", method = RequestMethod.POST)
+ public Message insertJsonEnvBatch(
+ @RequestBody List<DataSourceEnv> dataSourceEnvList,
+ @RequestParam("system") String system,
+ HttpServletRequest req)
+ throws ErrorException {
+ String userName = SecurityFilter.getLoginUsername(req);
+ if (!RestfulApiHelper.isAdminUser(userName) && !permitSystemList.contains(system)) {
+ return Message.error("User '" + userName + "' is not admin user[非管理员用户]");
+ }
+ for (DataSourceEnv dataSourceEnv : dataSourceEnvList) {
+ // Bean validation
+ Set<ConstraintViolation<DataSourceEnv>> result =
+ beanValidator.validate(dataSourceEnv, Default.class);
+ if (result.size() > 0) {
+ throw new ConstraintViolationException(result);
+ }
+ if (dataSourceInfoService.existDataSourceEnv(dataSourceEnv.getEnvName())) {
+ return Message.error(
+ "The data source env named: "
+ + dataSourceEnv.getEnvName()
+ + " has been existed [数据源环境: "
+ + dataSourceEnv.getEnvName()
+ + " 已经存在]");
+ }
+ dataSourceEnv.setCreateUser(userName);
+ // Get key definitions in environment scope
+ List<DataSourceParamKeyDefinition> keyDefinitionList =
+ dataSourceRelateService.getKeyDefinitionsByType(
+ dataSourceEnv.getDataSourceTypeId(),
+ DataSourceParamKeyDefinition.Scope.ENV);
+ dataSourceEnv.setKeyDefinitions(keyDefinitionList);
+ Map<String, Object> connectParams = dataSourceEnv.getConnectParams();
+ // Validate connect parameters
+ parameterValidator.validate(keyDefinitionList, connectParams);
+ }
+ dataSourceInfoService.saveBatchDataSourceEnv(dataSourceEnvList);
+ return RestfulApiHelper.doAndResponse(
+ () -> Message.ok().data("envs", dataSourceEnvList),
+ "Fail to insert data source environment[新增数据源环境失败]");
+ }
+
+ @ApiOperation(value = "insertJsonEnvBatch", notes = "update batch json env", response = Message.class)
+ @ApiOperationSupport(ignoreParameters = {"dataSourceEnvList", "system"})
+ @RequestMapping(value = "/env/json/batch", method = RequestMethod.PUT)
+ public Message updateEnvBatch(
+ @RequestBody List<DataSourceEnv> dataSourceEnvList,
+ @RequestParam("system") String system,
+ HttpServletRequest request)
+ throws ErrorException {
+ String userName = SecurityFilter.getLoginUsername(request);
+ if (!RestfulApiHelper.isAdminUser(userName) && !permitSystemList.contains(system)) {
+ return Message.error("User '" + userName + "' is not admin user[非管理员用户]");
+ }
+ for (DataSourceEnv dataSourceEnv : dataSourceEnvList) {
+ if (Objects.isNull(dataSourceEnv.getId())) {
+ return Message.error(
+ "Fail to update data source environment[更新数据源环境失败], "
+ + "[Please check the id if exists']");
+ }
+ // Bean validation
+ Set<ConstraintViolation<DataSourceEnv>> result =
+ beanValidator.validate(dataSourceEnv, Default.class);
+ if (result.size() > 0) {
+ throw new ConstraintViolationException(result);
+ }
+ Long envId = dataSourceEnv.getId();
+ dataSourceEnv.setModifyUser(userName);
+ dataSourceEnv.setModifyTime(Calendar.getInstance().getTime());
+ DataSourceEnv storedDataSourceEnv = dataSourceInfoService.getDataSourceEnv(envId);
+ if (null == storedDataSourceEnv) {
+ return Message.error(
+ "Fail to update data source environment[更新数据源环境失败], "
+ + "[Please check the id:'"
+ + envId
+ + " is correct ']");
+ }
+ if (!Objects.equals(dataSourceEnv.getEnvName(), storedDataSourceEnv.getEnvName())
+ && dataSourceInfoService.existDataSourceEnv(dataSourceEnv.getEnvName())) {
+ return Message.error(
+ "The data source env named: "
+ + dataSourceEnv.getEnvName()
+ + " has been existed [数据源环境: "
+ + dataSourceEnv.getEnvName()
+ + " 已经存在]");
+ }
+ dataSourceEnv.setCreateUser(storedDataSourceEnv.getCreateUser());
+ // Get key definitions in environment scope
+ List<DataSourceParamKeyDefinition> keyDefinitionList =
+ dataSourceRelateService.getKeyDefinitionsByType(
+ dataSourceEnv.getDataSourceTypeId(),
+ DataSourceParamKeyDefinition.Scope.ENV);
+ dataSourceEnv.setKeyDefinitions(keyDefinitionList);
+ Map<String, Object> connectParams = dataSourceEnv.getConnectParams();
+ // Validate connect parameters
+ parameterValidator.validate(keyDefinitionList, connectParams);
+ }
+ dataSourceInfoService.updateBatchDataSourceEnv(dataSourceEnvList);
+ return RestfulApiHelper.doAndResponse(
+ () -> Message.ok().data("envs", dataSourceEnvList),
+ "Fail to update data source environment[更新数据源环境失败]");
+ }
+
@ApiOperation(value = "getAllEnvListByDataSourceType", notes = "get all env list by data source type", response = Message.class)
@ApiImplicitParams({
@ApiImplicitParam(name = "typeId", required = true, dataType = "Long", value = "type id")
@@ -184,6 +296,15 @@ public class DataSourceAdminRestfulApi {
+ envId
+ " is correct ']");
}
+ if (!Objects.equals(dataSourceEnv.getEnvName(), storedDataSourceEnv.getEnvName())
+ && dataSourceInfoService.existDataSourceEnv(dataSourceEnv.getEnvName())) {
+ return Message.error(
+ "The data source env named: "
+ + dataSourceEnv.getEnvName()
+ + " has been existed [数据源环境: "
+ + dataSourceEnv.getEnvName()
+ + " 已经存在]");
+ }
dataSourceEnv.setCreateUser(storedDataSourceEnv.getCreateUser());
updateDataSourceEnv(dataSourceEnv, storedDataSourceEnv);
return Message.ok().data("updateId", envId);
diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java
index b0b1bdab1..b0c02792f 100644
--- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java
+++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/restful/DataSourceCoreRestfulApi.java
@@ -24,6 +24,7 @@ import org.apache.linkis.datasourcemanager.common.domain.DataSource;
import org.apache.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
import org.apache.linkis.datasourcemanager.common.domain.DataSourceType;
import org.apache.linkis.datasourcemanager.common.domain.DatasourceVersion;
+import org.apache.linkis.datasourcemanager.common.util.json.Json;
import org.apache.linkis.datasourcemanager.core.formdata.FormDataTransformerFactory;
import org.apache.linkis.datasourcemanager.core.formdata.MultiPartFormDataTransformer;
import org.apache.linkis.datasourcemanager.core.service.DataSourceInfoService;
@@ -216,6 +217,14 @@ public class DataSourceCoreRestfulApi {
+ dataSourceName
+ " 已经存在]");
}
+ List<DataSourceParamKeyDefinition> keyDefinitionList =
+ dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
+ dataSource.setKeyDefinitions(keyDefinitionList);
+ for (DataSourceParamsHook hook : dataSourceParamsHooks) {
+ hook.beforePersist(dataSource.getConnectParams(), keyDefinitionList);
+ }
+ String parameter = Json.toJson(dataSource.getConnectParams(), null);
+ dataSource.setParameter(parameter);
dataSourceInfoService.updateDataSourceInfo(dataSource);
return Message.ok().data("updateId", dataSourceId);
},
@@ -728,6 +737,11 @@ public class DataSourceCoreRestfulApi {
List<DataSourceParamKeyDefinition> keyDefinitionList =
dataSourceRelateService.getKeyDefinitionsByType(dataSource.getDataSourceTypeId());
dataSource.setKeyDefinitions(keyDefinitionList);
+ for (DataSourceParamsHook hook : dataSourceParamsHooks) {
+ hook.beforePersist(dataSource.getConnectParams(), keyDefinitionList);
+ }
+ String parameter = Json.toJson(dataSource.getConnectParams(), null);
+ dataSource.setParameter(parameter);
dataSourceInfoService.saveDataSourceInfo(dataSource);
}
}
diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/service/DataSourceInfoService.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/service/DataSourceInfoService.java
index 76118935f..cd31f0f77 100644
--- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/service/DataSourceInfoService.java
+++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/service/DataSourceInfoService.java
@@ -22,6 +22,7 @@ import org.apache.linkis.datasourcemanager.common.domain.DataSource;
import org.apache.linkis.datasourcemanager.common.domain.DataSourceEnv;
import org.apache.linkis.datasourcemanager.common.domain.DataSourceParamKeyDefinition;
import org.apache.linkis.datasourcemanager.common.domain.DatasourceVersion;
+import org.apache.linkis.datasourcemanager.common.exception.JsonErrorException;
import org.apache.linkis.datasourcemanager.core.vo.DataSourceEnvVo;
import org.apache.linkis.datasourcemanager.core.vo.DataSourceVo;
@@ -53,7 +54,7 @@ public interface DataSourceInfoService {
* @param dataSourceId id
* @return data source entity
*/
- DataSource getDataSourceInfo(Long dataSourceId);
+ DataSource getDataSourceInfo(Long dataSourceId) throws JsonErrorException;
/**
* Get data source for current version by data source name
@@ -61,7 +62,7 @@ public interface DataSourceInfoService {
* @param dataSourceName data source name
* @return data source entity
*/
- DataSource getDataSourceInfo(String dataSourceName);
+ DataSource getDataSourceInfo(String dataSourceName) throws JsonErrorException;
/**
* Get data source for latest published version by data source name
@@ -77,7 +78,7 @@ public interface DataSourceInfoService {
* @param dataSourceId id
* @return data source entity
*/
- DataSource getDataSourceInfo(Long dataSourceId, Long version);
+ DataSource getDataSourceInfo(Long dataSourceId, Long version) throws JsonErrorException;
/**
* Get data source brief information
@@ -117,6 +118,14 @@ public interface DataSourceInfoService {
*/
void saveDataSourceEnv(DataSourceEnv dataSourceEnv) throws ErrorException;
+ /**
+ * Batch save data source environment
+ *
+ * @param dataSourceEnvList
+ * @throws ErrorException
+ */
+ void saveBatchDataSourceEnv(List<DataSourceEnv> dataSourceEnvList) throws ErrorException;
+
/**
* List data source environments
*
@@ -149,6 +158,14 @@ public interface DataSourceInfoService {
*/
void updateDataSourceEnv(DataSourceEnv updatedOne, DataSourceEnv storedOne) throws ErrorException;
+ /**
+ * Batch update data source environment
+ *
+ * @param dataSourceEnvList
+ * @throws ErrorException
+ */
+ void updateBatchDataSourceEnv(List<DataSourceEnv> dataSourceEnvList) throws ErrorException;
+
/**
* Page query of data source environment
*
@@ -205,7 +222,7 @@ public interface DataSourceInfoService {
* @param dataSourceId
* @return
*/
- DataSource getDataSourceInfoForConnect(Long dataSourceId);
+ DataSource getDataSourceInfoForConnect(Long dataSourceId) throws JsonErrorException;
/**
* get datasource info for connect for published version by name, if there is a dependency
@@ -214,7 +231,7 @@ public interface DataSourceInfoService {
* @param dataSourceName
* @return
*/
- DataSource getDataSourceInfoForConnect(String dataSourceName);
+ DataSource getDataSourceInfoForConnect(String dataSourceName) throws JsonErrorException;
/**
* get datasource info for connect, if there is a dependency environment, merge datasource
* parameter and environment parameter.
@@ -223,7 +240,7 @@ public interface DataSourceInfoService {
* @param version
* @return
*/
- DataSource getDataSourceInfoForConnect(Long dataSourceId, Long version);
+ DataSource getDataSourceInfoForConnect(Long dataSourceId, Long version) throws JsonErrorException;
/**
* Check if exist data source
@@ -232,4 +249,12 @@ public interface DataSourceInfoService {
* @return boolean
*/
boolean existDataSource(String dataSourceName);
+
+ /**
+ * Check if exist data source env
+ *
+ * @param dataSourceEnvName
+ * @return
+ */
+ boolean existDataSourceEnv(String dataSourceEnvName);
}
diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/service/impl/DataSourceInfoServiceImpl.java b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/service/impl/DataSourceInfoServiceImpl.java
index 66651dee7..b9984f5ce 100644
--- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/service/impl/DataSourceInfoServiceImpl.java
+++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-manager/server/src/main/java/org/apache/linkis/datasourcemanager/core/service/impl/DataSourceInfoServiceImpl.java
@@ -28,10 +28,12 @@ import org.apache.linkis.datasourcemanager.core.dao.*;
import org.apache.linkis.datasourcemanager.core.formdata.FormStreamContent;
import org.apache.linkis.datasourcemanager.core.service.BmlAppService;
import org.apache.linkis.datasourcemanager.core.service.DataSourceInfoService;
+import org.apache.linkis.datasourcemanager.core.service.DataSourceRelateService;
import org.apache.linkis.datasourcemanager.core.service.hooks.DataSourceParamsHook;
import org.apache.linkis.datasourcemanager.core.vo.DataSourceEnvVo;
import org.apache.linkis.datasourcemanager.core.vo.DataSourceVo;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
@@ -62,6 +64,8 @@ public class DataSourceInfoServiceImpl implements DataSourceInfoService {
@Autowired private DataSourceVersionDao dataSourceVersionDao;
+ @Autowired private DataSourceRelateService dataSourceRelateService;
+
@Autowired private List<DataSourceParamsHook> dataSourceParamsHooks = new ArrayList<>();
@Override
@@ -90,9 +94,7 @@ public class DataSourceInfoServiceImpl implements DataSourceInfoService {
public DataSource getDataSourceInfo(Long dataSourceId) {
DataSource dataSource = dataSourceDao.selectOneDetail(dataSourceId);
if (Objects.nonNull(dataSource)) {
- String parameter =
- dataSourceVersionDao.selectOneVersion(dataSourceId, dataSource.getVersionId());
- dataSource.setParameter(parameter);
+ mergeVersionParams(dataSource, dataSource.getVersionId());
}
return dataSource;
}
@@ -101,9 +103,7 @@ public class DataSourceInfoServiceImpl implements DataSourceInfoService {
public DataSource getDataSourceInfo(String dataSourceName) {
DataSource dataSource = dataSourceDao.selectOneDetailByName(dataSourceName);
if (Objects.nonNull(dataSource)) {
- String parameter =
- dataSourceVersionDao.selectOneVersion(dataSource.getId(), dataSource.getVersionId());
- dataSource.setParameter(parameter);
+ mergeVersionParams(dataSource, dataSource.getVersionId());
}
return dataSource;
}
@@ -128,8 +128,7 @@ public class DataSourceInfoServiceImpl implements DataSourceInfoService {
public DataSource getDataSourceInfo(Long dataSourceId, Long version) {
DataSource dataSource = dataSourceDao.selectOneDetail(dataSourceId);
if (Objects.nonNull(dataSource)) {
- String parameter = dataSourceVersionDao.selectOneVersion(dataSourceId, version);
- dataSource.setParameter(parameter);
+ mergeVersionParams(dataSource, version);
}
return dataSource;
}
@@ -142,50 +141,42 @@ public class DataSourceInfoServiceImpl implements DataSourceInfoService {
public DataSource getDataSourceInfoForConnect(Long dataSourceId) {
DataSource dataSource = dataSourceDao.selectOneDetail(dataSourceId);
if (Objects.nonNull(dataSource)) {
- // TODO dataSource.getPublishedVersionId() NullPoint Exception
- String parameter =
- dataSourceVersionDao.selectOneVersion(dataSourceId, dataSource.getPublishedVersionId());
- return mergeParams(dataSource, parameter);
+ mergeVersionParams(dataSource, dataSource.getPublishedVersionId());
+ mergeEnvParams(dataSource);
}
- return null;
+ return dataSource;
}
@Override
public DataSource getDataSourceInfoForConnect(String dataSourceName) {
DataSource dataSource = dataSourceDao.selectOneDetailByName(dataSourceName);
if (Objects.nonNull(dataSource)) {
- String parameter =
- dataSourceVersionDao.selectOneVersion(
- dataSource.getId(), dataSource.getPublishedVersionId());
- return mergeParams(dataSource, parameter);
+ mergeVersionParams(dataSource, dataSource.getPublishedVersionId());
+ mergeEnvParams(dataSource);
}
- return null;
+ return dataSource;
}
- private DataSource mergeParams(DataSource dataSource, String parameter) {
- dataSource.setParameter(parameter);
- if (StringUtils.isNotBlank(parameter)) {
- Map<String, String> connectParams = new HashMap<>();
- try {
- connectParams = Objects.requireNonNull(Json.fromJson(parameter, Map.class));
- } catch (JsonErrorException e) {
- LOG.warn(
- "Unrecognized the parameter: "
- + parameter
- + " in data source, id: ["
- + dataSource.getId()
- + "]",
- e);
- // TODO throws Exception defined Exception
- }
- if (connectParams.containsKey("envId")) {
- Long envId = Long.valueOf(connectParams.get("envId"));
- // remove envId for connect
- dataSource.getConnectParams().remove("envId");
- addEnvParamsToDataSource(envId, dataSource);
+ private void mergeEnvParams(DataSource dataSource) {
+ Map<String, Object> connectParams = dataSource.getConnectParams();
+ if (connectParams.containsKey("envId")) {
+ Long envId = Long.valueOf(connectParams.get("envId").toString());
+ // remove envId for connect
+ connectParams.remove("envId");
+ addEnvParamsToDataSource(envId, dataSource);
+ }
+ // if exists multi env
+ if (connectParams.containsKey("envIdArray")) {
+ Object envIdArray = connectParams.get("envIdArray");
+ if (envIdArray instanceof List) {
+ List<String> envIdList = (List<String>) envIdArray;
+ if (CollectionUtils.isNotEmpty(envIdList)) {
+ addEnvParamsToDataSource(Long.valueOf(envIdList.get(0)), dataSource);
+ }
+ // remove envIdArray for connect
+ connectParams.remove("envIdArray");
}
}
- return dataSource;
}
/**
@@ -200,10 +191,10 @@ public class DataSourceInfoServiceImpl implements DataSourceInfoService {
public DataSource getDataSourceInfoForConnect(Long dataSourceId, Long version) {
DataSource dataSource = dataSourceDao.selectOneDetail(dataSourceId);
if (Objects.nonNull(dataSource)) {
- String parameter = dataSourceVersionDao.selectOneVersion(dataSourceId, version);
- return mergeParams(dataSource, parameter);
+ mergeVersionParams(dataSource, version);
+ mergeEnvParams(dataSource);
}
- return null;
+ return dataSource;
}
@Override
@@ -215,6 +206,15 @@ public class DataSourceInfoServiceImpl implements DataSourceInfoService {
return false;
}
+ @Override
+ public boolean existDataSourceEnv(String dataSourceEnvName) {
+ if (StringUtils.isNotBlank(dataSourceEnvName)) {
+ DataSourceEnv dataSourceEnv = dataSourceEnvDao.selectOneByName(dataSourceEnvName);
+ return Objects.nonNull(dataSourceEnv);
+ }
+ return false;
+ }
+
@Override
public DataSource getDataSourceInfoBrief(Long dataSourceId) {
return dataSourceDao.selectOne(dataSourceId);
@@ -268,6 +268,22 @@ public class DataSourceInfoServiceImpl implements DataSourceInfoService {
});
}
+ @Override
+ @Transactional(rollbackFor = Exception.class)
+ public void saveBatchDataSourceEnv(List<DataSourceEnv> dataSourceEnvList) throws ErrorException {
+ for (DataSourceEnv dataSourceEnv : dataSourceEnvList) {
+ storeConnectParams(
+ dataSourceEnv.getCreateUser(),
+ dataSourceEnv.getKeyDefinitions(),
+ dataSourceEnv.getConnectParams(),
+ parameter -> {
+ dataSourceEnv.setParameter(parameter);
+ // Save environment into database
+ dataSourceEnvDao.insertOne(dataSourceEnv);
+ });
+ }
+ }
+
@Override
public List<DataSourceEnv> listDataSourceEnvByType(Long dataSourceTypeId) {
return dataSourceEnvDao.listByTypeId(dataSourceTypeId);
@@ -329,6 +345,25 @@ public class DataSourceInfoServiceImpl implements DataSourceInfoService {
});
}
+ @Transactional(rollbackFor = Exception.class)
+ @Override
+ public void updateBatchDataSourceEnv(List<DataSourceEnv> dataSourceEnvList)
+ throws ErrorException {
+ for (DataSourceEnv updatedOne : dataSourceEnvList) {
+ DataSourceEnv storedOne = getDataSourceEnv(updatedOne.getId());
+ updateConnectParams(
+ updatedOne.getCreateUser(),
+ updatedOne.getKeyDefinitions(),
+ updatedOne.getConnectParams(),
+ storedOne.getConnectParams(),
+ parameter -> {
+ updatedOne.setParameter(parameter);
+ // Update environment into database
+ dataSourceEnvDao.updateOne(updatedOne);
+ });
+ }
+ }
+
@Override
public List<DataSourceEnv> queryDataSourceEnvPage(DataSourceEnvVo dataSourceEnvVo) {
PageHelper.startPage(dataSourceEnvVo.getCurrentPage(), dataSourceEnvVo.getPageSize());
@@ -609,4 +644,19 @@ public class DataSourceInfoServiceImpl implements DataSourceInfoService {
}
}
}
+
+ private void mergeVersionParams(DataSource dataSource, Long version) {
+ if (Objects.isNull(version)) {
+ return;
+ }
+ Map<String, Object> connectParams = dataSource.getConnectParams();
+ String versionParameter = dataSourceVersionDao.selectOneVersion(dataSource.getId(), version);
+ if (StringUtils.isNotBlank(versionParameter)) {
+ try {
+ connectParams.putAll(Objects.requireNonNull(Json.fromJson(versionParameter, Map.class)));
+ } catch (JsonErrorException e) {
+ LOG.warn("Parameter is not json string");
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org