You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by pe...@apache.org on 2023/04/29 14:32:09 UTC
[linkis] branch dev-1.4.0 updated: Support starrocks dataSources (#4496)
This is an automated email from the ASF dual-hosted git repository.
peacewong pushed a commit to branch dev-1.4.0
in repository https://gitbox.apache.org/repos/asf/linkis.git
The following commit(s) were added to refs/heads/dev-1.4.0 by this push:
new b3c105d81 Support starrocks dataSources (#4496)
b3c105d81 is described below
commit b3c105d81a494cb280c0892373012b671ad9a43f
Author: ChengJie1053 <18...@163.com>
AuthorDate: Sat Apr 29 22:32:03 2023 +0800
Support starrocks dataSources (#4496)
* Formatting code
* Modify sql
---
linkis-dist/package/db/linkis_dml.sql | 13 +++
.../db/upgrade/1.4.0_schema/mysql/linkis_dml.sql | 13 +++
.../query/common/cache/CacheConfiguration.java | 2 +-
.../query/service/StarrocksMetaService.java | 94 ++++++++++++++++++++++
4 files changed, 121 insertions(+), 1 deletion(-)
diff --git a/linkis-dist/package/db/linkis_dml.sql b/linkis-dist/package/db/linkis_dml.sql
index f64387866..2ec20f354 100644
--- a/linkis-dist/package/db/linkis_dml.sql
+++ b/linkis-dist/package/db/linkis_dml.sql
@@ -555,6 +555,7 @@ INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `cl
INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('doris', 'doris数据库', 'doris', 'olap', '', 4, 'Doris Database', 'Doris', 'Olap');
INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('clickhouse', 'clickhouse数据库', 'clickhouse', 'olap', '', 4, 'Clickhouse Database', 'Clickhouse', 'Olap');
INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('tidb', 'tidb数据库', 'tidb', '关系型数据库', '', 3, 'TiDB Database', 'TiDB', 'Relational Database');
+INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('starrocks', 'starrocks数据库', 'starrocks', 'olap', '', 4, 'StarRocks Database', 'StarRocks', 'Olap');
select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'mongodb';
@@ -735,3 +736,15 @@ VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL
(@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()),
(@data_source_type_id, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 0, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()),
(@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now());
+
+select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'starrocks';
+INSERT INTO `linkis_ps_dm_datasource_type_key`
+(`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`)
+VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL, 0, '地址(host1:port1,host2:port2...)', 'Address(host1:port1,host2:port2...)', NULL, NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'host', '主机名(Host)', 'Host', NULL, 'TEXT', NULL, 1, '主机名(Host)', 'Host', NULL, NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'port', '端口号(Port)', 'Port', NULL, 'TEXT', NULL, 1, '端口号(Port)', 'Port', NULL, NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'driverClassName', '驱动类名(Driver class name)', 'Driver class name', 'com.mysql.jdbc.Driver', 'TEXT', NULL, 1, '驱动类名(Driver class name)', 'Driver class name', NULL, NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'params', '连接参数(Connection params)', 'Connection params', NULL, 'TEXT', NULL, 0, '输入JSON格式(Input JSON format): {"param":"value"}', 'Input JSON format: {"param":"value"}', NULL, NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 0, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now());
diff --git a/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql
index fc099f20a..66d9beee1 100644
--- a/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql
+++ b/linkis-dist/package/db/upgrade/1.4.0_schema/mysql/linkis_dml.sql
@@ -29,3 +29,16 @@ VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL
(@data_source_type_id, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 0, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()),
(@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now());
+INSERT INTO `linkis_ps_dm_datasource_type` (`name`, `description`, `option`, `classifier`, `icon`, `layers`, `description_en`, `option_en`, `classifier_en`) VALUES ('starrocks', 'starrocks数据库', 'starrocks', 'olap', '', 4, 'StarRocks Database', 'StarRocks', 'Olap');
+
+select @data_source_type_id := id from `linkis_ps_dm_datasource_type` where `name` = 'starrocks';
+INSERT INTO `linkis_ps_dm_datasource_type_key`
+(`data_source_type_id`, `key`, `name`, `name_en`, `default_value`, `value_type`, `scope`, `require`, `description`, `description_en`, `value_regex`, `ref_id`, `ref_value`, `data_source`, `update_time`, `create_time`)
+VALUES (@data_source_type_id, 'address', '地址', 'Address', NULL, 'TEXT', NULL, 0, '地址(host1:port1,host2:port2...)', 'Address(host1:port1,host2:port2...)', NULL, NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'host', '主机名(Host)', 'Host', NULL, 'TEXT', NULL, 1, '主机名(Host)', 'Host', NULL, NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'port', '端口号(Port)', 'Port', NULL, 'TEXT', NULL, 1, '端口号(Port)', 'Port', NULL, NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'driverClassName', '驱动类名(Driver class name)', 'Driver class name', 'com.mysql.jdbc.Driver', 'TEXT', NULL, 1, '驱动类名(Driver class name)', 'Driver class name', NULL, NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'params', '连接参数(Connection params)', 'Connection params', NULL, 'TEXT', NULL, 0, '输入JSON格式(Input JSON format): {"param":"value"}', 'Input JSON format: {"param":"value"}', NULL, NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'username', '用户名(Username)', 'Username', NULL, 'TEXT', NULL, 1, '用户名(Username)', 'Username', '^[0-9A-Za-z_-]+$', NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'password', '密码(Password)', 'Password', NULL, 'PASSWORD', NULL, 0, '密码(Password)', 'Password', '', NULL, NULL, NULL, now(), now()),
+ (@data_source_type_id, 'instance', '实例名(instance)', 'Instance', NULL, 'TEXT', NULL, 1, '实例名(instance)', 'Instance', NULL, NULL, NULL, NULL, now(), now());
diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java
index 54211b6ae..54baec4b7 100644
--- a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java
+++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/common/src/main/java/org/apache/linkis/metadata/query/common/cache/CacheConfiguration.java
@@ -34,5 +34,5 @@ public class CacheConfiguration {
public static final CommonVars<String> MYSQL_RELATIONSHIP_LIST =
CommonVars.apply(
"wds.linkis.server.mdq.mysql.relationship",
- "mysql,oracle,kingbase,postgresql,sqlserver,db2,greenplum,dm,doris,clickhouse,tidb");
+ "mysql,oracle,kingbase,postgresql,sqlserver,db2,greenplum,dm,doris,clickhouse,tidb,starrocks");
}
diff --git a/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/StarrocksMetaService.java b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/StarrocksMetaService.java
new file mode 100644
index 000000000..6d4a146d7
--- /dev/null
+++ b/linkis-public-enhancements/linkis-datasource/linkis-metadata-query/service/jdbc/src/main/java/org/apache/linkis/metadata/query/service/StarrocksMetaService.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.metadata.query.service;
+
+import org.apache.linkis.datasourcemanager.common.util.json.Json;
+import org.apache.linkis.metadata.query.common.domain.MetaColumnInfo;
+import org.apache.linkis.metadata.query.common.service.AbstractDbMetaService;
+import org.apache.linkis.metadata.query.common.service.MetadataConnection;
+import org.apache.linkis.metadata.query.service.conf.SqlParamsMapper;
+import org.apache.linkis.metadata.query.service.mysql.SqlConnection;
+
+import org.springframework.stereotype.Component;
+
+import java.sql.SQLException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Component
+public class StarrocksMetaService extends AbstractDbMetaService<SqlConnection> {
+ @Override
+ public MetadataConnection<SqlConnection> getConnection(
+ String operator, Map<String, Object> params) throws Exception {
+ String host =
+ String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_HOST.getValue(), ""));
+ Integer port =
+ (Double.valueOf(
+ String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_PORT.getValue(), 0))))
+ .intValue();
+ String username =
+ String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_USERNAME.getValue(), ""));
+ String password =
+ String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_PASSWORD.getValue(), ""));
+
+ String database =
+ String.valueOf(params.getOrDefault(SqlParamsMapper.PARAM_SQL_DATABASE.getValue(), ""));
+ Map<String, Object> extraParams = new HashMap<>();
+ Object sqlParamObj = params.get(SqlParamsMapper.PARAM_SQL_EXTRA_PARAMS.getValue());
+ if (null != sqlParamObj) {
+ if (!(sqlParamObj instanceof Map)) {
+ extraParams =
+ Json.fromJson(String.valueOf(sqlParamObj), Map.class, String.class, Object.class);
+ } else {
+ extraParams = (Map<String, Object>) sqlParamObj;
+ }
+ }
+ assert extraParams != null;
+ return new MetadataConnection<>(
+ new SqlConnection(host, port, username, password, database, extraParams));
+ }
+
+ @Override
+ public List<String> queryDatabases(SqlConnection connection) {
+ try {
+ return connection.getAllDatabases();
+ } catch (SQLException e) {
+ throw new RuntimeException("Fail to get Sql databases(获取数据库列表失败)", e);
+ }
+ }
+
+ @Override
+ public List<String> queryTables(SqlConnection connection, String database) {
+ try {
+ return connection.getAllTables(database);
+ } catch (SQLException e) {
+ throw new RuntimeException("Fail to get Sql tables(获取表列表失败)", e);
+ }
+ }
+
+ @Override
+ public List<MetaColumnInfo> queryColumns(
+ SqlConnection connection, String database, String table) {
+ try {
+ return connection.getColumns(database, table);
+ } catch (SQLException | ClassNotFoundException e) {
+ throw new RuntimeException("Fail to get Sql columns(获取字段列表失败)", e);
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org