You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@linkis.apache.org by pe...@apache.org on 2022/03/24 10:53:01 UTC
[incubator-linkis] branch dev-1.1.0 updated: add test for datasource client (#1796)
This is an automated email from the ASF dual-hosted git repository.
peacewong pushed a commit to branch dev-1.1.0
in repository https://gitbox.apache.org/repos/asf/incubator-linkis.git
The following commit(s) were added to refs/heads/dev-1.1.0 by this push:
new 0852b6d add test for datasource client (#1796)
0852b6d is described below
commit 0852b6d2f616e930b8b3ad343c87e4af741f637a
Author: Casion <ca...@gmail.com>
AuthorDate: Thu Mar 24 18:52:56 2022 +0800
add test for datasource client (#1796)
---
.../request/PublishDataSourceVersionAction.scala | 8 +-
.../linkis/datasource/client/TestHiveClient.scala | 143 ++++++++++++++++++++
.../linkis/datasource/client/TestMysqlClient.scala | 149 +++++++++++++++++++++
web/src/apps/linkis/i18n/common/en.json | 3 +-
web/src/apps/linkis/i18n/common/zh.json | 3 +-
web/src/apps/linkis/module/datasource/index.vue | 16 ++-
6 files changed, 315 insertions(+), 7 deletions(-)
diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-client/src/main/scala/org/apache/linkis/datasource/client/request/PublishDataSourceVersionAction.scala b/linkis-public-enhancements/linkis-datasource/linkis-datasource-client/src/main/scala/org/apache/linkis/datasource/client/request/PublishDataSourceVersionAction.scala
index 404abd5..59b8652 100644
--- a/linkis-public-enhancements/linkis-datasource/linkis-datasource-client/src/main/scala/org/apache/linkis/datasource/client/request/PublishDataSourceVersionAction.scala
+++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-client/src/main/scala/org/apache/linkis/datasource/client/request/PublishDataSourceVersionAction.scala
@@ -27,13 +27,13 @@ class PublishDataSourceVersionAction extends POSTAction with DataSourceAction{
override def getRequestPayload: String = DWSHttpClient.jacksonJson.writeValueAsString(getRequestPayloads)
private var user: String = _
private var dataSourceId: Long = _
- private var versionId: String = _
+ private var versionId: Long = _
override def setUser(user: String): Unit = this.user = user
override def getUser: String = this.user
- override def suffixURLs: Array[String] = Array(DATA_SOURCE_SERVICE_MODULE.getValue, "publish", dataSourceId.toString, versionId)
+ override def suffixURLs: Array[String] = Array(DATA_SOURCE_SERVICE_MODULE.getValue, "publish", dataSourceId.toString, versionId.toString)
}
object PublishDataSourceVersionAction {
def builder(): Builder = new Builder
@@ -41,7 +41,7 @@ object PublishDataSourceVersionAction {
class Builder private[PublishDataSourceVersionAction]() {
private var user: String = _
private var dataSourceId: Long = _
- private var versionId: String = _
+ private var versionId: Long = _
def setUser(user: String): Builder = {
this.user = user
@@ -53,7 +53,7 @@ object PublishDataSourceVersionAction {
this
}
- def setVersion(versionId: String): Builder = {
+ def setVersion(versionId: Long): Builder = {
this.versionId = versionId
this
}
diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-client/src/test/java/org/apache/linkis/datasource/client/TestHiveClient.scala b/linkis-public-enhancements/linkis-datasource/linkis-datasource-client/src/test/java/org/apache/linkis/datasource/client/TestHiveClient.scala
new file mode 100644
index 0000000..c677f73
--- /dev/null
+++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-client/src/test/java/org/apache/linkis/datasource/client/TestHiveClient.scala
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.datasource.client
+
+import java.util
+import java.util.concurrent.TimeUnit
+
+import org.apache.linkis.common.utils.JsonUtils
+import org.apache.linkis.datasource.client.impl.{LinkisDataSourceRemoteClient, LinkisMetaDataRemoteClient}
+import org.apache.linkis.datasource.client.request._
+import org.apache.linkis.datasource.client.response._
+import org.apache.linkis.datasourcemanager.common.domain.DataSource
+import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy
+import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder
+import org.junit.jupiter.api.{Disabled, Test}
+
+object TestHiveClient {
+ val gatewayUrl = "http://127.0.0.1:9001"
+ val clientConfig = DWSClientConfigBuilder.newBuilder
+ .addServerUrl(gatewayUrl)
+ .connectionTimeout(30000)
+ .discoveryEnabled(false)
+ .discoveryFrequency(1, TimeUnit.MINUTES)
+ .loadbalancerEnabled(true)
+ .maxConnectionSize(1)
+ .retryEnabled(false)
+ .readTimeout(30000)
+ .setAuthenticationStrategy(new StaticAuthenticationStrategy)
+ .setAuthTokenKey("hadoop")
+ .setAuthTokenValue("xxxxx")
+ .setDWSVersion("v1")
+
+ val dataSourceclient = new LinkisDataSourceRemoteClient(clientConfig.build())
+
+ val clientConfig2 = DWSClientConfigBuilder.newBuilder
+ .addServerUrl(gatewayUrl)
+ .connectionTimeout(30000)
+ .discoveryEnabled(false)
+ .discoveryFrequency(1, TimeUnit.MINUTES)
+ .loadbalancerEnabled(true)
+ .maxConnectionSize(1)
+ .retryEnabled(false)
+ .readTimeout(30000)
+ .setAuthenticationStrategy(new StaticAuthenticationStrategy)
+ .setAuthTokenKey("hadoop")
+ .setAuthTokenValue("xxxxx")
+ .setDWSVersion("v1")
+
+ val metaDataClient = new LinkisMetaDataRemoteClient(clientConfig2.build())
+
+
+ @Test
+ @Disabled
+ def testCreateDataSourceMysql: Unit = {
+ val user = "hadoop"
+ val system = "Linkis"
+ val dataSource = new DataSource();
+ dataSource.setDataSourceName("for-hive-test")
+ dataSource.setDataSourceDesc("this is for hive test")
+ dataSource.setCreateSystem(system)
+ dataSource.setDataSourceTypeId(4L)
+
+ val map = JsonUtils.jackson.readValue(JsonUtils.jackson.writeValueAsString(dataSource), new util.HashMap[String, Any]().getClass)
+ val createDataSourceAction: CreateDataSourceAction = CreateDataSourceAction.builder()
+ .setUser(user)
+ .addRequestPayloads(map)
+ .build()
+ val createDataSourceResult: CreateDataSourceResult = dataSourceclient.createDataSource(createDataSourceAction)
+ val dataSourceId = createDataSourceResult.getInsertId
+
+ //set connectParams
+ val params = new util.HashMap[String, Any]
+ val connectParams = new util.HashMap[String, Any]
+ connectParams.put("envId", "3")
+ params.put("connectParams", connectParams)
+ params.put("comment", "init")
+
+ val updateParameterAction: UpdateDataSourceParameterAction = UpdateDataSourceParameterAction.builder()
+ .setUser(user)
+ .setDataSourceId(dataSourceId)
+ .addRequestPayloads(params)
+ .build()
+ val updateParameterResult: UpdateDataSourceParameterResult = dataSourceclient.updateDataSourceParameter(updateParameterAction)
+
+ val version: Long = updateParameterResult.getVersion
+
+ //publist dataSource version
+ dataSourceclient.publishDataSourceVersion(
+ PublishDataSourceVersionAction.builder()
+ .setDataSourceId(dataSourceId)
+ .setUser(user)
+ .setVersion(version)
+ .build())
+
+ //example of use
+ val metadataGetDatabasesAction: MetadataGetDatabasesAction = MetadataGetDatabasesAction.builder()
+ .setUser(user)
+ .setDataSourceId(dataSourceId)
+ .setSystem(system)
+ .build()
+ val metadataGetDatabasesResult: MetadataGetDatabasesResult = metaDataClient.getDatabases(metadataGetDatabasesAction)
+
+ val metadataGetTablesAction: MetadataGetTablesAction = MetadataGetTablesAction.builder()
+ .setUser(user)
+ .setDataSourceId(dataSourceId)
+ .setDatabase("linkis_test_ind")
+ .setSystem(system)
+ .build()
+ val metadataGetTablesResult: MetadataGetTablesResult = metaDataClient.getTables(metadataGetTablesAction)
+
+
+
+ val metadataGetColumnsAction = MetadataGetColumnsAction.builder()
+ .setUser(user)
+ .setDataSourceId(dataSourceId)
+ .setDatabase("linkis_test_ind")
+ .setSystem(system)
+ .setTable("test")
+ .build()
+ val metadataGetColumnsResult: MetadataGetColumnsResult = metaDataClient.getColumns(metadataGetColumnsAction)
+
+
+ }
+}
+
+
+
+
diff --git a/linkis-public-enhancements/linkis-datasource/linkis-datasource-client/src/test/java/org/apache/linkis/datasource/client/TestMysqlClient.scala b/linkis-public-enhancements/linkis-datasource/linkis-datasource-client/src/test/java/org/apache/linkis/datasource/client/TestMysqlClient.scala
new file mode 100644
index 0000000..7f9a81d
--- /dev/null
+++ b/linkis-public-enhancements/linkis-datasource/linkis-datasource-client/src/test/java/org/apache/linkis/datasource/client/TestMysqlClient.scala
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.linkis.datasource.client
+import java.util
+import java.util.concurrent.TimeUnit
+
+import org.apache.linkis.common.utils.JsonUtils
+import org.apache.linkis.datasource.client.impl.{LinkisDataSourceRemoteClient, LinkisMetaDataRemoteClient}
+import org.apache.linkis.datasource.client.request._
+import org.apache.linkis.datasource.client.response._
+import org.apache.linkis.datasourcemanager.common.domain.DataSource
+import org.apache.linkis.httpclient.dws.authentication.StaticAuthenticationStrategy
+import org.apache.linkis.httpclient.dws.config.DWSClientConfigBuilder
+import org.junit.jupiter.api.{Disabled, Test}
+
+object TestMysqlClient {
+
+ val gatewayUrl = "http://127.0.0.1:9001"
+ val clientConfig = DWSClientConfigBuilder.newBuilder
+ .addServerUrl(gatewayUrl)
+ .connectionTimeout(30000)
+ .discoveryEnabled(false)
+ .discoveryFrequency(1, TimeUnit.MINUTES)
+ .loadbalancerEnabled(true)
+ .maxConnectionSize(1)
+ .retryEnabled(false)
+ .readTimeout(30000)
+ .setAuthenticationStrategy(new StaticAuthenticationStrategy)
+ .setAuthTokenKey("hadoop")
+ .setAuthTokenValue("xxxxx")
+ .setDWSVersion("v1")
+
+ val dataSourceclient = new LinkisDataSourceRemoteClient(clientConfig.build())
+
+ val clientConfig2 = DWSClientConfigBuilder.newBuilder
+ .addServerUrl(gatewayUrl)
+ .connectionTimeout(30000)
+ .discoveryEnabled(false)
+ .discoveryFrequency(1, TimeUnit.MINUTES)
+ .loadbalancerEnabled(true)
+ .maxConnectionSize(1)
+ .retryEnabled(false)
+ .readTimeout(30000)
+ .setAuthenticationStrategy(new StaticAuthenticationStrategy)
+ .setAuthTokenKey("hadoop")
+ .setAuthTokenValue("xxxxx")
+ .setDWSVersion("v1")
+
+ val metaDataClient = new LinkisMetaDataRemoteClient(clientConfig2.build())
+
+ @Test
+ @Disabled
+ def testCreateDataSourceMysql: Unit = {
+ val user = "hadoop"
+ val system = "Linkis"
+ val dataSource = new DataSource();
+ dataSource.setDataSourceName("for-mysql-test")
+ dataSource.setDataSourceDesc("this is for mysql test")
+ dataSource.setCreateSystem(system)
+ dataSource.setDataSourceTypeId(1L)
+
+ val map = JsonUtils.jackson.readValue(JsonUtils.jackson.writeValueAsString(dataSource), new util.HashMap[String, Any]().getClass)
+ val createDataSourceAction: CreateDataSourceAction = CreateDataSourceAction.builder()
+ .setUser(user)
+ .addRequestPayloads(map)
+ .build()
+ val createDataSourceResult: CreateDataSourceResult = dataSourceclient.createDataSource(createDataSourceAction)
+ val dataSourceId = createDataSourceResult.getInsertId
+
+
+ //set connectParams
+ val params = new util.HashMap[String, Any]
+
+ val connectParams = new util.HashMap[String, Any]
+ connectParams.put("host", "127.0.0.1")
+ connectParams.put("port", "36000")
+ connectParams.put("username", "db username")
+ connectParams.put("password", "db password")
+
+ params.put("connectParams", connectParams)
+ params.put("comment", "init")
+
+ val updateParameterAction: UpdateDataSourceParameterAction = UpdateDataSourceParameterAction.builder()
+ .setUser(user)
+ .setDataSourceId(dataSourceId)
+ .addRequestPayloads(params)
+ .build()
+ val updateParameterResult: UpdateDataSourceParameterResult = dataSourceclient.updateDataSourceParameter(updateParameterAction)
+
+ val version: Long = updateParameterResult.getVersion
+
+ //publist dataSource version
+ dataSourceclient.publishDataSourceVersion(
+ PublishDataSourceVersionAction.builder()
+ .setDataSourceId(dataSourceId)
+ .setUser(user)
+ .setVersion(version)
+ .build())
+
+ //example of use
+ val metadataGetDatabasesAction: MetadataGetDatabasesAction = MetadataGetDatabasesAction.builder()
+ .setUser(user)
+ .setDataSourceId(dataSourceId)
+ .setSystem(system)
+ .build()
+ val metadataGetDatabasesResult: MetadataGetDatabasesResult = metaDataClient.getDatabases(metadataGetDatabasesAction)
+
+
+ val metadataGetTablesAction: MetadataGetTablesAction = MetadataGetTablesAction.builder()
+ .setUser(user)
+ .setDataSourceId(dataSourceId)
+ .setDatabase("linkis")
+ .setSystem(system)
+ .build()
+ val metadataGetTablesResult: MetadataGetTablesResult = metaDataClient.getTables(metadataGetTablesAction)
+
+
+
+ val metadataGetColumnsAction = MetadataGetColumnsAction.builder()
+ .setUser(user)
+ .setDataSourceId(dataSourceId)
+ .setDatabase("linkis")
+ .setSystem(system)
+ .setTable("linkis_datasource")
+ .build()
+ val metadataGetColumnsResult: MetadataGetColumnsResult = metaDataClient.getColumns(metadataGetColumnsAction)
+
+
+ }
+}
+
+
+
+
diff --git a/web/src/apps/linkis/i18n/common/en.json b/web/src/apps/linkis/i18n/common/en.json
index 8f671a8..bef9fe9 100644
--- a/web/src/apps/linkis/i18n/common/en.json
+++ b/web/src/apps/linkis/i18n/common/en.json
@@ -253,6 +253,7 @@
"versionList": "Version list",
"dataSourceName": "Data source name",
"dataSourceType": "Data source type",
+ "createSystem": "Create system",
"dataSourceEnv": "Available Space",
"status": "Status",
"permissions": "Permissions",
@@ -276,4 +277,4 @@
}
}
}
-}
\ No newline at end of file
+}
diff --git a/web/src/apps/linkis/i18n/common/zh.json b/web/src/apps/linkis/i18n/common/zh.json
index 3b8f724..8b93bd6 100644
--- a/web/src/apps/linkis/i18n/common/zh.json
+++ b/web/src/apps/linkis/i18n/common/zh.json
@@ -256,6 +256,7 @@
"versionList": "版本列表",
"dataSourceName": "数据源名称",
"dataSourceType": "数据源类型",
+ "createSystem": "创建系统",
"dataSourceEnv": "可用集群",
"status": "状态",
"permissions": "权限",
@@ -279,4 +280,4 @@
}
}
}
-}
\ No newline at end of file
+}
diff --git a/web/src/apps/linkis/module/datasource/index.vue b/web/src/apps/linkis/module/datasource/index.vue
index b9d35a4..f976dab 100644
--- a/web/src/apps/linkis/module/datasource/index.vue
+++ b/web/src/apps/linkis/module/datasource/index.vue
@@ -266,6 +266,13 @@ export default {
},
tableColumnNum: [
{
+ title: "Id",
+ key: 'id',
+ minWidth: 10,
+ tooltip: true,
+ align: 'center',
+ },
+ {
title: this.$t('message.linkis.datasource.dataSourceName'),
key: 'dataSourceName',
minWidth: 120,
@@ -275,7 +282,7 @@ export default {
{
title: this.$t('message.linkis.datasource.dataSourceType'),
key: 'dataSourceTypeId',
- minWidth: 100,
+ minWidth: 60,
render: (h, params) => {
let result = {}
if (this.typeList) {
@@ -303,6 +310,13 @@ export default {
// align: 'center'
// },
{
+ title: this.$t('message.linkis.datasource.createSystem'),
+ key: 'createSystem',
+ minWidth: 50,
+ tooltip: true,
+ align: 'center',
+ },
+ {
title: this.$t('message.linkis.datasource.status'),
key: 'status',
tooltip: true,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@linkis.apache.org
For additional commands, e-mail: commits-help@linkis.apache.org