You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zh...@apache.org on 2024/02/06 10:48:44 UTC
(dolphinscheduler) branch 3.2.1-prepare updated (58aba58079 -> 097ad90708)
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a change to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
from 58aba58079 [fix][worker]:Fix the issue of missing disk usage reporting information in the worker (#15565)
new 57a4325821 Revert "Fix k8sTaskExecutionContext setting configYaml (#15116)"
new 097ad90708 Revert "[Improvement] Refactoring K8S task plugin with connections managed in connection center (#14977)"
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../dolphinscheduler-datasource-all/pom.xml | 5 -
.../dolphinscheduler-datasource-k8s/pom.xml | 50 --------
.../plugin/datasource/k8s/K8sClientWrapper.java | 55 ---------
.../datasource/k8s/K8sDataSourceChannel.java | 37 ------
.../k8s/K8sDataSourceChannelFactory.java | 38 ------
.../datasource/k8s/param/K8sConnectionParam.java | 32 -----
.../k8s/param/K8sDataSourceParamDTO.java | 35 ------
.../k8s/param/K8sDataSourceProcessor.java | 129 ---------------------
.../datasource/k8s/K8sDataSourceProcessorTest.java | 107 -----------------
.../SagemakerDataSourceProcessorTest.java | 1 +
dolphinscheduler-datasource-plugin/pom.xml | 1 -
.../e2e/pages/datasource/DataSourcePage.java | 6 -
.../apache/dolphinscheduler/spi/enums/DbType.java | 4 +-
.../plugin/task/api/K8sTaskExecutionContext.java | 10 +-
.../plugin/task/api/k8s/K8sTaskMainParameters.java | 1 +
.../task/api/parameters/AbstractParameters.java | 13 ---
.../task/api/parameters/K8sTaskParameters.java | 17 +--
.../plugin/task/api/k8s/K8sTaskExecutorTest.java | 12 +-
.../dolphinscheduler/plugin/task/k8s/K8sTask.java | 39 ++-----
.../plugin/task/k8s/K8sTaskChannel.java | 2 +-
.../plugin/task/k8s/K8sParametersTest.java | 2 +-
.../plugin/task/k8s/K8sTaskTest.java | 83 ++++---------
.../plugin/task/sagemaker/SagemakerTaskTest.java | 1 +
.../plugin/task/zeppelin/ZeppelinTask.java | 2 +-
.../src/service/modules/data-source/types.ts | 4 -
.../src/views/datasource/list/detail.tsx | 41 +------
.../src/views/datasource/list/use-form.ts | 23 +---
.../task/components/node/fields/use-k8s.ts | 8 +-
.../projects/task/components/node/format-data.ts | 3 -
.../projects/task/components/node/tasks/use-k8s.ts | 7 +-
30 files changed, 71 insertions(+), 697 deletions(-)
delete mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/pom.xml
delete mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sClientWrapper.java
delete mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannel.java
delete mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannelFactory.java
delete mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sConnectionParam.java
delete mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceParamDTO.java
delete mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceProcessor.java
delete mode 100644 dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/test/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceProcessorTest.java
(dolphinscheduler) 02/02: Revert "[Improvement] Refactoring K8S task plugin with connections managed in connection center (#14977)"
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit 097ad90708e518f0a8a774ff1a378cf974f388b2
Author: Jay Chung <zh...@gmail.com>
AuthorDate: Tue Feb 6 18:47:09 2024 +0800
Revert "[Improvement] Refactoring K8S task plugin with connections managed in connection center (#14977)"
This reverts commit c532fea00ef876a920551067eb2f5c72e2b72d61.
---
.../dolphinscheduler-datasource-all/pom.xml | 5 -
.../dolphinscheduler-datasource-k8s/pom.xml | 50 --------
.../plugin/datasource/k8s/K8sClientWrapper.java | 55 ---------
.../datasource/k8s/K8sDataSourceChannel.java | 37 ------
.../k8s/K8sDataSourceChannelFactory.java | 38 ------
.../datasource/k8s/param/K8sConnectionParam.java | 32 -----
.../k8s/param/K8sDataSourceParamDTO.java | 35 ------
.../k8s/param/K8sDataSourceProcessor.java | 129 ---------------------
.../datasource/k8s/K8sDataSourceProcessorTest.java | 107 -----------------
.../SagemakerDataSourceProcessorTest.java | 1 +
dolphinscheduler-datasource-plugin/pom.xml | 1 -
.../e2e/pages/datasource/DataSourcePage.java | 6 -
.../apache/dolphinscheduler/spi/enums/DbType.java | 4 +-
.../plugin/task/api/K8sTaskExecutionContext.java | 10 +-
.../task/api/am/KubernetesApplicationManager.java | 3 -
.../plugin/task/api/k8s/K8sTaskMainParameters.java | 1 +
.../plugin/task/api/k8s/impl/K8sTaskExecutor.java | 4 +-
.../task/api/parameters/K8sTaskParameters.java | 28 +----
.../plugin/task/api/k8s/K8sTaskExecutorTest.java | 12 +-
.../dolphinscheduler/plugin/task/k8s/K8sTask.java | 37 ++----
.../plugin/task/k8s/K8sTaskChannel.java | 2 +-
.../plugin/task/k8s/K8sParametersTest.java | 2 +-
.../plugin/task/k8s/K8sTaskTest.java | 83 ++++---------
.../plugin/task/sagemaker/SagemakerTaskTest.java | 1 +
.../plugin/task/zeppelin/ZeppelinTask.java | 2 +-
.../src/service/modules/data-source/types.ts | 4 -
.../src/views/datasource/list/detail.tsx | 41 +------
.../src/views/datasource/list/use-form.ts | 23 +---
.../task/components/node/fields/use-k8s.ts | 8 +-
.../projects/task/components/node/format-data.ts | 3 -
.../projects/task/components/node/tasks/use-k8s.ts | 7 +-
31 files changed, 73 insertions(+), 698 deletions(-)
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml
index effe3c9abb..987e419e7f 100644
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml
@@ -148,11 +148,6 @@
<artifactId>dolphinscheduler-datasource-sagemaker</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-k8s</artifactId>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.dolphinscheduler</groupId>
<artifactId>dolphinscheduler-datasource-hana</artifactId>
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/pom.xml b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/pom.xml
deleted file mode 100644
index 2d51b85fed..0000000000
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/pom.xml
+++ /dev/null
@@ -1,50 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-plugin</artifactId>
- <version>dev-SNAPSHOT</version>
- </parent>
-
- <artifactId>dolphinscheduler-datasource-k8s</artifactId>
- <packaging>jar</packaging>
- <name>${project.artifactId}</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-spi</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.dolphinscheduler</groupId>
- <artifactId>dolphinscheduler-datasource-api</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>io.fabric8</groupId>
- <artifactId>kubernetes-client</artifactId>
- </dependency>
- </dependencies>
-
-</project>
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sClientWrapper.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sClientWrapper.java
deleted file mode 100644
index 292eaf2f3d..0000000000
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sClientWrapper.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.plugin.datasource.k8s;
-
-import lombok.extern.slf4j.Slf4j;
-import io.fabric8.kubernetes.api.model.NamespaceList;
-import io.fabric8.kubernetes.client.Config;
-import io.fabric8.kubernetes.client.KubernetesClient;
-import io.fabric8.kubernetes.client.KubernetesClientBuilder;
-
-@Slf4j
-public class K8sClientWrapper implements AutoCloseable {
-
- private KubernetesClient client;
-
- public K8sClientWrapper() {
- }
-
- public boolean checkConnect(String kubeConfigYaml, String namespace) {
- try {
- Config config = Config.fromKubeconfig(kubeConfigYaml);
- client = new KubernetesClientBuilder().withConfig(config).build();
- NamespaceList namespaceList = client.namespaces().list();
- if (!namespaceList.getItems().stream().anyMatch(ns -> ns.getMetadata().getName().equals(namespace))) {
- log.info("failed to connect to the K8S cluster, namespace not found\n");
- return false;
- }
- log.info("successfully connected to the K8S cluster");
- return true;
- } catch (Exception e) {
- log.info("failed to connect to the K8S cluster\n");
- return false;
- }
- }
-
- @Override
- public void close() throws Exception {
-
- }
-}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannel.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannel.java
deleted file mode 100644
index 3dac639259..0000000000
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannel.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.plugin.datasource.k8s;
-
-import org.apache.dolphinscheduler.spi.datasource.AdHocDataSourceClient;
-import org.apache.dolphinscheduler.spi.datasource.BaseConnectionParam;
-import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
-import org.apache.dolphinscheduler.spi.datasource.PooledDataSourceClient;
-import org.apache.dolphinscheduler.spi.enums.DbType;
-
-public class K8sDataSourceChannel implements DataSourceChannel {
-
- @Override
- public AdHocDataSourceClient createAdHocDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) {
- throw new UnsupportedOperationException("K8S AdHocDataSourceClient is not supported");
- }
-
- @Override
- public PooledDataSourceClient createPooledDataSourceClient(BaseConnectionParam baseConnectionParam, DbType dbType) {
- throw new UnsupportedOperationException("K8S AdHocDataSourceClient is not supported");
- }
-}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannelFactory.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannelFactory.java
deleted file mode 100644
index 03ec046de8..0000000000
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceChannelFactory.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.plugin.datasource.k8s;
-
-import org.apache.dolphinscheduler.spi.datasource.DataSourceChannel;
-import org.apache.dolphinscheduler.spi.datasource.DataSourceChannelFactory;
-
-import com.google.auto.service.AutoService;
-
-@AutoService(DataSourceChannelFactory.class)
-public class K8sDataSourceChannelFactory implements DataSourceChannelFactory {
-
- @Override
- public DataSourceChannel create() {
- return new K8sDataSourceChannel();
- }
-
- @Override
- public String getName() {
- return "k8s";
- }
-
-}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sConnectionParam.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sConnectionParam.java
deleted file mode 100644
index 09e3e706a1..0000000000
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sConnectionParam.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.plugin.datasource.k8s.param;
-
-import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
-
-import lombok.Data;
-
-import com.fasterxml.jackson.annotation.JsonInclude;
-
-@Data
-@JsonInclude(JsonInclude.Include.NON_NULL)
-public class K8sConnectionParam implements ConnectionParam {
-
- protected String kubeConfig;
- protected String namespace;
-}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceParamDTO.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceParamDTO.java
deleted file mode 100644
index f2ea50a40b..0000000000
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceParamDTO.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.plugin.datasource.k8s.param;
-
-import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
-import org.apache.dolphinscheduler.spi.enums.DbType;
-
-import lombok.Data;
-
-@Data
-public class K8sDataSourceParamDTO extends BaseDataSourceParamDTO {
-
- protected String kubeConfig;
- protected String namespace;
-
- @Override
- public DbType getType() {
- return DbType.K8S;
- }
-}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceProcessor.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceProcessor.java
deleted file mode 100644
index 9e7342d433..0000000000
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/main/java/org/apache/dolphinscheduler/plugin/datasource/k8s/param/K8sDataSourceProcessor.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.plugin.datasource.k8s.param;
-
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.plugin.datasource.api.datasource.AbstractDataSourceProcessor;
-import org.apache.dolphinscheduler.plugin.datasource.api.datasource.BaseDataSourceParamDTO;
-import org.apache.dolphinscheduler.plugin.datasource.api.datasource.DataSourceProcessor;
-import org.apache.dolphinscheduler.plugin.datasource.api.utils.PasswordUtils;
-import org.apache.dolphinscheduler.plugin.datasource.k8s.K8sClientWrapper;
-import org.apache.dolphinscheduler.spi.datasource.ConnectionParam;
-import org.apache.dolphinscheduler.spi.enums.DbType;
-
-import org.apache.commons.lang3.StringUtils;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.SQLException;
-import java.text.MessageFormat;
-
-import lombok.extern.slf4j.Slf4j;
-
-import com.google.auto.service.AutoService;
-
-@AutoService(DataSourceProcessor.class)
-@Slf4j
-public class K8sDataSourceProcessor extends AbstractDataSourceProcessor {
-
- @Override
- public BaseDataSourceParamDTO castDatasourceParamDTO(String paramJson) {
- return JSONUtils.parseObject(paramJson, K8sDataSourceParamDTO.class);
-
- }
-
- @Override
- public void checkDatasourceParam(BaseDataSourceParamDTO datasourceParam) {
- K8sDataSourceParamDTO k8sDataSourceParamDTO = (K8sDataSourceParamDTO) datasourceParam;
- if (StringUtils.isEmpty(k8sDataSourceParamDTO.getKubeConfig())) {
- throw new IllegalArgumentException("sagemaker datasource param is not valid");
- }
- }
-
- @Override
- public String getDatasourceUniqueId(ConnectionParam connectionParam, DbType dbType) {
- K8sConnectionParam baseConnectionParam = (K8sConnectionParam) connectionParam;
- return MessageFormat.format("{0}@{1}@{2}", dbType.getDescp(),
- PasswordUtils.encodePassword(baseConnectionParam.getKubeConfig()), baseConnectionParam.getNamespace());
- }
-
- @Override
- public BaseDataSourceParamDTO createDatasourceParamDTO(String connectionJson) {
- K8sConnectionParam connectionParams = (K8sConnectionParam) createConnectionParams(connectionJson);
- K8sDataSourceParamDTO k8sDataSourceParamDTO = new K8sDataSourceParamDTO();
- k8sDataSourceParamDTO.setKubeConfig(connectionParams.getKubeConfig());
- k8sDataSourceParamDTO.setNamespace(connectionParams.getNamespace());
- return k8sDataSourceParamDTO;
- }
-
- @Override
- public K8sConnectionParam createConnectionParams(BaseDataSourceParamDTO datasourceParam) {
- K8sDataSourceParamDTO k8sDataSourceParam = (K8sDataSourceParamDTO) datasourceParam;
- K8sConnectionParam k8sConnectionParam = new K8sConnectionParam();
- k8sConnectionParam.setKubeConfig(k8sDataSourceParam.getKubeConfig());
- k8sConnectionParam.setNamespace(k8sDataSourceParam.getNamespace());
- return k8sConnectionParam;
- }
-
- @Override
- public ConnectionParam createConnectionParams(String connectionJson) {
- return JSONUtils.parseObject(connectionJson, K8sConnectionParam.class);
- }
-
- @Override
- public String getDatasourceDriver() {
- return "";
- }
-
- @Override
- public String getValidationQuery() {
- return "";
- }
-
- @Override
- public String getJdbcUrl(ConnectionParam connectionParam) {
- return "";
- }
-
- @Override
- public Connection getConnection(ConnectionParam connectionParam) throws ClassNotFoundException, SQLException, IOException {
- return null;
- }
-
- @Override
- public boolean checkDataSourceConnectivity(ConnectionParam connectionParam) {
- K8sConnectionParam baseConnectionParam = (K8sConnectionParam) connectionParam;
- try (
- K8sClientWrapper k8sClientWrapper = new K8sClientWrapper()) {
- return k8sClientWrapper.checkConnect(baseConnectionParam.kubeConfig, baseConnectionParam.namespace);
- } catch (Exception e) {
- log.error("failed to connect to the K8S cluster", e);
- return false;
- }
- }
-
- @Override
- public DbType getDbType() {
- return DbType.K8S;
- }
-
- @Override
- public DataSourceProcessor create() {
- return new K8sDataSourceProcessor();
- }
-}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/test/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/test/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceProcessorTest.java
deleted file mode 100644
index fe41f97686..0000000000
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-k8s/src/test/java/org/apache/dolphinscheduler/plugin/datasource/k8s/K8sDataSourceProcessorTest.java
+++ /dev/null
@@ -1,107 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.dolphinscheduler.plugin.datasource.k8s;
-
-import org.apache.dolphinscheduler.plugin.datasource.k8s.param.K8sConnectionParam;
-import org.apache.dolphinscheduler.plugin.datasource.k8s.param.K8sDataSourceParamDTO;
-import org.apache.dolphinscheduler.plugin.datasource.k8s.param.K8sDataSourceProcessor;
-import org.apache.dolphinscheduler.spi.enums.DbType;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.extension.ExtendWith;
-import org.mockito.MockedConstruction;
-import org.mockito.Mockito;
-import org.mockito.junit.jupiter.MockitoExtension;
-
-@ExtendWith(MockitoExtension.class)
-public class K8sDataSourceProcessorTest {
-
- private K8sDataSourceProcessor k8sDataSourceProcessor;
-
- private String connectJson =
- "{\"namespace\":\"namespace\",\"kubeConfig\":\"kubeConfig\"}";
-
- @BeforeEach
- public void init() {
- k8sDataSourceProcessor = new K8sDataSourceProcessor();
- }
-
- @Test
- void testCheckDatasourceParam() {
- K8sDataSourceParamDTO k8sDataSourceParamDTO = new K8sDataSourceParamDTO();
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> k8sDataSourceProcessor.checkDatasourceParam(k8sDataSourceParamDTO));
- k8sDataSourceParamDTO.setNamespace("namespace");
- Assertions.assertThrows(IllegalArgumentException.class,
- () -> k8sDataSourceProcessor.checkDatasourceParam(k8sDataSourceParamDTO));
- k8sDataSourceParamDTO.setKubeConfig("kubeConfig");
- Assertions
- .assertDoesNotThrow(
- () -> k8sDataSourceProcessor.checkDatasourceParam(k8sDataSourceParamDTO));
- }
-
- @Test
- void testGetDatasourceUniqueId() {
- K8sConnectionParam k8sConnectionParam = new K8sConnectionParam();
- k8sConnectionParam.setNamespace("namespace");
- k8sConnectionParam.setKubeConfig("kubeConfig");
- Assertions.assertEquals("k8s@kubeConfig@namespace",
- k8sDataSourceProcessor.getDatasourceUniqueId(k8sConnectionParam, DbType.K8S));
-
- }
-
- @Test
- void testCreateDatasourceParamDTO() {
- K8sDataSourceParamDTO k8sDataSourceParamDTO =
- (K8sDataSourceParamDTO) k8sDataSourceProcessor.createDatasourceParamDTO(connectJson);
- Assertions.assertEquals("namespace", k8sDataSourceParamDTO.getNamespace());
- Assertions.assertEquals("kubeConfig", k8sDataSourceParamDTO.getKubeConfig());
- }
-
- @Test
- void testCreateConnectionParams() {
- K8sDataSourceParamDTO k8sDataSourceParamDTO =
- (K8sDataSourceParamDTO) k8sDataSourceProcessor.createDatasourceParamDTO(connectJson);
- K8sConnectionParam k8sConnectionParam =
- k8sDataSourceProcessor.createConnectionParams(k8sDataSourceParamDTO);
- Assertions.assertEquals("namespace", k8sConnectionParam.getNamespace());
- Assertions.assertEquals("kubeConfig", k8sConnectionParam.getKubeConfig());
- }
-
- @Test
- void testTestConnection() {
- K8sDataSourceParamDTO k8sDataSourceParamDTO =
- (K8sDataSourceParamDTO) k8sDataSourceProcessor.createDatasourceParamDTO(connectJson);
- K8sConnectionParam connectionParam =
- k8sDataSourceProcessor.createConnectionParams(k8sDataSourceParamDTO);
- Assertions.assertFalse(k8sDataSourceProcessor.checkDataSourceConnectivity(connectionParam));
-
- try (
- MockedConstruction<K8sClientWrapper> k8sClientWrapperMockedConstruction =
- Mockito.mockConstruction(K8sClientWrapper.class, (mock, context) -> {
- Mockito.when(
- mock.checkConnect(connectionParam.getKubeConfig(), connectionParam.getNamespace()))
- .thenReturn(true);
- })) {
- Assertions.assertTrue(k8sDataSourceProcessor.checkDataSourceConnectivity(connectionParam));
- }
-
- }
-}
diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceProcessorTest.java
index 395e7e5d3f..424f478699 100644
--- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceProcessorTest.java
+++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-sagemaker/src/test/java/org/apache/dolphinscheduler/plugin/datasource/sagemaker/SagemakerDataSourceProcessorTest.java
@@ -31,6 +31,7 @@ import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
@ExtendWith(MockitoExtension.class)
+
public class SagemakerDataSourceProcessorTest {
private SagemakerDataSourceProcessor sagemakerDataSourceProcessor;
diff --git a/dolphinscheduler-datasource-plugin/pom.xml b/dolphinscheduler-datasource-plugin/pom.xml
index c30a6b4258..a71b6ef683 100644
--- a/dolphinscheduler-datasource-plugin/pom.xml
+++ b/dolphinscheduler-datasource-plugin/pom.xml
@@ -54,7 +54,6 @@
<module>dolphinscheduler-datasource-vertica</module>
<module>dolphinscheduler-datasource-doris</module>
<module>dolphinscheduler-datasource-sagemaker</module>
- <module>dolphinscheduler-datasource-k8s</module>
<module>dolphinscheduler-datasource-hana</module>
</modules>
diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/datasource/DataSourcePage.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/datasource/DataSourcePage.java
index bd2f7e795b..3b8633443d 100644
--- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/datasource/DataSourcePage.java
+++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/pages/datasource/DataSourcePage.java
@@ -193,11 +193,5 @@ public class DataSourcePage extends NavBarPage implements NavBarPage.NavBarItem
})
private WebElement inputZeppelinRestEndpoint;
- @FindBys({
- @FindBy(className = "input-kubeConfig"),
- @FindBy(tagName = "textarea"),
- })
- private WebElement inputKubeConfig;
-
}
}
diff --git a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java
index e7ebbeee0a..2dd8958924 100644
--- a/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java
+++ b/dolphinscheduler-spi/src/main/java/org/apache/dolphinscheduler/spi/enums/DbType.java
@@ -53,9 +53,7 @@ public enum DbType {
HANA(22, "hana"),
DORIS(23, "doris"),
ZEPPELIN(24, "zeppelin"),
- SAGEMAKER(25, "sagemaker"),
-
- K8S(26, "k8s");
+ SAGEMAKER(25, "sagemaker");
private static final Map<Integer, DbType> DB_TYPE_MAP =
Arrays.stream(DbType.values()).collect(toMap(DbType::getCode, Functions.identity()));
@EnumValue
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/K8sTaskExecutionContext.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/K8sTaskExecutionContext.java
index 4138e5465f..aa5bf62fe2 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/K8sTaskExecutionContext.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/K8sTaskExecutionContext.java
@@ -19,7 +19,7 @@ package org.apache.dolphinscheduler.plugin.task.api;
import java.io.Serializable;
-import lombok.Data;
+import lombok.Value;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonProperty;
@@ -27,18 +27,13 @@ import com.fasterxml.jackson.annotation.JsonProperty;
/**
* k8s Task ExecutionContext
*/
-@Data
+@Value
public class K8sTaskExecutionContext implements Serializable {
private String configYaml;
private String namespace;
- private String connectionParams;
-
- public K8sTaskExecutionContext() {
- }
-
@JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
public K8sTaskExecutionContext(
@JsonProperty("configYaml") String configYaml,
@@ -52,7 +47,6 @@ public class K8sTaskExecutionContext implements Serializable {
return "K8sTaskExecutionContext{"
+ "namespace=" + namespace
+ ", configYaml='" + configYaml + '\''
- + ", connectionParams='" + connectionParams + '\''
+ '}';
}
}
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/KubernetesApplicationManager.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/KubernetesApplicationManager.java
index a18637a4ff..ac1ce69f76 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/KubernetesApplicationManager.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/KubernetesApplicationManager.java
@@ -22,7 +22,6 @@ import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.UNIQUE_L
import org.apache.dolphinscheduler.common.enums.ResourceManagerType;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
-import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.K8sTaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.TaskException;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
@@ -132,8 +131,6 @@ public class KubernetesApplicationManager implements ApplicationManager {
private KubernetesClient getClient(KubernetesApplicationManagerContext kubernetesApplicationManagerContext) {
K8sTaskExecutionContext k8sTaskExecutionContext =
kubernetesApplicationManagerContext.getK8sTaskExecutionContext();
- k8sTaskExecutionContext
- .setConfigYaml(JSONUtils.getNodeString(k8sTaskExecutionContext.getConnectionParams(), "kubeConfig"));
return cacheClientMap.computeIfAbsent(kubernetesApplicationManagerContext.getLabelValue(),
key -> new KubernetesClientBuilder()
.withConfig(Config.fromKubeconfig(k8sTaskExecutionContext.getConfigYaml())).build());
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java
index 8adc912161..a6c4703f10 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskMainParameters.java
@@ -34,6 +34,7 @@ public class K8sTaskMainParameters {
private String args;
private String pullSecret;
private String namespaceName;
+ private String clusterName;
private String imagePullPolicy;
private double minCpuCores;
private double minMemorySpace;
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java
index 4d41a85bbb..986c9dc8a7 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java
@@ -300,9 +300,7 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor {
return result;
}
K8sTaskExecutionContext k8sTaskExecutionContext = taskRequest.getK8sTaskExecutionContext();
- String connectionParams = k8sTaskExecutionContext.getConnectionParams();
- String kubeConfig = JSONUtils.getNodeString(connectionParams, "kubeConfig");
- String configYaml = kubeConfig;
+ String configYaml = k8sTaskExecutionContext.getConfigYaml();
k8sUtils.buildClient(configYaml);
submitJob2k8s(k8sParameterStr);
parsePodLogOutput();
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java
index d3d5e4963b..ba9f6498af 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java
@@ -17,28 +17,21 @@
package org.apache.dolphinscheduler.plugin.task.api.parameters;
-import org.apache.dolphinscheduler.plugin.task.api.K8sTaskExecutionContext;
-import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType;
import org.apache.dolphinscheduler.plugin.task.api.model.Label;
import org.apache.dolphinscheduler.plugin.task.api.model.NodeSelectorExpression;
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.List;
-import java.util.Objects;
import lombok.Data;
-import lombok.extern.slf4j.Slf4j;
/**
* k8s task parameters
*/
@Data
-@Slf4j
public class K8sTaskParameters extends AbstractParameters {
private String image;
@@ -51,29 +44,14 @@ public class K8sTaskParameters extends AbstractParameters {
private double minMemorySpace;
private List<Label> customizedLabels;
private List<NodeSelectorExpression> nodeSelectors;
- private String kubeConfig;
- private int datasource;
- private String type;
+
@Override
public boolean checkParameters() {
- return StringUtils.isNotEmpty(image);
- }
- public K8sTaskExecutionContext generateExtendedContext(ResourceParametersHelper parametersHelper) {
- DataSourceParameters dataSourceParameters =
- (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, datasource);
- K8sTaskExecutionContext k8sTaskExecutionContext = new K8sTaskExecutionContext();
- k8sTaskExecutionContext.setConnectionParams(
- Objects.nonNull(dataSourceParameters) ? dataSourceParameters.getConnectionParams() : null);
- return k8sTaskExecutionContext;
+ return StringUtils.isNotEmpty(image) && StringUtils.isNotEmpty(namespace);
}
+
@Override
public List<ResourceInfo> getResourceFilesList() {
return new ArrayList<>();
}
- @Override
- public ResourceParametersHelper getResources() {
- ResourceParametersHelper resources = super.getResources();
- resources.put(ResourceType.DATASOURCE, datasource);
- return resources;
- }
}
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java
index 1e7629acce..d6b4abff03 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/test/java/org/apache/dolphinscheduler/plugin/task/api/k8s/K8sTaskExecutorTest.java
@@ -17,6 +17,10 @@
package org.apache.dolphinscheduler.plugin.task.api.k8s;
+import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.CLUSTER;
+import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.NAMESPACE_NAME;
+
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.TaskException;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.k8s.impl.K8sTaskExecutor;
@@ -44,7 +48,7 @@ public class K8sTaskExecutorTest {
private K8sTaskMainParameters k8sTaskMainParameters = null;
private final String image = "ds-dev";
private final String imagePullPolicy = "IfNotPresent";
- private final String namespace = "namespace";
+ private final String namespace = "{\"name\":\"default\",\"cluster\":\"lab\"}";
private final double minCpuCores = 2;
private final double minMemorySpace = 10;
private final int taskInstanceId = 1000;
@@ -55,6 +59,9 @@ public class K8sTaskExecutorTest {
TaskExecutionContext taskRequest = new TaskExecutionContext();
taskRequest.setTaskInstanceId(taskInstanceId);
taskRequest.setTaskName(taskName);
+ Map<String, String> namespace = JSONUtils.toMap(this.namespace);
+ String namespaceName = namespace.get(NAMESPACE_NAME);
+ String clusterName = namespace.get(CLUSTER);
Map<String, String> labelMap = new HashMap<>();
labelMap.put("test", "1234");
@@ -66,7 +73,8 @@ public class K8sTaskExecutorTest {
k8sTaskMainParameters = new K8sTaskMainParameters();
k8sTaskMainParameters.setImage(image);
k8sTaskMainParameters.setImagePullPolicy(imagePullPolicy);
- k8sTaskMainParameters.setNamespaceName(namespace);
+ k8sTaskMainParameters.setNamespaceName(namespaceName);
+ k8sTaskMainParameters.setClusterName(clusterName);
k8sTaskMainParameters.setMinCpuCores(minCpuCores);
k8sTaskMainParameters.setMinMemorySpace(minMemorySpace);
k8sTaskMainParameters.setCommand("[\"perl\" ,\"-Mbignum=bpi\", \"-wle\", \"print bpi(2000)\"]");
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java
index fceb29e163..269b2fd2a4 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java
@@ -17,10 +17,10 @@
package org.apache.dolphinscheduler.plugin.task.k8s;
+import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.CLUSTER;
+import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.NAMESPACE_NAME;
+
import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils;
-import org.apache.dolphinscheduler.plugin.datasource.k8s.param.K8sConnectionParam;
-import org.apache.dolphinscheduler.plugin.task.api.K8sTaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.TaskException;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.k8s.AbstractK8sTask;
@@ -31,7 +31,6 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.AbstractParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.K8sTaskParameters;
import org.apache.dolphinscheduler.plugin.task.api.utils.ParameterUtils;
-import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
@@ -53,32 +52,17 @@ public class K8sTask extends AbstractK8sTask {
private K8sTaskParameters k8sTaskParameters;
- private K8sTaskExecutionContext k8sTaskExecutionContext;
-
- private K8sConnectionParam k8sConnectionParam;
+ /**
+ * @param taskRequest taskRequest
+ */
public K8sTask(TaskExecutionContext taskRequest) {
super(taskRequest);
this.taskExecutionContext = taskRequest;
- }
-
- @Override
- public void init() {
- String taskParams = taskExecutionContext.getTaskParams();
- k8sTaskParameters = JSONUtils.parseObject(taskParams, K8sTaskParameters.class);
+ this.k8sTaskParameters = JSONUtils.parseObject(taskExecutionContext.getTaskParams(), K8sTaskParameters.class);
+ log.info("Initialize k8s task parameters {}", JSONUtils.toPrettyJsonString(k8sTaskParameters));
if (k8sTaskParameters == null || !k8sTaskParameters.checkParameters()) {
throw new TaskException("K8S task params is not valid");
}
-
- k8sTaskExecutionContext =
- k8sTaskParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper());
- taskRequest.setK8sTaskExecutionContext(k8sTaskExecutionContext);
- k8sConnectionParam =
- (K8sConnectionParam) DataSourceUtils.buildConnectionParams(DbType.valueOf(k8sTaskParameters.getType()),
- k8sTaskExecutionContext.getConnectionParams());
- String kubeConfig = k8sConnectionParam.getKubeConfig();
- k8sTaskParameters.setNamespace(k8sConnectionParam.getNamespace());
- k8sTaskParameters.setKubeConfig(kubeConfig);
- log.info("Initialize k8s task params:{}", JSONUtils.toPrettyJsonString(k8sTaskParameters));
}
@Override
@@ -95,10 +79,13 @@ public class K8sTask extends AbstractK8sTask {
protected String buildCommand() {
K8sTaskMainParameters k8sTaskMainParameters = new K8sTaskMainParameters();
Map<String, Property> paramsMap = taskExecutionContext.getPrepareParamsMap();
- String namespaceName = k8sTaskParameters.getNamespace();
+ Map<String, String> namespace = JSONUtils.toMap(k8sTaskParameters.getNamespace());
+ String namespaceName = namespace.get(NAMESPACE_NAME);
+ String clusterName = namespace.get(CLUSTER);
k8sTaskMainParameters.setImage(k8sTaskParameters.getImage());
k8sTaskMainParameters.setPullSecret(k8sTaskParameters.getPullSecret());
k8sTaskMainParameters.setNamespaceName(namespaceName);
+ k8sTaskMainParameters.setClusterName(clusterName);
k8sTaskMainParameters.setMinCpuCores(k8sTaskParameters.getMinCpuCores());
k8sTaskMainParameters.setMinMemorySpace(k8sTaskParameters.getMinMemorySpace());
k8sTaskMainParameters.setParamsMap(ParameterUtils.convert(paramsMap));
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java
index 91c0faf684..f15ab4496e 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskChannel.java
@@ -40,7 +40,7 @@ public class K8sTaskChannel implements TaskChannel {
@Override
public ResourceParametersHelper getResources(String parameters) {
- return JSONUtils.parseObject(parameters, K8sTaskParameters.class).getResources();
+ return null;
}
@Override
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sParametersTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sParametersTest.java
index 543ec3e4bc..eb65ede9d0 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sParametersTest.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sParametersTest.java
@@ -33,7 +33,7 @@ public class K8sParametersTest {
private K8sTaskParameters k8sTaskParameters = null;
private final String image = "ds-dev";
private final String imagePullPolicy = "IfNotPresent";
- private final String namespace = "namespace";
+ private final String namespace = "{\"name\":\"default\",\"cluster\":\"lab\"}";
private final double minCpuCores = 2;
private final double minMemorySpace = 10;
private final String command = "[\"/bin/bash\", \"-c\"]";
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskTest.java
index 3895190cf2..e1066e9d24 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskTest.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/test/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTaskTest.java
@@ -17,13 +17,7 @@
package org.apache.dolphinscheduler.plugin.task.k8s;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-
import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.plugin.datasource.api.utils.DataSourceUtils;
-import org.apache.dolphinscheduler.plugin.datasource.k8s.param.K8sConnectionParam;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.DataType;
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
@@ -31,7 +25,6 @@ import org.apache.dolphinscheduler.plugin.task.api.model.Label;
import org.apache.dolphinscheduler.plugin.task.api.model.NodeSelectorExpression;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.parameters.K8sTaskParameters;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
import java.util.ArrayList;
import java.util.Arrays;
@@ -39,12 +32,9 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import org.mockito.MockedStatic;
-import org.mockito.Mockito;
import com.google.common.collect.ImmutableMap;
import io.fabric8.kubernetes.api.model.NodeSelectorRequirement;
@@ -59,7 +49,7 @@ public class K8sTaskTest {
private final String pullSecret = "ds-secret";
- private final String namespace = "namespace";
+ private final String namespace = "{\"name\":\"default\",\"cluster\":\"lab\"}";
private final double minCpuCores = 2;
@@ -71,53 +61,13 @@ public class K8sTaskTest {
private final String date = "20220507";
private final String command = "[\"/bin/bash\", \"-c\"]";
private final String args = "[\"echo hello world\"]";
- private final String kubeConfig = "{}";
- private final String type = "K8S";
-
- private final Map<String, Property> prepareParamsMap = new HashMap<String, Property>() {
-
- {
- put(DAY, new Property() {
-
- {
- setProp(DAY);
- setValue(date);
- }
- });
- }
- };
-
- private final int datasource = 0;
private final List<Label> labels = Arrays.asList(new Label("test", "1234"));
private final List<NodeSelectorExpression> nodeSelectorExpressions =
Arrays.asList(new NodeSelectorExpression("node-label", "In", "1234,12345"));
- private static MockedStatic<DataSourceUtils> dataSourceUtilsStaticMock = null;
-
@BeforeEach
public void before() {
- String k8sTaskParameters = buildK8sTaskParameters();
- TaskExecutionContext taskExecutionContext = mock(TaskExecutionContext.class);
- ResourceParametersHelper resourceParametersHelper = mock(ResourceParametersHelper.class);
- K8sConnectionParam k8sConnectionParam = mock(K8sConnectionParam.class);
- when(taskExecutionContext.getTaskParams()).thenReturn(k8sTaskParameters);
- when(k8sConnectionParam.getNamespace()).thenReturn(namespace);
- when(k8sConnectionParam.getKubeConfig()).thenReturn(kubeConfig);
- when(taskExecutionContext.getPrepareParamsMap()).thenReturn(prepareParamsMap);
- when(taskExecutionContext.getResourceParametersHelper()).thenReturn(resourceParametersHelper);
- dataSourceUtilsStaticMock = Mockito.mockStatic(DataSourceUtils.class);
- dataSourceUtilsStaticMock.when(() -> DataSourceUtils.buildConnectionParams(Mockito.any(), Mockito.any()))
- .thenReturn(k8sConnectionParam);
- k8sTask = spy(new K8sTask(taskExecutionContext));
- k8sTask.init();
- }
-
- @AfterEach
- public void afterEach() {
- dataSourceUtilsStaticMock.close();
- }
- private String buildK8sTaskParameters() {
- K8sTaskParameters k8sTaskParameters = new K8sTaskParameters();
+ k8sTaskParameters = new K8sTaskParameters();
k8sTaskParameters.setImage(image);
k8sTaskParameters.setImagePullPolicy(imagePullPolicy);
k8sTaskParameters.setNamespace(namespace);
@@ -129,15 +79,32 @@ public class K8sTaskTest {
k8sTaskParameters.setNodeSelectors(nodeSelectorExpressions);
k8sTaskParameters.setLocalParams(new ArrayList<>());
k8sTaskParameters.setPullSecret(pullSecret);
- k8sTaskParameters.setType(type);
- k8sTaskParameters.setKubeConfig(kubeConfig);
- k8sTaskParameters.setDatasource(datasource);
- return JSONUtils.toJsonString(k8sTaskParameters);
+ TaskExecutionContext taskRequest = new TaskExecutionContext();
+ taskRequest.setTaskInstanceId(taskInstanceId);
+ taskRequest.setTaskName(taskName);
+ taskRequest.setTaskParams(JSONUtils.toJsonString(k8sTaskParameters));
+ Property property = new Property();
+ property.setProp(DAY);
+ property.setDirect(Direct.IN);
+ property.setType(DataType.VARCHAR);
+ property.setValue(date);
+ Map<String, Property> paramsMap = new HashMap<>();
+ paramsMap.put(DAY, property);
+ taskRequest.setParamsMap(paramsMap);
+
+ Map<String, Property> prepareParamsMap = new HashMap<>();
+ Property property1 = new Property();
+ property1.setProp("day");
+ property1.setValue("20220507");
+ prepareParamsMap.put("day", property1);
+ taskRequest.setPrepareParamsMap(prepareParamsMap);
+ k8sTask = new K8sTask(taskRequest);
}
+
@Test
public void testBuildCommandNormal() {
String expectedStr =
- "{\"image\":\"ds-dev\",\"command\":\"[\\\"/bin/bash\\\", \\\"-c\\\"]\",\"args\":\"[\\\"echo hello world\\\"]\",\"pullSecret\":\"ds-secret\",\"namespaceName\":\"namespace\",\"imagePullPolicy\":\"IfNotPresent\",\"minCpuCores\":2.0,\"minMemorySpace\":10.0,\"paramsMap\":{\"day\":\"20220507\"},\"labelMap\":{\"test\":\"1234\"},\"nodeSelectorRequirements\":[{\"key\":\"node-label\",\"operator\":\"In\",\"values\":[\"1234\",\"12345\"]}]}";
+ "{\"image\":\"ds-dev\",\"command\":\"[\\\"/bin/bash\\\", \\\"-c\\\"]\",\"args\":\"[\\\"echo hello world\\\"]\",\"pullSecret\":\"ds-secret\",\"namespaceName\":\"default\",\"clusterName\":\"lab\",\"imagePullPolicy\":\"IfNotPresent\",\"minCpuCores\":2.0,\"minMemorySpace\":10.0,\"paramsMap\":{\"day\":\"20220507\"},\"labelMap\":{\"test\":\"1234\"},\"nodeSelectorRequirements\":[{\"key\":\"node-label\",\"operator\":\"In\",\"values\":[\"1234\",\"12345\"]}]}";
String commandStr = k8sTask.buildCommand();
Assertions.assertEquals(expectedStr, commandStr);
}
@@ -145,7 +112,7 @@ public class K8sTaskTest {
@Test
public void testGetParametersNormal() {
String expectedStr =
- "K8sTaskParameters(image=ds-dev, namespace=namespace, command=[\"/bin/bash\", \"-c\"], args=[\"echo hello world\"], pullSecret=ds-secret, imagePullPolicy=IfNotPresent, minCpuCores=2.0, minMemorySpace=10.0, customizedLabels=[Label(label=test, value=1234)], nodeSelectors=[NodeSelectorExpression(key=node-label, operator=In, values=1234,12345)], kubeConfig={}, datasource=0, type=K8S)";
+ "K8sTaskParameters(image=ds-dev, namespace={\"name\":\"default\",\"cluster\":\"lab\"}, command=[\"/bin/bash\", \"-c\"], args=[\"echo hello world\"], pullSecret=ds-secret, imagePullPolicy=IfNotPresent, minCpuCores=2.0, minMemorySpace=10.0, customizedLabels=[Label(label=test, value=1234)], nodeSelectors=[NodeSelectorExpression(key=node-label, operator=In, values=1234,12345)])";
String result = k8sTask.getParameters().toString();
Assertions.assertEquals(expectedStr, result);
}
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/test/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/test/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskTest.java
index 0c8dd6cf6b..bbe3136dbf 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/test/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskTest.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-sagemaker/src/test/java/org/apache/dolphinscheduler/plugin/task/sagemaker/SagemakerTaskTest.java
@@ -66,6 +66,7 @@ public class SagemakerTaskTest {
@BeforeEach
public void before() {
String parameters = buildParameters();
+ System.out.println(parameters);
TaskExecutionContext taskExecutionContext = Mockito.mock(TaskExecutionContext.class);
ResourceParametersHelper resourceParametersHelper = Mockito.mock(ResourceParametersHelper.class);
SagemakerConnectionParam sagemakerConnectionParam = Mockito.mock(SagemakerConnectionParam.class);
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTask.java
index c3b580de35..b7552cc553 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTask.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-zeppelin/src/main/java/org/apache/dolphinscheduler/plugin/task/zeppelin/ZeppelinTask.java
@@ -82,7 +82,7 @@ public class ZeppelinTask extends AbstractRemoteTask {
zeppelinParameters.setUsername(zeppelinConnectionParam.getUsername());
zeppelinParameters.setPassword(zeppelinConnectionParam.getPassword());
zeppelinParameters.setRestEndpoint(zeppelinConnectionParam.getRestEndpoint());
- log.info("Initialize zeppelin task params:{}", JSONUtils.toPrettyJsonString(zeppelinParameters));
+ log.info("Initialize zeppelin task params:{}", JSONUtils.toPrettyJsonString(taskParams));
this.zClient = getZeppelinClient();
}
diff --git a/dolphinscheduler-ui/src/service/modules/data-source/types.ts b/dolphinscheduler-ui/src/service/modules/data-source/types.ts
index 444f5293dd..a5fbb1a8a5 100644
--- a/dolphinscheduler-ui/src/service/modules/data-source/types.ts
+++ b/dolphinscheduler-ui/src/service/modules/data-source/types.ts
@@ -41,7 +41,6 @@ type IDataBase =
| 'KYUUBI'
| 'ZEPPELIN'
| 'SAGEMAKER'
- | 'K8S'
type IDataBaseLabel =
| 'MYSQL'
@@ -64,7 +63,6 @@ type IDataBaseLabel =
| 'KYUUBI'
| 'ZEPPELIN'
| 'SAGEMAKER'
- | 'K8S'
interface IDataSource {
id?: number
@@ -87,8 +85,6 @@ interface IDataSource {
other?: object
endpoint?: string
restEndpoint?: string
- kubeConfig?: string
- namespace?: string
MSIClientId?: string
dbUser?: string
compatibleMode?: string
diff --git a/dolphinscheduler-ui/src/views/datasource/list/detail.tsx b/dolphinscheduler-ui/src/views/datasource/list/detail.tsx
index 4842651290..3a7c1e6074 100644
--- a/dolphinscheduler-ui/src/views/datasource/list/detail.tsx
+++ b/dolphinscheduler-ui/src/views/datasource/list/detail.tsx
@@ -163,8 +163,6 @@ const DetailModal = defineComponent({
showDataBaseName,
showJDBCConnectParameters,
showPublicKey,
- showNamespace,
- showKubeConfig,
modeOptions,
redShiftModeOptions,
sagemakerModeOption,
@@ -544,10 +542,7 @@ const DetailModal = defineComponent({
/>
</NFormItem>
<NFormItem
- v-show={
- (!showMode || detailForm.mode === 'password') &&
- detailForm.type != 'K8S'
- }
+ v-show={!showMode || detailForm.mode === 'password'}
label={t('datasource.user_name')}
path='userName'
show-require-mark
@@ -562,10 +557,7 @@ const DetailModal = defineComponent({
/>
</NFormItem>
<NFormItem
- v-show={
- (!showMode || detailForm.mode === 'password') &&
- detailForm.type != 'K8S'
- }
+ v-show={!showMode || detailForm.mode === 'password'}
label={t('datasource.user_password')}
path='password'
>
@@ -686,35 +678,6 @@ const DetailModal = defineComponent({
}}
/>
</NFormItem>
- <NFormItem
- v-show={showKubeConfig}
- label={t('datasource.kubeConfig')}
- path='kubeConfig'
- show-require-mark
- >
- <NInput
- allowInput={this.trim}
- class='input-kubeConfig'
- v-model={[detailForm.kubeConfig, 'value']}
- type='textarea'
- autosize={{
- minRows: 14
- }}
- placeholder={t('datasource.kubeConfig_tips')}
- />
- </NFormItem>
- <NFormItem
- v-show={showNamespace}
- label={t('datasource.namespace')}
- path='namespace'
- show-require-mark
- >
- <NInput
- allowInput={this.trim}
- v-model={[detailForm.namespace, 'value']}
- placeholder={t('datasource.namespace_tips')}
- />
- </NFormItem>
</NForm>
</NSpin>
),
diff --git a/dolphinscheduler-ui/src/views/datasource/list/use-form.ts b/dolphinscheduler-ui/src/views/datasource/list/use-form.ts
index 21916667d2..ed383c3720 100644
--- a/dolphinscheduler-ui/src/views/datasource/list/use-form.ts
+++ b/dolphinscheduler-ui/src/views/datasource/list/use-form.ts
@@ -69,8 +69,6 @@ export function useForm(id?: number) {
showDataBaseName: true,
showJDBCConnectParameters: true,
showPublicKey: false,
- showNamespace: false,
- showKubeConfig: false,
rules: {
name: {
trigger: ['input'],
@@ -120,8 +118,7 @@ export function useForm(id?: number) {
validator() {
if (
!state.detailForm.userName &&
- state.detailForm.type !== 'AZURESQL' &&
- state.detailForm.type !== 'K8S'
+ state.detailForm.type !== 'AZURESQL'
) {
return new Error(t('datasource.user_name_tips'))
}
@@ -264,12 +261,7 @@ export function useForm(id?: number) {
} else {
state.showPrincipal = false
}
- if (
- type === 'SSH' ||
- type === 'ZEPPELIN' ||
- type === 'SAGEMAKER' ||
- type === 'K8S'
- ) {
+ if (type === 'SSH' || type === 'ZEPPELIN' || type === 'SAGEMAKER') {
state.showDataBaseName = false
state.requiredDataBase = false
state.showJDBCConnectParameters = false
@@ -282,14 +274,10 @@ export function useForm(id?: number) {
state.showPort = false
state.showRestEndpoint = true
}
- if (type === 'SAGEMAKER' || type === 'K8S') {
+ if (type === 'SAGEMAKER') {
state.showHost = false
state.showPort = false
}
- if (type === 'K8S') {
- state.showNamespace = true
- state.showKubeConfig = true
- }
} else {
state.showDataBaseName = true
state.requiredDataBase = true
@@ -453,11 +441,6 @@ export const datasourceType: IDataBaseOptionKeys = {
value: 'SAGEMAKER',
label: 'SAGEMAKER',
defaultPort: 0
- },
- K8S: {
- value: 'K8S',
- label: 'K8S',
- defaultPort: 6443
}
}
diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-k8s.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-k8s.ts
index ed9d83a6ac..b545d15463 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-k8s.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-k8s.ts
@@ -14,7 +14,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import { useCustomParams, useCustomLabels, useNodeSelectors } from '.'
+import {
+ useCustomParams,
+ useNamespace,
+ useCustomLabels,
+ useNodeSelectors
+} from '.'
import type { IJsonItem } from '../types'
import { useI18n } from 'vue-i18n'
@@ -22,6 +27,7 @@ export function useK8s(model: { [field: string]: any }): IJsonItem[] {
const { t } = useI18n()
return [
+ useNamespace(),
{
type: 'input-number',
field: 'minCpuCores',
diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
index 1c9e072fbf..f2235951ec 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/format-data.ts
@@ -351,9 +351,6 @@ export function formatParams(data: INodeData): {
taskParams.args = data.args
taskParams.customizedLabels = data.customizedLabels
taskParams.nodeSelectors = data.nodeSelectors
- taskParams.datasource = data.datasource
- taskParams.type = data.type
- taskParams.kubeConfig = data.kubeConfig
taskParams.pullSecret = data.pullSecret
}
diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-k8s.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-k8s.ts
index c72443c218..8cce04b568 100644
--- a/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-k8s.ts
+++ b/dolphinscheduler-ui/src/views/projects/task/components/node/tasks/use-k8s.ts
@@ -46,11 +46,7 @@ export function useK8s({
workerGroup: 'default',
delayTime: 0,
timeout: 30,
- type: 'K8S',
- displayRows: 10,
- timeoutNotifyStrategy: ['WARN'],
- kubeConfig: '',
- namespace: ''
+ timeoutNotifyStrategy: ['WARN']
} as INodeData)
return {
@@ -67,7 +63,6 @@ export function useK8s({
...Fields.useFailed(),
Fields.useDelayTime(model),
...Fields.useTimeoutAlarm(model),
- ...Fields.useDatasource(model),
...Fields.useK8s(model),
Fields.usePreTasks()
] as IJsonItem[],
(dolphinscheduler) 01/02: Revert "Fix k8sTaskExecutionContext setting configYaml (#15116)"
Posted by zh...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.2.1-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit 57a4325821a439bb40071fccc9a05a49f7cb1bf4
Author: Jay Chung <zh...@gmail.com>
AuthorDate: Tue Feb 6 18:32:26 2024 +0800
Revert "Fix k8sTaskExecutionContext setting configYaml (#15116)"
This reverts commit ce11674668cf26747f34c3474763c32e1ba5b6d1.
---
.../plugin/task/api/am/KubernetesApplicationManager.java | 3 +++
.../plugin/task/api/k8s/impl/K8sTaskExecutor.java | 4 +++-
.../plugin/task/api/parameters/AbstractParameters.java | 13 -------------
.../plugin/task/api/parameters/K8sTaskParameters.java | 13 +++++++++++--
.../apache/dolphinscheduler/plugin/task/k8s/K8sTask.java | 6 ++----
5 files changed, 19 insertions(+), 20 deletions(-)
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/KubernetesApplicationManager.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/KubernetesApplicationManager.java
index ac1ce69f76..a18637a4ff 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/KubernetesApplicationManager.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/am/KubernetesApplicationManager.java
@@ -22,6 +22,7 @@ import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.UNIQUE_L
import org.apache.dolphinscheduler.common.enums.ResourceManagerType;
import org.apache.dolphinscheduler.common.thread.ThreadUtils;
+import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.plugin.task.api.K8sTaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.TaskException;
import org.apache.dolphinscheduler.plugin.task.api.enums.TaskExecutionStatus;
@@ -131,6 +132,8 @@ public class KubernetesApplicationManager implements ApplicationManager {
private KubernetesClient getClient(KubernetesApplicationManagerContext kubernetesApplicationManagerContext) {
K8sTaskExecutionContext k8sTaskExecutionContext =
kubernetesApplicationManagerContext.getK8sTaskExecutionContext();
+ k8sTaskExecutionContext
+ .setConfigYaml(JSONUtils.getNodeString(k8sTaskExecutionContext.getConnectionParams(), "kubeConfig"));
return cacheClientMap.computeIfAbsent(kubernetesApplicationManagerContext.getLabelValue(),
key -> new KubernetesClientBuilder()
.withConfig(Config.fromKubeconfig(k8sTaskExecutionContext.getConfigYaml())).build());
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java
index 986c9dc8a7..4d41a85bbb 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/k8s/impl/K8sTaskExecutor.java
@@ -300,7 +300,9 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor {
return result;
}
K8sTaskExecutionContext k8sTaskExecutionContext = taskRequest.getK8sTaskExecutionContext();
- String configYaml = k8sTaskExecutionContext.getConfigYaml();
+ String connectionParams = k8sTaskExecutionContext.getConnectionParams();
+ String kubeConfig = JSONUtils.getNodeString(connectionParams, "kubeConfig");
+ String configYaml = kubeConfig;
k8sUtils.buildClient(configYaml);
submitJob2k8s(k8sParameterStr);
parsePodLogOutput();
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java
index 6ca1be7d7a..a57eececf5 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/AbstractParameters.java
@@ -18,12 +18,9 @@
package org.apache.dolphinscheduler.plugin.task.api.parameters;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
-import org.apache.dolphinscheduler.plugin.task.api.K8sTaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.Direct;
-import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType;
import org.apache.dolphinscheduler.plugin.task.api.model.Property;
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
-import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
import org.apache.commons.collections4.CollectionUtils;
@@ -89,16 +86,6 @@ public abstract class AbstractParameters implements IParameters {
return localParametersMaps;
}
- public K8sTaskExecutionContext generateK8sTaskExecutionContext(ResourceParametersHelper parametersHelper,
- int datasource) {
- DataSourceParameters dataSourceParameters =
- (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, datasource);
- K8sTaskExecutionContext k8sTaskExecutionContext = new K8sTaskExecutionContext();
- k8sTaskExecutionContext.setConnectionParams(
- Objects.nonNull(dataSourceParameters) ? dataSourceParameters.getConnectionParams() : null);
- return k8sTaskExecutionContext;
- }
-
/**
* get input local parameters map if the param direct is IN
* @return parameters map
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java
index 4f045abe19..d3d5e4963b 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-api/src/main/java/org/apache/dolphinscheduler/plugin/task/api/parameters/K8sTaskParameters.java
@@ -17,16 +17,19 @@
package org.apache.dolphinscheduler.plugin.task.api.parameters;
+import org.apache.dolphinscheduler.plugin.task.api.K8sTaskExecutionContext;
import org.apache.dolphinscheduler.plugin.task.api.enums.ResourceType;
import org.apache.dolphinscheduler.plugin.task.api.model.Label;
import org.apache.dolphinscheduler.plugin.task.api.model.NodeSelectorExpression;
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
+import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.DataSourceParameters;
import org.apache.dolphinscheduler.plugin.task.api.parameters.resource.ResourceParametersHelper;
import org.apache.commons.lang3.StringUtils;
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
@@ -55,12 +58,18 @@ public class K8sTaskParameters extends AbstractParameters {
public boolean checkParameters() {
return StringUtils.isNotEmpty(image);
}
-
+ public K8sTaskExecutionContext generateExtendedContext(ResourceParametersHelper parametersHelper) {
+ DataSourceParameters dataSourceParameters =
+ (DataSourceParameters) parametersHelper.getResourceParameters(ResourceType.DATASOURCE, datasource);
+ K8sTaskExecutionContext k8sTaskExecutionContext = new K8sTaskExecutionContext();
+ k8sTaskExecutionContext.setConnectionParams(
+ Objects.nonNull(dataSourceParameters) ? dataSourceParameters.getConnectionParams() : null);
+ return k8sTaskExecutionContext;
+ }
@Override
public List<ResourceInfo> getResourceFilesList() {
return new ArrayList<>();
}
-
@Override
public ResourceParametersHelper getResources() {
ResourceParametersHelper resources = super.getResources();
diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java
index fdb39d7c28..fceb29e163 100644
--- a/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java
+++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-k8s/src/main/java/org/apache/dolphinscheduler/plugin/task/k8s/K8sTask.java
@@ -70,16 +70,14 @@ public class K8sTask extends AbstractK8sTask {
}
k8sTaskExecutionContext =
- k8sTaskParameters.generateK8sTaskExecutionContext(taskExecutionContext.getResourceParametersHelper(),
- k8sTaskParameters.getDatasource());
+ k8sTaskParameters.generateExtendedContext(taskExecutionContext.getResourceParametersHelper());
+ taskRequest.setK8sTaskExecutionContext(k8sTaskExecutionContext);
k8sConnectionParam =
(K8sConnectionParam) DataSourceUtils.buildConnectionParams(DbType.valueOf(k8sTaskParameters.getType()),
k8sTaskExecutionContext.getConnectionParams());
String kubeConfig = k8sConnectionParam.getKubeConfig();
k8sTaskParameters.setNamespace(k8sConnectionParam.getNamespace());
k8sTaskParameters.setKubeConfig(kubeConfig);
- k8sTaskExecutionContext.setConfigYaml(kubeConfig);
- taskRequest.setK8sTaskExecutionContext(k8sTaskExecutionContext);
log.info("Initialize k8s task params:{}", JSONUtils.toPrettyJsonString(k8sTaskParameters));
}