You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zh...@apache.org on 2022/11/28 14:01:57 UTC
[dolphinscheduler] 03/05: [fix] Add token as authentication for python gateway (#12893)
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch 3.1.2-prepare
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
commit 416c41465dd490bd0546ac79f2c993004482ffe0
Author: Jay Chung <zh...@gmail.com>
AuthorDate: Mon Nov 14 18:43:08 2022 +0800
[fix] Add token as authentication for python gateway (#12893)
separate from #6407. Authentication, add secret to ensure only trusted people could connect to gateway.
fix: #8255
(cherry picked from commit 6d8befa0752c1e8005651c7b57b2301c7b9606fc)
---
.../configuration/PythonGatewayConfiguration.java | 68 +++-------------------
.../dolphinscheduler/api/python/PythonGateway.java | 56 +++++++++---------
.../src/main/resources/application.yaml | 3 +
.../src/main/resources/application.yaml | 3 +
4 files changed, 43 insertions(+), 87 deletions(-)
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java
index 5735e27fd2..8a3a2e521c 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/configuration/PythonGatewayConfiguration.java
@@ -17,13 +17,14 @@
package org.apache.dolphinscheduler.api.configuration;
+import lombok.Data;
+
import org.springframework.boot.context.properties.ConfigurationProperties;
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
-import org.springframework.stereotype.Component;
+import org.springframework.context.annotation.Configuration;
-@Component
-@EnableConfigurationProperties
-@ConfigurationProperties(value = "python-gateway", ignoreUnknownFields = false)
+@Data
+@Configuration
+@ConfigurationProperties(value = "python-gateway")
public class PythonGatewayConfiguration {
private boolean enabled;
private String gatewayServerAddress;
@@ -32,60 +33,5 @@ public class PythonGatewayConfiguration {
private int pythonPort;
private int connectTimeout;
private int readTimeout;
-
- public boolean getEnabled() {
- return enabled;
- }
-
- public void setEnabled(boolean enabled) {
- this.enabled = enabled;
- }
-
- public String getGatewayServerAddress() {
- return gatewayServerAddress;
- }
-
- public void setGatewayServerAddress(String gatewayServerAddress) {
- this.gatewayServerAddress = gatewayServerAddress;
- }
-
- public int getGatewayServerPort() {
- return gatewayServerPort;
- }
-
- public void setGatewayServerPort(int gatewayServerPort) {
- this.gatewayServerPort = gatewayServerPort;
- }
-
- public String getPythonAddress() {
- return pythonAddress;
- }
-
- public void setPythonAddress(String pythonAddress) {
- this.pythonAddress = pythonAddress;
- }
-
- public int getPythonPort() {
- return pythonPort;
- }
-
- public void setPythonPort(int pythonPort) {
- this.pythonPort = pythonPort;
- }
-
- public int getConnectTimeout() {
- return connectTimeout;
- }
-
- public void setConnectTimeout(int connectTimeout) {
- this.connectTimeout = connectTimeout;
- }
-
- public int getReadTimeout() {
- return readTimeout;
- }
-
- public void setReadTimeout(int readTimeout) {
- this.readTimeout = readTimeout;
- }
+ private String authToken;
}
diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java
index d9f0c78674..b79eaf307e 100644
--- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java
+++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/python/PythonGateway.java
@@ -17,18 +17,6 @@
package org.apache.dolphinscheduler.api.python;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-import javax.annotation.PostConstruct;
-
-import org.apache.commons.collections.CollectionUtils;
import org.apache.dolphinscheduler.api.configuration.PythonGatewayConfiguration;
import org.apache.dolphinscheduler.api.dto.EnvironmentDto;
import org.apache.dolphinscheduler.api.dto.resources.ResourceComponent;
@@ -72,6 +60,24 @@ import org.apache.dolphinscheduler.dao.mapper.ProjectUserMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
+
+import py4j.GatewayServer;
+import py4j.GatewayServer.GatewayServerBuilder;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang3.StringUtils;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import javax.annotation.PostConstruct;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
@@ -626,29 +632,27 @@ public class PythonGateway {
@PostConstruct
public void init() {
- if (pythonGatewayConfiguration.getEnabled()) {
+ if (pythonGatewayConfiguration.isEnabled()) {
this.start();
}
}
private void start() {
- GatewayServer server;
try {
InetAddress gatewayHost = InetAddress.getByName(pythonGatewayConfiguration.getGatewayServerAddress());
- InetAddress pythonHost = InetAddress.getByName(pythonGatewayConfiguration.getPythonAddress());
- server = new GatewayServer(
- this,
- pythonGatewayConfiguration.getGatewayServerPort(),
- pythonGatewayConfiguration.getPythonPort(),
- gatewayHost,
- pythonHost,
- pythonGatewayConfiguration.getConnectTimeout(),
- pythonGatewayConfiguration.getReadTimeout(),
- null
- );
+ GatewayServerBuilder serverBuilder = new GatewayServer.GatewayServerBuilder()
+ .entryPoint(this)
+ .javaAddress(gatewayHost)
+ .javaPort(pythonGatewayConfiguration.getGatewayServerPort())
+ .connectTimeout(pythonGatewayConfiguration.getConnectTimeout())
+ .readTimeout(pythonGatewayConfiguration.getReadTimeout());
+ if (!StringUtils.isEmpty(pythonGatewayConfiguration.getAuthToken())) {
+ serverBuilder.authToken(pythonGatewayConfiguration.getAuthToken());
+ }
+
GatewayServer.turnLoggingOn();
logger.info("PythonGatewayService started on: " + gatewayHost.toString());
- server.start();
+ serverBuilder.build().start();
} catch (UnknownHostException e) {
logger.error("exception occurred while constructing PythonGatewayService().", e);
}
diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml
index 9a8381454b..2b51881e07 100644
--- a/dolphinscheduler-api/src/main/resources/application.yaml
+++ b/dolphinscheduler-api/src/main/resources/application.yaml
@@ -121,6 +121,9 @@ metrics:
python-gateway:
# Weather enable python gateway server or not. The default value is true.
enabled: true
+ # Authentication token for connection from python api to python gateway server. Should be changed the default value
+ # when you deploy in public network.
+ auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc
# The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different
# between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost`
gateway-server-address: 0.0.0.0
diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
index a97d03e34c..66c8abd70d 100644
--- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml
+++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
@@ -187,6 +187,9 @@ alert:
python-gateway:
# Weather enable python gateway server or not. The default value is true.
enabled: true
+ # Authentication token for connection from python api to python gateway server. Should be changed the default value
+ # when you deploy in public network.
+ auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc
# The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different
# between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost`
gateway-server-address: 0.0.0.0