You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zh...@apache.org on 2022/01/29 08:48:53 UTC
[dolphinscheduler] branch dev updated: [python] Fix python api can't connect to remote gateway server (#8248)
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler.git
The following commit(s) were added to refs/heads/dev by this push:
new 01936a6 [python] Fix python api can't connect to remote gateway server (#8248)
01936a6 is described below
commit 01936a660e14bfc6ea061bec065e47f4541b7519
Author: Jiajie Zhong <zh...@hotmail.com>
AuthorDate: Sat Jan 29 16:48:18 2022 +0800
[python] Fix python api can't connect to remote gateway server (#8248)
For now, python API could only communicate python gateway server
in the same hosts, this patch makes it could work with different hosts,
and export java gateway setting to configure file
Co-authored-by: kezhenxu94 <ke...@apache.org>
Co-authored-by: ruanwenjun <86...@qq.com>
---
.../src/pydolphinscheduler/constants.py | 4 ++
.../src/pydolphinscheduler/java_gateway.py | 13 +++-
.../server/PythonGatewayServer.java | 30 ++++++--
.../server/config/PythonGatewayConfig.java | 83 ++++++++++++++++++++++
.../src/main/resources/application.yaml | 18 +++++
.../src/main/resources/application.yaml | 18 +++++
6 files changed, 160 insertions(+), 6 deletions(-)
diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py
index 7bd71b9..65bf6c5 100644
--- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py
+++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/constants.py
@@ -99,6 +99,10 @@ class JavaGatewayDefault(str):
RESULT_DATA = "data"
+ SERVER_ADDRESS = "127.0.0.1"
+ SERVER_PORT = 25333
+ AUTO_CONVERT = True
+
class Delimiter(str):
"""Constants for delimiter."""
diff --git a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py
index d0b4c05..2876ed5 100644
--- a/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py
+++ b/dolphinscheduler-python/pydolphinscheduler/src/pydolphinscheduler/java_gateway.py
@@ -26,14 +26,23 @@ from pydolphinscheduler.constants import JavaGatewayDefault
from pydolphinscheduler.exceptions import PyDSJavaGatewayException
-def launch_gateway() -> JavaGateway:
+def launch_gateway(
+ address: Optional[str] = None,
+ port: Optional[int] = None,
+ auto_convert: Optional[bool] = True,
+) -> JavaGateway:
"""Launch java gateway to pydolphinscheduler.
TODO Note that automatic conversion makes calling Java methods slightly less efficient because
in the worst case, Py4J needs to go through all registered converters for all parameters.
This is why automatic conversion is disabled by default.
"""
- gateway = JavaGateway(gateway_parameters=GatewayParameters(auto_convert=True))
+ gateway_parameters = GatewayParameters(
+ address=address or JavaGatewayDefault.SERVER_ADDRESS,
+ port=port or JavaGatewayDefault.SERVER_PORT,
+ auto_convert=auto_convert or JavaGatewayDefault.AUTO_CONVERT,
+ )
+ gateway = JavaGateway(gateway_parameters=gateway_parameters)
return gateway
diff --git a/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java b/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java
index 2b06286..ae97a9d 100644
--- a/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java
+++ b/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/PythonGatewayServer.java
@@ -53,8 +53,11 @@ import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionMapper;
import org.apache.dolphinscheduler.dao.mapper.ProjectMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskDefinitionMapper;
+import org.apache.dolphinscheduler.server.config.PythonGatewayConfig;
import org.apache.dolphinscheduler.spi.enums.ResourceType;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -132,6 +135,9 @@ public class PythonGatewayServer extends SpringBootServletInitializer {
@Autowired
private DataSourceMapper dataSourceMapper;
+ @Autowired
+ private PythonGatewayConfig pythonGatewayConfig;
+
// TODO replace this user to build in admin user if we make sure build in one could not be change
private final User dummyAdminUser = new User() {
{
@@ -501,10 +507,26 @@ public class PythonGatewayServer extends SpringBootServletInitializer {
@PostConstruct
public void run() {
- GatewayServer server = new GatewayServer(this);
- GatewayServer.turnLoggingOn();
- // Start server to accept python client socket
- server.start();
+ GatewayServer server;
+ try {
+ InetAddress gatewayHost = InetAddress.getByName(pythonGatewayConfig.getGatewayServerAddress());
+ InetAddress pythonHost = InetAddress.getByName(pythonGatewayConfig.getPythonAddress());
+ server = new GatewayServer(
+ this,
+ pythonGatewayConfig.getGatewayServerPort(),
+ pythonGatewayConfig.getPythonPort(),
+ gatewayHost,
+ pythonHost,
+ pythonGatewayConfig.getConnectTimeout(),
+ pythonGatewayConfig.getReadTimeout(),
+ null
+ );
+ GatewayServer.turnLoggingOn();
+ logger.info("PythonGatewayServer started on: " + gatewayHost.toString());
+ server.start();
+ } catch (UnknownHostException e) {
+ logger.error("exception occurred while constructing PythonGatewayServer().", e);
+ }
}
public static void main(String[] args) {
diff --git a/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/config/PythonGatewayConfig.java b/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/config/PythonGatewayConfig.java
new file mode 100644
index 0000000..f06ea76
--- /dev/null
+++ b/dolphinscheduler-python/src/main/java/org/apache/dolphinscheduler/server/config/PythonGatewayConfig.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.dolphinscheduler.server.config;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.stereotype.Component;
+
+@Component
+@EnableConfigurationProperties
+@ConfigurationProperties("python-gateway")
+public class PythonGatewayConfig {
+ private String gatewayServerAddress;
+ private int gatewayServerPort;
+ private String pythonAddress;
+ private int pythonPort;
+ private int connectTimeout;
+ private int readTimeout;
+
+ public String getGatewayServerAddress() {
+ return gatewayServerAddress;
+ }
+
+ public void setGatewayServerAddress(String gatewayServerAddress) {
+ this.gatewayServerAddress = gatewayServerAddress;
+ }
+
+ public int getGatewayServerPort() {
+ return gatewayServerPort;
+ }
+
+ public void setGatewayServerPort(int gatewayServerPort) {
+ this.gatewayServerPort = gatewayServerPort;
+ }
+
+ public String getPythonAddress() {
+ return pythonAddress;
+ }
+
+ public void setPythonAddress(String pythonAddress) {
+ this.pythonAddress = pythonAddress;
+ }
+
+ public int getPythonPort() {
+ return pythonPort;
+ }
+
+ public void setPythonPort(int pythonPort) {
+ this.pythonPort = pythonPort;
+ }
+
+ public int getConnectTimeout() {
+ return connectTimeout;
+ }
+
+ public void setConnectTimeout(int connectTimeout) {
+ this.connectTimeout = connectTimeout;
+ }
+
+ public int getReadTimeout() {
+ return readTimeout;
+ }
+
+ public void setReadTimeout(int readTimeout) {
+ this.readTimeout = readTimeout;
+ }
+}
diff --git a/dolphinscheduler-python/src/main/resources/application.yaml b/dolphinscheduler-python/src/main/resources/application.yaml
index af315b1..b5dc00c 100644
--- a/dolphinscheduler-python/src/main/resources/application.yaml
+++ b/dolphinscheduler-python/src/main/resources/application.yaml
@@ -37,6 +37,24 @@ spring:
hibernate:
ddl-auto: none
+python-gateway:
+ # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different
+ # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost`
+ gateway-server-address: 0.0.0.0
+ # The port of Python gateway server start. Define which port you could connect to Python gateway server from
+ # Python API side.
+ gateway-server-port: 25333
+ # The address of Python callback client.
+ python-address: 127.0.0.1
+ # The port of Python callback client.
+ python-port: 25334
+ # Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite),
+ # and socket server would never close even though no requests accept
+ connect-timeout: 0
+ # Close each active connection of socket server if python program not active after x milliseconds. Define value is
+ # (0 = infinite), and socket server would never close even though no requests accept
+ read-timeout: 0
+
server:
port: 54321
diff --git a/dolphinscheduler-standalone-server/src/main/resources/application.yaml b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
index 5499e6f..53cfcbc 100644
--- a/dolphinscheduler-standalone-server/src/main/resources/application.yaml
+++ b/dolphinscheduler-standalone-server/src/main/resources/application.yaml
@@ -138,6 +138,24 @@ worker:
alert:
port: 50052
+python-gateway:
+ # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different
+ # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost`
+ gateway-server-address: 0.0.0.0
+ # The port of Python gateway server start. Define which port you could connect to Python gateway server from
+ # Python API side.
+ gateway-server-port: 25333
+ # The address of Python callback client.
+ python-address: 127.0.0.1
+ # The port of Python callback client.
+ python-port: 25334
+ # Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite),
+ # and socket server would never close even though no requests accept
+ connect-timeout: 0
+ # Close each active connection of socket server if python program not active after x milliseconds. Define value is
+ # (0 = infinite), and socket server would never close even though no requests accept
+ read-timeout: 0
+
server:
port: 12345
servlet: