You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zh...@apache.org on 2024/01/09 02:04:50 UTC
(dolphinscheduler-sdk-python) branch main updated: Change param `http_params` to dict type in task http (#130)
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler-sdk-python.git
The following commit(s) were added to refs/heads/main by this push:
new 0f120f6 Change param `http_params` to dict type in task http (#130)
0f120f6 is described below
commit 0f120f64eecc848f872c8b8d2eb8776fdf1364ea
Author: Harshit Nagpal <14...@users.noreply.github.com>
AuthorDate: Tue Jan 9 07:34:44 2024 +0530
Change param `http_params` to dict type in task http (#130)
---
UPDATING.md | 2 ++
src/pydolphinscheduler/tasks/http.py | 49 ++++++++++++++++++++++++++++++--
tests/tasks/test_http.py | 55 ++++++++++++++++++++++++++++++++++++
3 files changed, 103 insertions(+), 3 deletions(-)
diff --git a/UPDATING.md b/UPDATING.md
index a38dff2..bdd7f52 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -27,6 +27,8 @@ It started after version 2.0.5 released
* Remove attribute tenant from pydolphinscheduler.core.workflow.workflow ([#54](https://github.com/apache/dolphinscheduler-sdk-python/pull/54))
and please change tenant name in ``config.yaml`` in ``PYDS_HOME``
* Drop support of python3.6 and python3.7 ([#126](https://github.com/apache/dolphinscheduler-sdk-python/pull/126))
+* Change parameter `http_params` to dict type for easy to use in task http.([#130](https://github.com/apache/dolphinscheduler-sdk-python/pull/130))
+
## 4.0.0
diff --git a/src/pydolphinscheduler/tasks/http.py b/src/pydolphinscheduler/tasks/http.py
index 7849317..e64494b 100644
--- a/src/pydolphinscheduler/tasks/http.py
+++ b/src/pydolphinscheduler/tasks/http.py
@@ -19,7 +19,10 @@
from __future__ import annotations
+import warnings
+
from pydolphinscheduler.constants import TaskType
+from pydolphinscheduler.core.parameter import Direction, ParameterHelper
from pydolphinscheduler.core.task import Task
from pydolphinscheduler.exceptions import PyDSParamException
@@ -51,7 +54,30 @@ class HttpCheckCondition:
class Http(Task):
- """Task HTTP object, declare behavior for HTTP task to dolphinscheduler."""
+ """Task HTTP object, declare behavior for HTTP task to dolphinscheduler.
+
+ :param name: The name or identifier for the HTTP task.
+ :param url: The URL endpoint for the HTTP request.
+ :param http_method: The HTTP method for the request (GET, POST, etc.). Defaults to HttpMethod.GET.
+ :param http_params: Parameters for the HTTP request. Defaults to None.
+ :param http_check_condition: Condition for checking the HTTP response status.
+ Defaults to HttpCheckCondition.STATUS_CODE_DEFAULT.
+ :param condition: Additional condition to evaluate if `http_check_condition` is not STATUS_CODE_DEFAULT.
+ :param connect_timeout: Connection timeout for the HTTP request in milliseconds. Defaults to 60000.
+ :param socket_timeout: Socket timeout for the HTTP request in milliseconds. Defaults to 60000.
+
+ Attributes:
+ _task_custom_attr (set): A set containing custom attributes specific to the Http task,
+ including 'url', 'http_method', 'http_params', and more.
+
+
+ Raises:
+ PyDSParamException: Exception raised for invalid parameters, such as unsupported HTTP methods or conditions.
+
+ Example:
+ Usage example for creating an HTTP task:
+ http_task = Http(name="http_task", url="https://api.example.com", http_method="POST", http_params={"key": "value"})
+ """
_task_custom_attr = {
"url",
@@ -68,7 +94,7 @@ class Http(Task):
name: str,
url: str,
http_method: str | None = HttpMethod.GET,
- http_params: str | None = None,
+ http_params: dict | None = None,
http_check_condition: str | None = HttpCheckCondition.STATUS_CODE_DEFAULT,
condition: str | None = None,
connect_timeout: int | None = 60000,
@@ -82,8 +108,16 @@ class Http(Task):
raise PyDSParamException(
"Parameter http_method %s not support.", http_method
)
+
+ if isinstance(http_params, list):
+ warnings.warn(
+ "The `http_params` parameter currently accepts a dictionary instead of a list. Your parameter is being ignored.",
+ DeprecationWarning,
+ )
+
self.http_method = http_method
- self.http_params = http_params or []
+ self._http_params = http_params
+
if not hasattr(HttpCheckCondition, http_check_condition):
raise PyDSParamException(
"Parameter http_check_condition %s not support.", http_check_condition
@@ -99,3 +133,12 @@ class Http(Task):
self.condition = condition
self.connect_timeout = connect_timeout
self.socket_timeout = socket_timeout
+
+ @property
+ def http_params(self):
+ """Property to convert http_params using ParameterHelper when accessed."""
+ return (
+ ParameterHelper.convert_params(self._http_params, direction=Direction.IN)
+ if self._http_params
+ else []
+ )
diff --git a/tests/tasks/test_http.py b/tests/tasks/test_http.py
index 1d13fa6..0c48e55 100644
--- a/tests/tasks/test_http.py
+++ b/tests/tasks/test_http.py
@@ -17,6 +17,7 @@
"""Test Task HTTP."""
+import warnings
from unittest.mock import patch
import pytest
@@ -126,3 +127,57 @@ def test_http_get_define():
):
http = Http(name, url)
assert http.task_params == expect_task_params
+
+
+@patch(
+ "pydolphinscheduler.core.task.Task.gen_code_and_version",
+ return_value=(123, 1),
+)
+def test_http_params(mock_code):
+ """Test the transformation and conversion of http_params."""
+ http_params_dict = {"prop1": "value1", "prop2": 135, "prop3": "value3"}
+
+ http_instance = Http(
+ name="test_http",
+ url="http://www.example.com",
+ http_method="GET",
+ http_params=http_params_dict,
+ )
+ expect_transformation = [
+ {"prop": "prop1", "direct": "IN", "type": "VARCHAR", "value": "value1"},
+ {"prop": "prop2", "direct": "IN", "type": "INTEGER", "value": 135},
+ {"prop": "prop3", "direct": "IN", "type": "VARCHAR", "value": "value3"},
+ ]
+
+ assert isinstance(http_instance.http_params, list)
+ assert len(http_instance.http_params) == len(http_params_dict)
+ assert http_instance.http_params == expect_transformation
+
+
+def test_http_params_deprecation_warning():
+ """Test deprecation warning when user passes list to http_params."""
+ code = 123
+ version = 1
+ name = "test_http_params_deprecation_warning"
+ http_params_list = [
+ {"prop": "abc", "httpParametersType": "PARAMETER", "value": "def"}
+ ]
+
+ with patch(
+ "pydolphinscheduler.core.task.Task.gen_code_and_version",
+ return_value=(code, version),
+ ):
+ with warnings.catch_warnings(record=True) as w:
+ Http(
+ name=name,
+ url="http://www.example.com",
+ http_method="GET",
+ http_params=http_params_list,
+ )
+
+ # Check if a deprecation warning is raised
+ assert len(w) == 1
+ assert issubclass(w[-1].category, DeprecationWarning)
+ assert "The `http_params` parameter currently accepts a dictionary" in str(
+ w[-1].message
+ )