You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@dolphinscheduler.apache.org by zh...@apache.org on 2022/12/09 01:52:54 UTC
[dolphinscheduler-sdk-python] branch main updated: [fix] Resource upload in dev branch (#35)
This is an automated email from the ASF dual-hosted git repository.
zhongjiajie pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/dolphinscheduler-sdk-python.git
The following commit(s) were added to refs/heads/main by this push:
new 85a1dfa [fix] Resource upload in dev branch (#35)
85a1dfa is described below
commit 85a1dfaaa72b41dc6e9376ba56c404f93140a18a
Author: Jay Chung <zh...@gmail.com>
AuthorDate: Fri Dec 9 09:52:48 2022 +0800
[fix] Resource upload in dev branch (#35)
---
.github/workflows/ci.yaml | 2 +-
src/pydolphinscheduler/constants.py | 2 +-
src/pydolphinscheduler/core/resource.py | 9 ++++-----
src/pydolphinscheduler/core/task.py | 10 ++++++----
src/pydolphinscheduler/java_gateway.py | 8 ++------
tests/core/test_task.py | 12 ++++++------
tests/integration/test_resources.py | 6 ++++++
tests/testing/constants.py | 2 ++
8 files changed, 28 insertions(+), 23 deletions(-)
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 7268186..35af018 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -151,7 +151,7 @@ jobs:
integrate-test:
needs: license
runs-on: ubuntu-latest
- if: ${{ github.event_name == 'schedule' || contains(github.event.head_commit.message, '[run-it]') }}
+ if: ${{ github.event_name == 'schedule' }} || contains(github.event.head_commit.message, '[run-it]')
timeout-minutes: 30
steps:
- name: Checkout Dolphinscheduler SDK Python
diff --git a/src/pydolphinscheduler/constants.py b/src/pydolphinscheduler/constants.py
index e916ed2..62b6885 100644
--- a/src/pydolphinscheduler/constants.py
+++ b/src/pydolphinscheduler/constants.py
@@ -111,7 +111,7 @@ class Time(str):
class ResourceKey(str):
"""Constants for key of resource."""
- ID = "id"
+ NAME = "resourceName"
class Symbol(str):
diff --git a/src/pydolphinscheduler/core/resource.py b/src/pydolphinscheduler/core/resource.py
index 6833efa..907114f 100644
--- a/src/pydolphinscheduler/core/resource.py
+++ b/src/pydolphinscheduler/core/resource.py
@@ -55,9 +55,9 @@ class Resource(Base):
)
return gateway.query_resources_file_info(self.user_name, self.name)
- def get_id_from_database(self):
- """Get resource id from java gateway."""
- return self.get_info_from_database().getId()
+ def get_fullname_from_database(self):
+ """Get resource fullname from java gateway."""
+ return self.get_info_from_database().getFullName()
def create_or_update_resource(self):
"""Create or update resource via java gateway."""
@@ -65,9 +65,8 @@ class Resource(Base):
raise PyDSParamException(
"`user_name` and `content` are required when create or update resource from python gate."
)
- gateway.create_or_update_resource(
+ return gateway.create_or_update_resource(
self.user_name,
self.name,
self.content,
- self.description,
)
diff --git a/src/pydolphinscheduler/core/task.py b/src/pydolphinscheduler/core/task.py
index efab3e5..29f0ed5 100644
--- a/src/pydolphinscheduler/core/task.py
+++ b/src/pydolphinscheduler/core/task.py
@@ -197,9 +197,11 @@ class Task(Base):
for res in self._resource_list:
if type(res) == str:
resources.add(
- Resource(name=res, user_name=self.user_name).get_id_from_database()
+ Resource(
+ name=res, user_name=self.user_name
+ ).get_fullname_from_database()
)
- elif type(res) == dict and res.get(ResourceKey.ID) is not None:
+ elif type(res) == dict and ResourceKey.NAME in res:
warnings.warn(
"""`resource_list` should be defined using List[str] with resource paths,
the use of ids to define resources will be remove in version 3.2.0.
@@ -207,8 +209,8 @@ class Task(Base):
DeprecationWarning,
stacklevel=2,
)
- resources.add(res.get(ResourceKey.ID))
- return [{ResourceKey.ID: r} for r in resources]
+ resources.add(res.get(ResourceKey.NAME))
+ return [{ResourceKey.NAME: r} for r in resources]
@property
def user_name(self) -> Optional[str]:
diff --git a/src/pydolphinscheduler/java_gateway.py b/src/pydolphinscheduler/java_gateway.py
index 9ec039a..ea15ae5 100644
--- a/src/pydolphinscheduler/java_gateway.py
+++ b/src/pydolphinscheduler/java_gateway.py
@@ -113,13 +113,9 @@ class GatewayEntryPoint:
"""Get resources file info through java gateway."""
return self.gateway.entry_point.getResourcesFileInfo(program_type, main_package)
- def create_or_update_resource(
- self, user_name: str, name: str, content: str, description: Optional[str] = None
- ):
+ def create_or_update_resource(self, user_name: str, name: str, content: str):
"""Create or update resource through java gateway."""
- return self.gateway.entry_point.createOrUpdateResource(
- user_name, name, description, content
- )
+ return self.gateway.entry_point.createOrUpdateResource(user_name, name, content)
def query_resources_file_info(self, user_name: str, name: str):
"""Get resources file info through java gateway."""
diff --git a/tests/core/test_task.py b/tests/core/test_task.py
index 892e2f0..82bfec9 100644
--- a/tests/core/test_task.py
+++ b/tests/core/test_task.py
@@ -123,7 +123,7 @@ def test__get_attr(addition: Set, ignore: Set, expect: Set):
},
{
"localParams": ["foo", "bar"],
- "resourceList": [{"id": 1}],
+ "resourceList": [{"resourceName": 1}],
"dependence": {"foo", "bar"},
"waitStartTimeout": {"foo", "bar"},
"conditionResult": {"foo": ["bar"]},
@@ -132,7 +132,7 @@ def test__get_attr(addition: Set, ignore: Set, expect: Set):
],
)
@patch(
- "pydolphinscheduler.core.resource.Resource.get_id_from_database",
+ "pydolphinscheduler.core.resource.Resource.get_fullname_from_database",
return_value=1,
)
@patch(
@@ -438,11 +438,11 @@ def test_task_obtain_res_plugin_exception(m_get_content, m_code_version, attr):
[
(
["/dev/test.py"],
- [{"id": 1}],
+ [{"resourceName": 1}],
),
(
- ["/dev/test.py", {"id": 2}],
- [{"id": 1}, {"id": 2}],
+ ["/dev/test.py", {"resourceName": 2}],
+ [{"resourceName": 1}, {"resourceName": 2}],
),
],
)
@@ -451,7 +451,7 @@ def test_task_obtain_res_plugin_exception(m_get_content, m_code_version, attr):
return_value=(123, 1),
)
@patch(
- "pydolphinscheduler.core.resource.Resource.get_id_from_database",
+ "pydolphinscheduler.core.resource.Resource.get_fullname_from_database",
return_value=1,
)
@patch(
diff --git a/tests/integration/test_resources.py b/tests/integration/test_resources.py
index ee1d99d..a8fd8f7 100644
--- a/tests/integration/test_resources.py
+++ b/tests/integration/test_resources.py
@@ -45,6 +45,9 @@ def tmp_user():
user.delete()
+@pytest.mark.skip(
+ "activate it when dolphinscheduler default resource center is local file"
+)
def test_create_or_update(tmp_user):
"""Test create or update resource to java gateway."""
resource = Resource(name=name, content=content, user_name=UNIT_TEST_USER_NAME)
@@ -53,6 +56,9 @@ def test_create_or_update(tmp_user):
assert result.getAlias() == name
+@pytest.mark.skip(
+ "activate it when dolphinscheduler default resource center is local file"
+)
def test_get_resource_info(tmp_user):
"""Test get resource info from java gateway."""
resource = Resource(name=name, user_name=UNIT_TEST_USER_NAME)
diff --git a/tests/testing/constants.py b/tests/testing/constants.py
index dbbf5e5..4dfaa37 100644
--- a/tests/testing/constants.py
+++ b/tests/testing/constants.py
@@ -36,6 +36,8 @@ ignore_exec_examples = {
"task_flink_example",
"task_map_reduce_example",
"task_spark_example",
+ # TODO activate it when dolphinscheduler default resource center is local file
+ "multi_resources_example",
}
# pydolphinscheduler environment home