You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by xt...@apache.org on 2025/06/26 09:38:35 UTC
(flink-agents) branch main updated: [plan][python] Support serialize/deserialize workflow plan to/from json. (#39)
This is an automated email from the ASF dual-hosted git repository.
xtsong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-agents.git
The following commit(s) were added to refs/heads/main by this push:
new f799c05 [plan][python] Support serialize/deserialize workflow plan to/from json. (#39)
f799c05 is described below
commit f799c053b50c56bd5a72cc87c97f1061fe52a85b
Author: Wenjin Xie <16...@users.noreply.github.com>
AuthorDate: Thu Jun 26 17:38:29 2025 +0800
[plan][python] Support serialize/deserialize workflow plan to/from json. (#39)
---
.licenserc.yaml | 1 +
python/flink_agents/plan/action.py | 36 +++++++++++++++----
.../flink_agents/plan/tests/resources/action.json | 11 ++++++
.../plan/tests/resources/python_function.json | 4 +++
.../plan/tests/resources/workflow_plan.json | 36 +++++++++++++++++++
python/flink_agents/plan/tests/test_action.py | 37 +++++++++++++++++--
python/flink_agents/plan/tests/test_function.py | 24 ++++++++++++-
.../flink_agents/plan/tests/test_workflow_plan.py | 42 ++++++++++++++++++++--
python/flink_agents/plan/workflow_plan.py | 36 ++++++++++---------
python/flink_agents/runtime/local_runner.py | 3 +-
10 files changed, 201 insertions(+), 29 deletions(-)
diff --git a/.licenserc.yaml b/.licenserc.yaml
index cd87cd8..673b1ac 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -25,6 +25,7 @@ header:
- 'LICENSE'
- 'NOTICE'
- '.gitignore'
+ - '**/*.json'
comment: on-failure
dependency:
files:
diff --git a/python/flink_agents/plan/action.py b/python/flink_agents/plan/action.py
index c52da8e..df49f84 100644
--- a/python/flink_agents/plan/action.py
+++ b/python/flink_agents/plan/action.py
@@ -15,13 +15,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#################################################################################
-from typing import List, Type
+from typing import List
-from pydantic import BaseModel
+from pydantic import BaseModel, field_serializer, model_validator
from flink_agents.api.event import Event
from flink_agents.api.runner_context import RunnerContext
-from flink_agents.plan.function import Function
+from flink_agents.plan.function import Function, JavaFunction, PythonFunction
class Action(BaseModel):
@@ -36,23 +36,45 @@ class Action(BaseModel):
Name/identifier of the workflow Action.
exec : Function
To be executed when the Action is triggered.
- listen_event_types : List[Type[Event]]
+ listen_event_types : List[str]
List of event types that will trigger this Action's execution.
"""
name: str
#TODO: Raise a warning when the action has a return value, as it will be ignored.
exec: Function
- listen_event_types: List[Type[Event]]
+ listen_event_types: List[str]
+
+ @field_serializer('exec')
+ def __serialize_exec(self, exec: Function) -> dict:
+ # append meta info to help deserialize exec
+ data = exec.model_dump()
+ data['func_type'] = exec.__class__.__qualname__
+ return data
+
+ @model_validator(mode='before')
+ def __custom_deserialize(self) -> 'Action':
+ exec = self['exec']
+ # restore exec from serialized json.
+ if isinstance(exec, dict):
+ func_type = exec['func_type']
+ if func_type == 'PythonFunction':
+ self['exec'] = PythonFunction(**exec)
+ elif func_type == 'JavaFunction':
+ self['exec'] = JavaFunction(**exec)
+ else:
+ err_msg = f'Unknown function type: {func_type}'
+ raise NotImplementedError(err_msg)
+ return self
def __init__(
self,
name: str,
exec: Function,
- listen_event_types: List[Type[Event]],
+ listen_event_types: List[str],
) -> None:
"""Action will check function signature when init."""
super().__init__(name=name, exec=exec, listen_event_types=listen_event_types)
#TODO: Update expected signature after import State and Context.
- exec.check_signature(Event, RunnerContext)
+ self.exec.check_signature(Event, RunnerContext)
diff --git a/python/flink_agents/plan/tests/resources/action.json b/python/flink_agents/plan/tests/resources/action.json
new file mode 100644
index 0000000..ae2b1d4
--- /dev/null
+++ b/python/flink_agents/plan/tests/resources/action.json
@@ -0,0 +1,11 @@
+{
+ "name": "legal",
+ "exec": {
+ "module": "flink_agents.plan.tests.test_action",
+ "qualname": "legal_signature",
+ "func_type": "PythonFunction"
+ },
+ "listen_event_types": [
+ "flink_agents.api.event.InputEvent"
+ ]
+}
\ No newline at end of file
diff --git a/python/flink_agents/plan/tests/resources/python_function.json b/python/flink_agents/plan/tests/resources/python_function.json
new file mode 100644
index 0000000..3b9b67a
--- /dev/null
+++ b/python/flink_agents/plan/tests/resources/python_function.json
@@ -0,0 +1,4 @@
+{
+ "module": "flink_agents.plan.tests.test_function",
+ "qualname": "check_class"
+}
\ No newline at end of file
diff --git a/python/flink_agents/plan/tests/resources/workflow_plan.json b/python/flink_agents/plan/tests/resources/workflow_plan.json
new file mode 100644
index 0000000..f5b3dca
--- /dev/null
+++ b/python/flink_agents/plan/tests/resources/workflow_plan.json
@@ -0,0 +1,36 @@
+{
+ "actions": {
+ "first_action": {
+ "name": "first_action",
+ "exec": {
+ "module": "flink_agents.plan.tests.test_workflow_plan",
+ "qualname": "MyWorkflow.first_action",
+ "func_type": "PythonFunction"
+ },
+ "listen_event_types": [
+ "flink_agents.api.event.InputEvent"
+ ]
+ },
+ "second_action": {
+ "name": "second_action",
+ "exec": {
+ "module": "flink_agents.plan.tests.test_workflow_plan",
+ "qualname": "MyWorkflow.second_action",
+ "func_type": "PythonFunction"
+ },
+ "listen_event_types": [
+ "flink_agents.api.event.InputEvent",
+ "flink_agents.plan.tests.test_workflow_plan.MyEvent"
+ ]
+ }
+ },
+ "event_trigger_actions": {
+ "flink_agents.api.event.InputEvent": [
+ "first_action",
+ "second_action"
+ ],
+ "flink_agents.plan.tests.test_workflow_plan.MyEvent": [
+ "second_action"
+ ]
+ }
+}
\ No newline at end of file
diff --git a/python/flink_agents/plan/tests/test_action.py b/python/flink_agents/plan/tests/test_action.py
index 2bb400f..79ed2e7 100644
--- a/python/flink_agents/plan/tests/test_action.py
+++ b/python/flink_agents/plan/tests/test_action.py
@@ -15,6 +15,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#################################################################################
+import json
+from pathlib import Path
+
import pytest
from flink_agents.api.event import InputEvent
@@ -33,7 +36,7 @@ def test_action_signature_legal() -> None: # noqa: D103
Action(
name="legal",
exec=PythonFunction.from_callable(legal_signature),
- listen_event_types=[InputEvent],
+ listen_event_types=[f'{InputEvent.__module__}.{InputEvent.__qualname__}'],
)
def test_action_signature_illegal() -> None: # noqa: D103
@@ -41,7 +44,37 @@ def test_action_signature_illegal() -> None: # noqa: D103
Action(
name="illegal",
exec=PythonFunction.from_callable(illegal_signature),
- listen_event_types=[InputEvent],
+ listen_event_types=[f'{InputEvent.__module__}.{InputEvent.__qualname__}'],
)
+@pytest.fixture(scope="module")
+def action() -> Action: # noqa: D103
+ func = PythonFunction.from_callable(legal_signature)
+ return Action(
+ name="legal",
+ exec=func,
+ listen_event_types=[f'{InputEvent.__module__}.{InputEvent.__qualname__}'],
+ )
+
+current_dir = Path(__file__).parent
+
+def test_action_serialize(action: Action) -> None: # noqa: D103
+ json_value = action.model_dump_json(serialize_as_any=True)
+ with Path.open(Path(f'{current_dir}/resources/action.json')) as f:
+ expected_json = f.read()
+ actual = json.loads(json_value)
+ expected = json.loads(expected_json)
+ assert actual == expected
+
+def test_action_deserialize(action: Action) -> None: # noqa: D103
+ with Path.open(Path(f'{current_dir}/resources/action.json')) as f:
+ expected_json = f.read()
+ action = Action.model_validate_json(expected_json)
+ assert action.name == 'legal'
+ assert action.listen_event_types == ['flink_agents.api.event.InputEvent']
+ func = action.exec
+ assert func.module == 'flink_agents.plan.tests.test_action'
+ assert func.qualname == 'legal_signature'
+
+
diff --git a/python/flink_agents/plan/tests/test_function.py b/python/flink_agents/plan/tests/test_function.py
index 8133938..1c9eabb 100644
--- a/python/flink_agents/plan/tests/test_function.py
+++ b/python/flink_agents/plan/tests/test_function.py
@@ -15,12 +15,14 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#################################################################################
+import json
+from pathlib import Path
from typing import Any, Dict, Tuple
import pytest
from flink_agents.api.event import Event, InputEvent, OutputEvent
-from flink_agents.plan.function import PythonFunction
+from flink_agents.plan.function import Function, PythonFunction
def check_class(input_event: InputEvent, output_event: OutputEvent) -> None: # noqa: D103
@@ -85,3 +87,23 @@ def test_function_signature_generic_type_mismatch() -> None: # noqa: D103
func = PythonFunction.from_callable(check_generic_type)
with pytest.raises(TypeError):
func.check_signature(Tuple[str, ...], Dict[str, Any])
+
+current_dir = Path(__file__).parent
+
+@pytest.fixture(scope="module")
+def func() -> Function: # noqa: D103
+ return PythonFunction.from_callable(check_class)
+
+def test_python_function_serialize(func: Function) -> None: # noqa: D103
+ json_value = func.model_dump_json(serialize_as_any=True)
+ with Path.open(Path(f'{current_dir}/resources/python_function.json')) as f:
+ expected_json = f.read()
+ actual = json.loads(json_value)
+ expected = json.loads(expected_json)
+ assert actual == expected
+
+def test_python_function_deserialize(func: Function) -> None: # noqa: D103
+ with Path.open(Path(f'{current_dir}/resources/python_function.json')) as f:
+ expected_json = f.read()
+ deserialized_func = PythonFunction.model_validate_json(expected_json)
+ assert deserialized_func == func
diff --git a/python/flink_agents/plan/tests/test_workflow_plan.py b/python/flink_agents/plan/tests/test_workflow_plan.py
index 62d316f..49714b2 100644
--- a/python/flink_agents/plan/tests/test_workflow_plan.py
+++ b/python/flink_agents/plan/tests/test_workflow_plan.py
@@ -15,6 +15,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#################################################################################
+import json
+from pathlib import Path
+
import pytest
from flink_agents.api.decorators import action
@@ -36,7 +39,8 @@ class TestWorkflow(Workflow): #noqa D101
def test_from_workflow(): #noqa D102
workflow = TestWorkflow()
workflow_plan = WorkflowPlan.from_workflow(workflow)
- actions = workflow_plan.get_actions(InputEvent)
+ event_type = f'{InputEvent.__module__}.{InputEvent.__name__}'
+ actions = workflow_plan.get_actions(event_type)
assert len(actions) == 1
action = actions[0]
assert action.name == "increment"
@@ -44,7 +48,7 @@ def test_from_workflow(): #noqa D102
assert isinstance(func, PythonFunction)
assert func.module == "flink_agents.plan.tests.test_workflow_plan"
assert func.qualname == "TestWorkflow.increment"
- assert action.listen_event_types == [InputEvent]
+ assert action.listen_event_types == [event_type]
class InvalidWorkflow(Workflow): #noqa D101
@action(InputEvent)
@@ -56,3 +60,37 @@ def test_to_workflow_invalid_signature() -> None: #noqa D103
workflow = InvalidWorkflow()
with pytest.raises(TypeError):
WorkflowPlan.from_workflow(workflow)
+
+class MyEvent(Event):
+ """Event for testing purposes."""
+
+class MyWorkflow(Workflow): # noqa: D101
+ @action(InputEvent)
+ @staticmethod
+ def first_action(event: InputEvent, ctx: RunnerContext) -> None: # noqa: D102
+ pass
+
+ @action(InputEvent, MyEvent)
+ @staticmethod
+ def second_action(event: InputEvent, ctx: RunnerContext) -> None: # noqa: D102
+ pass
+
+@pytest.fixture(scope="module")
+def workflow_plan() -> WorkflowPlan: # noqa: D103
+ return WorkflowPlan.from_workflow(MyWorkflow())
+
+current_dir = Path(__file__).parent
+
+def test_workflow_plan_serialize(workflow_plan: WorkflowPlan) -> None: # noqa: D103
+ json_value = workflow_plan.model_dump_json(serialize_as_any=True, indent=4)
+ with Path.open(Path(f'{current_dir}/resources/workflow_plan.json')) as f:
+ expected_json = f.read()
+ actual = json.loads(json_value)
+ expected = json.loads(expected_json)
+ assert actual == expected
+
+def test_workflow_plan_deserialize(workflow_plan: WorkflowPlan) -> None: # noqa: D103
+ with Path.open(Path(f'{current_dir}/resources/workflow_plan.json')) as f:
+ expected_json = f.read()
+ deserialized_workflow_plan = WorkflowPlan.model_validate_json(expected_json)
+ assert deserialized_workflow_plan == workflow_plan
diff --git a/python/flink_agents/plan/workflow_plan.py b/python/flink_agents/plan/workflow_plan.py
index 2f0a1bb..35136ce 100644
--- a/python/flink_agents/plan/workflow_plan.py
+++ b/python/flink_agents/plan/workflow_plan.py
@@ -15,12 +15,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#################################################################################
-
-from typing import Dict, List, Type
+from typing import Dict, List
from pydantic import BaseModel
-from flink_agents.api.event import Event
from flink_agents.api.workflow import Workflow
from flink_agents.plan.action import Action
from flink_agents.plan.function import PythonFunction
@@ -31,26 +29,29 @@ class WorkflowPlan(BaseModel):
Attributes:
----------
- actions : Dict[Type[Event], List[Action]]
- Mapping of event types to the list of Actions that listen to them.
+ actions: Dict[str, Action]
+ Mapping of action names to actions
+ event_trigger_actions : Dict[Type[Event], str]
+ Mapping of event types to the list of actions name that listen to them.
"""
-
- actions: Dict[Type[Event], List[Action]]
+ actions: Dict[str, Action]
+ event_trigger_actions: Dict[str, List[str]]
@staticmethod
def from_workflow(workflow: Workflow) -> "WorkflowPlan":
"""Build a WorkflowPlan from user defined workflow."""
actions = {}
+ event_trigger_actions = {}
for action in _get_actions(workflow):
assert action.name not in actions, f"Duplicate action name: {action.name}"
+ actions[action.name] = action
for event_type in action.listen_event_types:
- if event_type not in actions:
- actions[event_type] = []
- actions[event_type].append(action)
- return WorkflowPlan(actions=actions)
+ if event_type not in event_trigger_actions:
+ event_trigger_actions[event_type] = []
+ event_trigger_actions[event_type].append(action.name)
+ return WorkflowPlan(actions=actions, event_trigger_actions=event_trigger_actions)
-
- def get_actions(self, event_type: Type[Event]) -> List[Action]:
+ def get_actions(self, event_type: str) -> List[Action]:
"""Get actions that listen to the specified event type.
Parameters
@@ -63,7 +64,8 @@ class WorkflowPlan(BaseModel):
list[Action]
List of Actions that will respond to this event type.
"""
- return self.actions[event_type]
+ return [self.actions[name] for name in self.event_trigger_actions[event_type]]
+
def _get_actions(workflow: Workflow) -> List[Action]:
"""Extract all registered workflow actions from a workflow.
@@ -82,8 +84,10 @@ def _get_actions(workflow: Workflow) -> List[Action]:
for name, value in workflow.__class__.__dict__.items():
if isinstance(value, staticmethod) and hasattr(value, '_listen_events'):
actions.append(Action(name=name, exec=PythonFunction.from_callable(value.__func__),
- listen_event_types=value._listen_events))
+ listen_event_types=[f'{event_type.__module__}.{event_type.__name__}'
+ for event_type in value._listen_events]))
elif callable(value) and hasattr(value, '_listen_events'):
actions.append(Action(name=name, exec=PythonFunction.from_callable(value),
- listen_event_types=value._listen_events))
+ listen_event_types=[f'{event_type.__module__}.{event_type.__name__}'
+ for event_type in value._listen_events]))
return actions
diff --git a/python/flink_agents/runtime/local_runner.py b/python/flink_agents/runtime/local_runner.py
index bf1825f..29a3829 100644
--- a/python/flink_agents/runtime/local_runner.py
+++ b/python/flink_agents/runtime/local_runner.py
@@ -162,7 +162,8 @@ class LocalRunner(WorkflowRunner):
if isinstance(event, OutputEvent):
self.__outputs.append({key: event.output})
continue
- for action in self.__workflow_plan.get_actions(type(event)):
+ event_type = f'{event.__class__.__module__}.{event.__class__.__name__}'
+ for action in self.__workflow_plan.get_actions(event_type):
logger.info(
"key: %s, performing action: %s", key, action.name
)