You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by xt...@apache.org on 2025/06/26 09:38:35 UTC

(flink-agents) branch main updated: [plan][python] Support serialize/deserialize workflow plan to/from json. (#39)

This is an automated email from the ASF dual-hosted git repository.

xtsong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/flink-agents.git


The following commit(s) were added to refs/heads/main by this push:
     new f799c05  [plan][python] Support serialize/deserialize workflow plan to/from json. (#39)
f799c05 is described below

commit f799c053b50c56bd5a72cc87c97f1061fe52a85b
Author: Wenjin Xie <16...@users.noreply.github.com>
AuthorDate: Thu Jun 26 17:38:29 2025 +0800

    [plan][python] Support serialize/deserialize workflow plan to/from json. (#39)
---
 .licenserc.yaml                                    |  1 +
 python/flink_agents/plan/action.py                 | 36 +++++++++++++++----
 .../flink_agents/plan/tests/resources/action.json  | 11 ++++++
 .../plan/tests/resources/python_function.json      |  4 +++
 .../plan/tests/resources/workflow_plan.json        | 36 +++++++++++++++++++
 python/flink_agents/plan/tests/test_action.py      | 37 +++++++++++++++++--
 python/flink_agents/plan/tests/test_function.py    | 24 ++++++++++++-
 .../flink_agents/plan/tests/test_workflow_plan.py  | 42 ++++++++++++++++++++--
 python/flink_agents/plan/workflow_plan.py          | 36 ++++++++++---------
 python/flink_agents/runtime/local_runner.py        |  3 +-
 10 files changed, 201 insertions(+), 29 deletions(-)

diff --git a/.licenserc.yaml b/.licenserc.yaml
index cd87cd8..673b1ac 100644
--- a/.licenserc.yaml
+++ b/.licenserc.yaml
@@ -25,6 +25,7 @@ header:
     - 'LICENSE'
     - 'NOTICE'
     - '.gitignore'
+    - '**/*.json'
   comment: on-failure
   dependency:
     files:
diff --git a/python/flink_agents/plan/action.py b/python/flink_agents/plan/action.py
index c52da8e..df49f84 100644
--- a/python/flink_agents/plan/action.py
+++ b/python/flink_agents/plan/action.py
@@ -15,13 +15,13 @@
 #  See the License for the specific language governing permissions and
 # limitations under the License.
 #################################################################################
-from typing import List, Type
+from typing import List
 
-from pydantic import BaseModel
+from pydantic import BaseModel, field_serializer, model_validator
 
 from flink_agents.api.event import Event
 from flink_agents.api.runner_context import RunnerContext
-from flink_agents.plan.function import Function
+from flink_agents.plan.function import Function, JavaFunction, PythonFunction
 
 
 class Action(BaseModel):
@@ -36,23 +36,45 @@ class Action(BaseModel):
         Name/identifier of the workflow Action.
     exec : Function
         To be executed when the Action is triggered.
-    listen_event_types : List[Type[Event]]
+    listen_event_types : List[str]
         List of event types that will trigger this Action's execution.
     """
 
     name: str
     #TODO: Raise a warning when the action has a return value, as it will be ignored.
     exec: Function
-    listen_event_types: List[Type[Event]]
+    listen_event_types: List[str]
+
+    @field_serializer('exec')
+    def __serialize_exec(self, exec: Function) -> dict:
+        # append meta info to help deserialize exec
+        data = exec.model_dump()
+        data['func_type'] = exec.__class__.__qualname__
+        return data
+
+    @model_validator(mode='before')
+    def __custom_deserialize(self) -> 'Action':
+        exec = self['exec']
+        # restore exec from serialized json.
+        if isinstance(exec, dict):
+            func_type = exec['func_type']
+            if func_type == 'PythonFunction':
+                self['exec'] = PythonFunction(**exec)
+            elif func_type == 'JavaFunction':
+                self['exec'] = JavaFunction(**exec)
+            else:
+                err_msg = f'Unknown function type: {func_type}'
+                raise NotImplementedError(err_msg)
+        return self
 
     def __init__(
             self,
             name: str,
             exec: Function,
-            listen_event_types: List[Type[Event]],
+            listen_event_types: List[str],
     ) -> None:
         """Action will check function signature when init."""
         super().__init__(name=name, exec=exec, listen_event_types=listen_event_types)
         #TODO: Update expected signature after import State and Context.
-        exec.check_signature(Event, RunnerContext)
+        self.exec.check_signature(Event, RunnerContext)
 
diff --git a/python/flink_agents/plan/tests/resources/action.json b/python/flink_agents/plan/tests/resources/action.json
new file mode 100644
index 0000000..ae2b1d4
--- /dev/null
+++ b/python/flink_agents/plan/tests/resources/action.json
@@ -0,0 +1,11 @@
+{
+    "name": "legal",
+    "exec": {
+        "module": "flink_agents.plan.tests.test_action",
+        "qualname": "legal_signature",
+        "func_type": "PythonFunction"
+    },
+    "listen_event_types": [
+        "flink_agents.api.event.InputEvent"
+    ]
+}
\ No newline at end of file
diff --git a/python/flink_agents/plan/tests/resources/python_function.json b/python/flink_agents/plan/tests/resources/python_function.json
new file mode 100644
index 0000000..3b9b67a
--- /dev/null
+++ b/python/flink_agents/plan/tests/resources/python_function.json
@@ -0,0 +1,4 @@
+{
+    "module": "flink_agents.plan.tests.test_function",
+    "qualname": "check_class"
+}
\ No newline at end of file
diff --git a/python/flink_agents/plan/tests/resources/workflow_plan.json b/python/flink_agents/plan/tests/resources/workflow_plan.json
new file mode 100644
index 0000000..f5b3dca
--- /dev/null
+++ b/python/flink_agents/plan/tests/resources/workflow_plan.json
@@ -0,0 +1,36 @@
+{
+    "actions": {
+        "first_action": {
+            "name": "first_action",
+            "exec": {
+                "module": "flink_agents.plan.tests.test_workflow_plan",
+                "qualname": "MyWorkflow.first_action",
+                "func_type": "PythonFunction"
+            },
+            "listen_event_types": [
+                "flink_agents.api.event.InputEvent"
+            ]
+        },
+        "second_action": {
+            "name": "second_action",
+            "exec": {
+                "module": "flink_agents.plan.tests.test_workflow_plan",
+                "qualname": "MyWorkflow.second_action",
+                "func_type": "PythonFunction"
+            },
+            "listen_event_types": [
+                "flink_agents.api.event.InputEvent",
+                "flink_agents.plan.tests.test_workflow_plan.MyEvent"
+            ]
+        }
+    },
+    "event_trigger_actions": {
+        "flink_agents.api.event.InputEvent": [
+            "first_action",
+            "second_action"
+        ],
+        "flink_agents.plan.tests.test_workflow_plan.MyEvent": [
+            "second_action"
+        ]
+    }
+}
\ No newline at end of file
diff --git a/python/flink_agents/plan/tests/test_action.py b/python/flink_agents/plan/tests/test_action.py
index 2bb400f..79ed2e7 100644
--- a/python/flink_agents/plan/tests/test_action.py
+++ b/python/flink_agents/plan/tests/test_action.py
@@ -15,6 +15,9 @@
 #  See the License for the specific language governing permissions and
 # limitations under the License.
 #################################################################################
+import json
+from pathlib import Path
+
 import pytest
 
 from flink_agents.api.event import InputEvent
@@ -33,7 +36,7 @@ def test_action_signature_legal() -> None: # noqa: D103
     Action(
         name="legal",
         exec=PythonFunction.from_callable(legal_signature),
-        listen_event_types=[InputEvent],
+        listen_event_types=[f'{InputEvent.__module__}.{InputEvent.__qualname__}'],
     )
 
 def test_action_signature_illegal() -> None:  # noqa: D103
@@ -41,7 +44,37 @@ def test_action_signature_illegal() -> None:  # noqa: D103
         Action(
             name="illegal",
             exec=PythonFunction.from_callable(illegal_signature),
-            listen_event_types=[InputEvent],
+            listen_event_types=[f'{InputEvent.__module__}.{InputEvent.__qualname__}'],
         )
 
+@pytest.fixture(scope="module")
+def action() -> Action: # noqa: D103
+    func = PythonFunction.from_callable(legal_signature)
+    return Action(
+        name="legal",
+        exec=func,
+        listen_event_types=[f'{InputEvent.__module__}.{InputEvent.__qualname__}'],
+    )
+
+current_dir = Path(__file__).parent
+
+def test_action_serialize(action: Action) -> None:  # noqa: D103
+    json_value = action.model_dump_json(serialize_as_any=True)
+    with Path.open(Path(f'{current_dir}/resources/action.json')) as f:
+        expected_json = f.read()
+    actual = json.loads(json_value)
+    expected = json.loads(expected_json)
+    assert actual == expected
+
+def test_action_deserialize(action: Action) -> None: # noqa: D103
+    with Path.open(Path(f'{current_dir}/resources/action.json')) as f:
+        expected_json = f.read()
+    action = Action.model_validate_json(expected_json)
+    assert action.name == 'legal'
+    assert action.listen_event_types == ['flink_agents.api.event.InputEvent']
+    func = action.exec
+    assert func.module == 'flink_agents.plan.tests.test_action'
+    assert func.qualname == 'legal_signature'
+
+
 
diff --git a/python/flink_agents/plan/tests/test_function.py b/python/flink_agents/plan/tests/test_function.py
index 8133938..1c9eabb 100644
--- a/python/flink_agents/plan/tests/test_function.py
+++ b/python/flink_agents/plan/tests/test_function.py
@@ -15,12 +15,14 @@
 #  See the License for the specific language governing permissions and
 # limitations under the License.
 #################################################################################
+import json
+from pathlib import Path
 from typing import Any, Dict, Tuple
 
 import pytest
 
 from flink_agents.api.event import Event, InputEvent, OutputEvent
-from flink_agents.plan.function import PythonFunction
+from flink_agents.plan.function import Function, PythonFunction
 
 
 def check_class(input_event: InputEvent, output_event: OutputEvent) -> None: # noqa: D103
@@ -85,3 +87,23 @@ def test_function_signature_generic_type_mismatch() -> None: # noqa: D103
     func = PythonFunction.from_callable(check_generic_type)
     with pytest.raises(TypeError):
         func.check_signature(Tuple[str, ...], Dict[str, Any])
+
+current_dir = Path(__file__).parent
+
+@pytest.fixture(scope="module")
+def func() -> Function: # noqa: D103
+    return PythonFunction.from_callable(check_class)
+
+def test_python_function_serialize(func: Function) -> None: # noqa: D103
+    json_value = func.model_dump_json(serialize_as_any=True)
+    with Path.open(Path(f'{current_dir}/resources/python_function.json')) as f:
+        expected_json = f.read()
+    actual = json.loads(json_value)
+    expected = json.loads(expected_json)
+    assert actual == expected
+
+def test_python_function_deserialize(func: Function) -> None: # noqa: D103
+    with Path.open(Path(f'{current_dir}/resources/python_function.json')) as f:
+        expected_json = f.read()
+    deserialized_func = PythonFunction.model_validate_json(expected_json)
+    assert deserialized_func == func
diff --git a/python/flink_agents/plan/tests/test_workflow_plan.py b/python/flink_agents/plan/tests/test_workflow_plan.py
index 62d316f..49714b2 100644
--- a/python/flink_agents/plan/tests/test_workflow_plan.py
+++ b/python/flink_agents/plan/tests/test_workflow_plan.py
@@ -15,6 +15,9 @@
 #  See the License for the specific language governing permissions and
 # limitations under the License.
 #################################################################################
+import json
+from pathlib import Path
+
 import pytest
 
 from flink_agents.api.decorators import action
@@ -36,7 +39,8 @@ class TestWorkflow(Workflow): #noqa D101
 def test_from_workflow(): #noqa D102
     workflow = TestWorkflow()
     workflow_plan = WorkflowPlan.from_workflow(workflow)
-    actions = workflow_plan.get_actions(InputEvent)
+    event_type = f'{InputEvent.__module__}.{InputEvent.__name__}'
+    actions = workflow_plan.get_actions(event_type)
     assert len(actions) == 1
     action = actions[0]
     assert action.name == "increment"
@@ -44,7 +48,7 @@ def test_from_workflow(): #noqa D102
     assert isinstance(func, PythonFunction)
     assert func.module == "flink_agents.plan.tests.test_workflow_plan"
     assert func.qualname == "TestWorkflow.increment"
-    assert action.listen_event_types == [InputEvent]
+    assert action.listen_event_types == [event_type]
 
 class InvalidWorkflow(Workflow): #noqa D101
     @action(InputEvent)
@@ -56,3 +60,37 @@ def test_to_workflow_invalid_signature() -> None: #noqa D103
     workflow = InvalidWorkflow()
     with pytest.raises(TypeError):
         WorkflowPlan.from_workflow(workflow)
+
+class MyEvent(Event):
+    """Event for testing purposes."""
+
+class MyWorkflow(Workflow): # noqa: D101
+    @action(InputEvent)
+    @staticmethod
+    def first_action(event: InputEvent, ctx: RunnerContext) -> None: # noqa: D102
+        pass
+
+    @action(InputEvent, MyEvent)
+    @staticmethod
+    def second_action(event: InputEvent, ctx: RunnerContext) -> None: # noqa: D102
+        pass
+
+@pytest.fixture(scope="module")
+def workflow_plan() -> WorkflowPlan: # noqa: D103
+    return WorkflowPlan.from_workflow(MyWorkflow())
+
+current_dir = Path(__file__).parent
+
+def test_workflow_plan_serialize(workflow_plan: WorkflowPlan) -> None: # noqa: D103
+    json_value = workflow_plan.model_dump_json(serialize_as_any=True, indent=4)
+    with Path.open(Path(f'{current_dir}/resources/workflow_plan.json')) as f:
+        expected_json = f.read()
+    actual = json.loads(json_value)
+    expected = json.loads(expected_json)
+    assert actual == expected
+
+def test_workflow_plan_deserialize(workflow_plan: WorkflowPlan) -> None: # noqa: D103
+    with Path.open(Path(f'{current_dir}/resources/workflow_plan.json')) as f:
+        expected_json = f.read()
+    deserialized_workflow_plan = WorkflowPlan.model_validate_json(expected_json)
+    assert deserialized_workflow_plan == workflow_plan
diff --git a/python/flink_agents/plan/workflow_plan.py b/python/flink_agents/plan/workflow_plan.py
index 2f0a1bb..35136ce 100644
--- a/python/flink_agents/plan/workflow_plan.py
+++ b/python/flink_agents/plan/workflow_plan.py
@@ -15,12 +15,10 @@
 #  See the License for the specific language governing permissions and
 # limitations under the License.
 #################################################################################
-
-from typing import Dict, List, Type
+from typing import Dict, List
 
 from pydantic import BaseModel
 
-from flink_agents.api.event import Event
 from flink_agents.api.workflow import Workflow
 from flink_agents.plan.action import Action
 from flink_agents.plan.function import PythonFunction
@@ -31,26 +29,29 @@ class WorkflowPlan(BaseModel):
 
     Attributes:
     ----------
-    actions : Dict[Type[Event], List[Action]]
-        Mapping of event types to the list of Actions that listen to them.
+    actions: Dict[str, Action]
+        Mapping of action names to actions
+    event_trigger_actions : Dict[Type[Event], str]
+        Mapping of event types to the list of actions name that listen to them.
     """
-
-    actions: Dict[Type[Event], List[Action]]
+    actions: Dict[str, Action]
+    event_trigger_actions: Dict[str, List[str]]
 
     @staticmethod
     def from_workflow(workflow: Workflow) -> "WorkflowPlan":
         """Build a WorkflowPlan from user defined workflow."""
         actions = {}
+        event_trigger_actions = {}
         for action in _get_actions(workflow):
             assert action.name not in actions, f"Duplicate action name: {action.name}"
+            actions[action.name] = action
             for event_type in action.listen_event_types:
-                if event_type not in actions:
-                    actions[event_type] = []
-                actions[event_type].append(action)
-        return WorkflowPlan(actions=actions)
+                if event_type not in event_trigger_actions:
+                    event_trigger_actions[event_type] = []
+                event_trigger_actions[event_type].append(action.name)
+        return WorkflowPlan(actions=actions, event_trigger_actions=event_trigger_actions)
 
-
-    def get_actions(self, event_type: Type[Event]) -> List[Action]:
+    def get_actions(self, event_type: str) -> List[Action]:
         """Get actions that listen to the specified event type.
 
         Parameters
@@ -63,7 +64,8 @@ class WorkflowPlan(BaseModel):
         list[Action]
             List of Actions that will respond to this event type.
         """
-        return self.actions[event_type]
+        return [self.actions[name] for name in self.event_trigger_actions[event_type]]
+
 
 def _get_actions(workflow: Workflow) -> List[Action]:
     """Extract all registered workflow actions from a workflow.
@@ -82,8 +84,10 @@ def _get_actions(workflow: Workflow) -> List[Action]:
     for name, value in workflow.__class__.__dict__.items():
         if isinstance(value, staticmethod) and hasattr(value, '_listen_events'):
             actions.append(Action(name=name, exec=PythonFunction.from_callable(value.__func__),
-                                  listen_event_types=value._listen_events))
+                                  listen_event_types=[f'{event_type.__module__}.{event_type.__name__}'
+                                                      for event_type in value._listen_events]))
         elif callable(value) and hasattr(value, '_listen_events'):
             actions.append(Action(name=name, exec=PythonFunction.from_callable(value),
-                                  listen_event_types=value._listen_events))
+                                  listen_event_types=[f'{event_type.__module__}.{event_type.__name__}'
+                                                      for event_type in value._listen_events]))
     return actions
diff --git a/python/flink_agents/runtime/local_runner.py b/python/flink_agents/runtime/local_runner.py
index bf1825f..29a3829 100644
--- a/python/flink_agents/runtime/local_runner.py
+++ b/python/flink_agents/runtime/local_runner.py
@@ -162,7 +162,8 @@ class LocalRunner(WorkflowRunner):
             if isinstance(event, OutputEvent):
                 self.__outputs.append({key: event.output})
                 continue
-            for action in self.__workflow_plan.get_actions(type(event)):
+            event_type = f'{event.__class__.__module__}.{event.__class__.__name__}'
+            for action in self.__workflow_plan.get_actions(event_type):
                 logger.info(
                     "key: %s, performing action: %s", key, action.name
                 )