You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by lh...@apache.org on 2026/07/25 17:50:59 UTC

(pulsar) branch master updated: [fix][fn] Forward source message properties in Python runtime (#26191)

This is an automated email from the ASF dual-hosted git repository.

lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git


The following commit(s) were added to refs/heads/master by this push:
     new f07e0ed5899d [fix][fn] Forward source message properties in Python runtime (#26191)
f07e0ed5899d is described below

commit f07e0ed5899d29351774df25b138d4b92557ae35
Author: sbourkeostk <58...@users.noreply.github.com>
AuthorDate: Sat Jul 25 18:50:52 2026 +0100

    [fix][fn] Forward source message properties in Python runtime (#26191)
    
    Co-authored-by: Stephen Bourke <sb...@beyond.com>
---
 .../instance/src/main/python/python_instance.py    |  5 +-
 pulsar-functions/instance/src/main/python/util.py  |  4 +-
 .../src/scripts/run_python_instance_tests.sh       |  4 +-
 .../src/test/python/test_python_instance.py        | 61 +++++++++++++++++++++-
 4 files changed, 68 insertions(+), 6 deletions(-)

diff --git a/pulsar-functions/instance/src/main/python/python_instance.py b/pulsar-functions/instance/src/main/python/python_instance.py
index 2ab3ccc46171..5c57dfef7900 100755
--- a/pulsar-functions/instance/src/main/python/python_instance.py
+++ b/pulsar-functions/instance/src/main/python/python_instance.py
@@ -337,7 +337,10 @@ class PythonInstance(object):
         output_object = self.output_serde.serialize(output)
 
       if output_object is not None:
-        props = {"__pfn_input_topic__" : str(msg.topic), "__pfn_input_msg_id__" : base64ify(msg.message.message_id().serialize())}
+        props = {}
+        if self.instance_config.function_details.sink.forwardSourceMessageProperty:
+          props = msg.message.properties()
+        props.update({"__pfn_input_topic__" : str(msg.topic), "__pfn_input_msg_id__" : base64ify(msg.message.message_id().serialize())})
         if self.effectively_once:
           self.producer.send_async(output_object,
                                    partial(self.done_producing, msg.consumer, msg.message, self.producer.topic()),
diff --git a/pulsar-functions/instance/src/main/python/util.py b/pulsar-functions/instance/src/main/python/util.py
index f5868093faea..7b2a2d7b4d17 100755
--- a/pulsar-functions/instance/src/main/python/util.py
+++ b/pulsar-functions/instance/src/main/python/util.py
@@ -105,8 +105,8 @@ class FixedTimer():
         self.t = t
         self.hFunction = hFunction
         self.thread = Timer(self.t, self.handle_function)
-        self.thread.setName(name)
-        self.thread.setDaemon(True)
+        self.thread.name = name
+        self.thread.daemon = True
 
     def handle_function(self):
         self.hFunction()
diff --git a/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh b/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh
index 5cb729686790..11cec15e14c0 100755
--- a/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh
+++ b/pulsar-functions/instance/src/scripts/run_python_instance_tests.sh
@@ -21,11 +21,11 @@
 
 # Make sure dependencies are installed
 pip3 install mock --user
-pip3 install protobuf==3.20.1 --user
+pip3 install protobuf==6.31.1 --user
 pip3 install fastavro --user
 
 CUR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 PULSAR_HOME="$( cd "$CUR_DIR/../../../../" >/dev/null && pwd )"
 
 # run instance tests
-PULSAR_HOME=${PULSAR_HOME} PYTHONPATH=${PULSAR_HOME}/pulsar-functions/instance/target/python-instance python3 -m unittest discover -v ${PULSAR_HOME}/pulsar-functions/instance/target/python-instance/tests
+PULSAR_HOME=${PULSAR_HOME} PYTHONPATH=${PULSAR_HOME}/pulsar-functions/instance/src/main/python python3 -m unittest discover -v -s ${PULSAR_HOME}/pulsar-functions/instance/src/test/python
diff --git a/pulsar-functions/instance/src/test/python/test_python_instance.py b/pulsar-functions/instance/src/test/python/test_python_instance.py
index 975ddfd90288..1e72db854581 100644
--- a/pulsar-functions/instance/src/test/python/test_python_instance.py
+++ b/pulsar-functions/instance/src/test/python/test_python_instance.py
@@ -23,6 +23,12 @@
 from mock import Mock
 import sys
 sys.modules['prometheus_client'] = Mock()
+sys.modules['bookkeeper'] = Mock()
+sys.modules['bookkeeper.types'] = Mock()
+sys.modules['bookkeeper.common'] = Mock()
+sys.modules['bookkeeper.common.exceptions'] = Mock()
+sys.modules['bookkeeper.proto'] = Mock()
+sys.modules['bookkeeper.proto.stream_pb2'] = Mock()
 
 from contextimpl import ContextImpl
 from python_instance import PythonInstance, InstanceConfig
@@ -42,7 +48,8 @@ class TestContextImpl(unittest.TestCase):
     return Any()
 
   def setUp(self):
-    log.init_logger("INFO", "foo", os.environ.get("PULSAR_HOME") + "/conf/functions-logging/console_logging_config.ini")
+    if not hasattr(sys.stdout, 'logger'):
+      log.init_logger("INFO", "foo", os.environ.get("PULSAR_HOME") + "/conf/functions-logging/console_logging_config.ini")
 
   def test_context_publish(self):
     instance_id = 'test_instance_id'
@@ -90,3 +97,55 @@ class TestContextImpl(unittest.TestCase):
 
     args, kwargs = consumer.acknowledge.call_args
     self.assertEqual(args[0], "test_message_id")
+
+class TestPropertiesForwarding(unittest.TestCase):
+
+  def _setup_mock_instance(self, forward_property):
+    function_details = Function_pb2.FunctionDetails()
+    function_details.sink.topic = "test_sink_topic"
+    function_details.sink.forwardSourceMessageProperty = forward_property
+    
+    mock_pulsar_client = Mock()
+    mock_producer = Mock()
+    mock_pulsar_client.create_producer.return_value = mock_producer
+    
+    instance = PythonInstance('test_instance', 'test_func', '1.0', function_details, 100, 30, 'user_code', mock_pulsar_client, Mock(), 'test_cluster', 'test_url', None)
+    instance.producer = mock_producer
+    instance.contextimpl = Mock()
+    instance.contextimpl.get_message_partition_index.return_value = None
+    instance.output_schema = "DEFAULT_SCHEMA"
+    instance.output_serde = Mock()
+    instance.output_serde.serialize.return_value = b'serialized_output'
+    instance.effectively_once = False
+    
+    return instance, mock_producer
+
+  def test_forwards_properties(self):
+    instance, mock_producer = self._setup_mock_instance(forward_property=True)
+    
+    mock_msg = Mock()
+    mock_msg.topic = "source-topic"
+    mock_msg.message.message_id().serialize.return_value = b'msg-id'
+    mock_msg.message.properties.return_value = {"custom-key": "custom-value"}
+    
+    instance.process_result("output-data", mock_msg)
+    
+    args, kwargs = mock_producer.send_async.call_args
+    self.assertIn("custom-key", kwargs['properties'])
+    self.assertEqual(kwargs['properties']["custom-key"], "custom-value")
+    self.assertIn("__pfn_input_topic__", kwargs['properties'])
+
+  def test_do_not_forward_properties(self):
+    instance, mock_producer = self._setup_mock_instance(forward_property=False)
+    
+    mock_msg = Mock()
+    mock_msg.topic = "source-topic"
+    mock_msg.message.message_id().serialize.return_value = b'msg-id'
+    mock_msg.message.properties.return_value = {"custom-key": "custom-value"}
+    
+    instance.process_result("output-data", mock_msg)
+    
+    args, kwargs = mock_producer.send_async.call_args
+    self.assertNotIn("custom-key", kwargs['properties'])
+    self.assertIn("__pfn_input_topic__", kwargs['properties'])
+