(iceberg-python) branch main updated: Correct partition evaluator thread safety (#115)

[fo...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

fokko pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-python.git


The following commit(s) were added to refs/heads/main by this push:
     new b40ee83  Correct partition evaluator thread safety (#115)
b40ee83 is described below

commit b40ee83a4e3a7d4cead99c0fcfcbe2c1bd43007e
Author: Sean Kelly <10...@users.noreply.github.com>
AuthorDate: Thu Nov 2 08:25:26 2023 -0400

    Correct partition evaluator thread safety (#115)
---
 pyiceberg/table/__init__.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py
index ae35b34..21a3b7f 100644
--- a/pyiceberg/table/__init__.py
+++ b/pyiceberg/table/__init__.py
@@ -828,8 +828,12 @@ class DataScan(TableScan):
         partition_schema = Schema(*partition_type.fields)
         partition_expr = self.partition_filters[spec_id]
 
-        evaluator = visitors.expression_evaluator(partition_schema, partition_expr, self.case_sensitive)
-        return lambda data_file: evaluator(data_file.partition)
+        # The lambda created here is run in multiple threads.
+        # So we avoid creating _EvaluatorExpression methods bound to a single
+        # shared instance across multiple threads.
+        return lambda data_file: visitors.expression_evaluator(partition_schema, partition_expr, self.case_sensitive)(
+            data_file.partition
+        )
 
     def _check_sequence_number(self, min_data_sequence_number: int, manifest: ManifestFile) -> bool:
         """Ensure that no manifests are loaded that contain deletes that are older than the data.