You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by pa...@apache.org on 2026/07/15 14:16:51 UTC

(sedona-db) branch main updated: fix(python/seconadb): Serialize pyogrio open/close (#1051)

This is an automated email from the ASF dual-hosted git repository.

paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/sedona-db.git


The following commit(s) were added to refs/heads/main by this push:
     new 1222c26fd fix(python/seconadb): Serialize pyogrio open/close (#1051)
1222c26fd is described below

commit 1222c26fd68d7779b7e6898bc5e1a3a0393086e9
Author: James Willis <ja...@users.noreply.github.com>
AuthorDate: Wed Jul 15 07:16:46 2026 -0700

    fix(python/seconadb): Serialize pyogrio open/close (#1051)
---
 .github/workflows/python-wheels.yml           | 12 ++++++++----
 python/sedonadb/python/sedonadb/datasource.py | 18 ++++++++++++++++--
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml
index bfbcfa97f..274d31b12 100644
--- a/.github/workflows/python-wheels.yml
+++ b/.github/workflows/python-wheels.yml
@@ -46,6 +46,7 @@ env:
 jobs:
   windows-x86_64:
     runs-on: windows-2022
+    timeout-minutes: 150
 
     steps:
       - uses: actions/checkout@v7
@@ -75,7 +76,7 @@ jobs:
           CMAKE_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
           CIBW_BUILD: "*-win_amd64"
           CIBW_TEST_REQUIRES: pytest adbc_driver_manager geoarrow-pyarrow geopandas
-          CIBW_TEST_COMMAND: pytest {package}/tests -vv
+          CIBW_TEST_COMMAND: pytest {package}/tests -vv -o faulthandler_timeout=600
 
       - uses: actions/upload-artifact@v7
         with:
@@ -84,6 +85,7 @@ jobs:
 
   macOS-arm64:
     runs-on: macOS-latest
+    timeout-minutes: 150
 
     steps:
       - uses: actions/checkout@v7
@@ -110,7 +112,7 @@ jobs:
         env:
           VCPKG_ROOT: ${{ github.workspace }}/vcpkg
           CIBW_TEST_REQUIRES: pytest adbc_driver_manager geoarrow-pyarrow geopandas duckdb
-          CIBW_TEST_COMMAND: pytest {package}/tests -vv
+          CIBW_TEST_COMMAND: pytest {package}/tests -vv -o faulthandler_timeout=600
 
       - uses: actions/upload-artifact@v7
         with:
@@ -119,6 +121,7 @@ jobs:
 
   macOS-amd64:
     runs-on: macos-15-intel
+    timeout-minutes: 150
 
     steps:
       - uses: actions/checkout@v7
@@ -145,7 +148,7 @@ jobs:
         env:
           VCPKG_ROOT: ${{ github.workspace }}/vcpkg
           CIBW_TEST_REQUIRES: pytest adbc_driver_manager geoarrow-pyarrow geopandas duckdb
-          CIBW_TEST_COMMAND: pytest {package}/tests -vv
+          CIBW_TEST_COMMAND: pytest {package}/tests -vv -o faulthandler_timeout=600
           SEDONADB_MACOS_ARCH: x86_64
 
       - uses: actions/upload-artifact@v7
@@ -156,6 +159,7 @@ jobs:
   wheels-linux:
     name: ${{ matrix.config.label }}
     runs-on: ${{ matrix.config.os }}
+    timeout-minutes: 150
     strategy:
       matrix:
         config:
@@ -177,7 +181,7 @@ jobs:
         env:
           CIBW_SKIP: "*musllinux*"
           CIBW_TEST_REQUIRES: pytest adbc_driver_manager geoarrow-pyarrow geopandas duckdb
-          CIBW_TEST_COMMAND: pytest {package}/tests -vv
+          CIBW_TEST_COMMAND: pytest {package}/tests -vv -o faulthandler_timeout=600
 
       - uses: actions/upload-artifact@v7
         with:
diff --git a/python/sedonadb/python/sedonadb/datasource.py b/python/sedonadb/python/sedonadb/datasource.py
index 7156d46f0..2a2eaec25 100644
--- a/python/sedonadb/python/sedonadb/datasource.py
+++ b/python/sedonadb/python/sedonadb/datasource.py
@@ -16,6 +16,7 @@
 # under the License.
 
 import sys
+import threading
 from typing import Any, Mapping
 
 from sedonadb._lib import PyExternalFormat, PyProjectedRecordBatchReader
@@ -202,13 +203,26 @@ class PyogrioReaderShelter:
     is no longer required).
     """
 
+    # Serializes dataset open/close. pyogrio releases the GIL around the
+    # GDAL open, so a multi-file scan otherwise opens datasets from several
+    # threads at once, which can crash the process (SIGABRT) depending on
+    # the GDAL build and driver. Reentrant because a garbage-collected
+    # shelter's __del__ may run on a thread that is already holding the
+    # lock in __init__.
+    _open_close_lock = threading.RLock()
+
     def __init__(self, inner, output_names=None):
         self._inner = inner
         self._output_names = output_names
-        self._meta, self._reader = self._inner.__enter__()
+        self._entered = False
+        with PyogrioReaderShelter._open_close_lock:
+            self._meta, self._reader = self._inner.__enter__()
+            self._entered = True
 
     def __del__(self):
-        self._inner.__exit__(None, None, None)
+        if self._entered:
+            with PyogrioReaderShelter._open_close_lock:
+                self._inner.__exit__(None, None, None)
 
     def __arrow_c_stream__(self, requested_schema=None):
         if self._output_names is None: