You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by pa...@apache.org on 2026/07/15 14:16:51 UTC
(sedona-db) branch main updated: fix(python/seconadb): Serialize pyogrio open/close (#1051)
This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/sedona-db.git
The following commit(s) were added to refs/heads/main by this push:
new 1222c26fd fix(python/seconadb): Serialize pyogrio open/close (#1051)
1222c26fd is described below
commit 1222c26fd68d7779b7e6898bc5e1a3a0393086e9
Author: James Willis <ja...@users.noreply.github.com>
AuthorDate: Wed Jul 15 07:16:46 2026 -0700
fix(python/seconadb): Serialize pyogrio open/close (#1051)
---
.github/workflows/python-wheels.yml | 12 ++++++++----
python/sedonadb/python/sedonadb/datasource.py | 18 ++++++++++++++++--
2 files changed, 24 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/python-wheels.yml b/.github/workflows/python-wheels.yml
index bfbcfa97f..274d31b12 100644
--- a/.github/workflows/python-wheels.yml
+++ b/.github/workflows/python-wheels.yml
@@ -46,6 +46,7 @@ env:
jobs:
windows-x86_64:
runs-on: windows-2022
+ timeout-minutes: 150
steps:
- uses: actions/checkout@v7
@@ -75,7 +76,7 @@ jobs:
CMAKE_TOOLCHAIN_FILE: ${{ github.workspace }}/vcpkg/scripts/buildsystems/vcpkg.cmake
CIBW_BUILD: "*-win_amd64"
CIBW_TEST_REQUIRES: pytest adbc_driver_manager geoarrow-pyarrow geopandas
- CIBW_TEST_COMMAND: pytest {package}/tests -vv
+ CIBW_TEST_COMMAND: pytest {package}/tests -vv -o faulthandler_timeout=600
- uses: actions/upload-artifact@v7
with:
@@ -84,6 +85,7 @@ jobs:
macOS-arm64:
runs-on: macOS-latest
+ timeout-minutes: 150
steps:
- uses: actions/checkout@v7
@@ -110,7 +112,7 @@ jobs:
env:
VCPKG_ROOT: ${{ github.workspace }}/vcpkg
CIBW_TEST_REQUIRES: pytest adbc_driver_manager geoarrow-pyarrow geopandas duckdb
- CIBW_TEST_COMMAND: pytest {package}/tests -vv
+ CIBW_TEST_COMMAND: pytest {package}/tests -vv -o faulthandler_timeout=600
- uses: actions/upload-artifact@v7
with:
@@ -119,6 +121,7 @@ jobs:
macOS-amd64:
runs-on: macos-15-intel
+ timeout-minutes: 150
steps:
- uses: actions/checkout@v7
@@ -145,7 +148,7 @@ jobs:
env:
VCPKG_ROOT: ${{ github.workspace }}/vcpkg
CIBW_TEST_REQUIRES: pytest adbc_driver_manager geoarrow-pyarrow geopandas duckdb
- CIBW_TEST_COMMAND: pytest {package}/tests -vv
+ CIBW_TEST_COMMAND: pytest {package}/tests -vv -o faulthandler_timeout=600
SEDONADB_MACOS_ARCH: x86_64
- uses: actions/upload-artifact@v7
@@ -156,6 +159,7 @@ jobs:
wheels-linux:
name: ${{ matrix.config.label }}
runs-on: ${{ matrix.config.os }}
+ timeout-minutes: 150
strategy:
matrix:
config:
@@ -177,7 +181,7 @@ jobs:
env:
CIBW_SKIP: "*musllinux*"
CIBW_TEST_REQUIRES: pytest adbc_driver_manager geoarrow-pyarrow geopandas duckdb
- CIBW_TEST_COMMAND: pytest {package}/tests -vv
+ CIBW_TEST_COMMAND: pytest {package}/tests -vv -o faulthandler_timeout=600
- uses: actions/upload-artifact@v7
with:
diff --git a/python/sedonadb/python/sedonadb/datasource.py b/python/sedonadb/python/sedonadb/datasource.py
index 7156d46f0..2a2eaec25 100644
--- a/python/sedonadb/python/sedonadb/datasource.py
+++ b/python/sedonadb/python/sedonadb/datasource.py
@@ -16,6 +16,7 @@
# under the License.
import sys
+import threading
from typing import Any, Mapping
from sedonadb._lib import PyExternalFormat, PyProjectedRecordBatchReader
@@ -202,13 +203,26 @@ class PyogrioReaderShelter:
is no longer required).
"""
+ # Serializes dataset open/close. pyogrio releases the GIL around the
+ # GDAL open, so a multi-file scan otherwise opens datasets from several
+ # threads at once, which can crash the process (SIGABRT) depending on
+ # the GDAL build and driver. Reentrant because a garbage-collected
+ # shelter's __del__ may run on a thread that is already holding the
+ # lock in __init__.
+ _open_close_lock = threading.RLock()
+
def __init__(self, inner, output_names=None):
self._inner = inner
self._output_names = output_names
- self._meta, self._reader = self._inner.__enter__()
+ self._entered = False
+ with PyogrioReaderShelter._open_close_lock:
+ self._meta, self._reader = self._inner.__enter__()
+ self._entered = True
def __del__(self):
- self._inner.__exit__(None, None, None)
+ if self._entered:
+ with PyogrioReaderShelter._open_close_lock:
+ self._inner.__exit__(None, None, None)
def __arrow_c_stream__(self, requested_schema=None):
if self._output_names is None: