You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hamilton.apache.org by sk...@apache.org on 2026/09/25 17:22:30 UTC
(hamilton) branch main updated: Fix CI compatibility with current dependencies (#1723)
This is an automated email from the ASF dual-hosted git repository.
skrawcz pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hamilton.git
The following commit(s) were added to refs/heads/main by this push:
new 2206e09d Fix CI compatibility with current dependencies (#1723)
2206e09d is described below
commit 2206e09d49fa997d8609ebf3dff3966ab4a4de0f
Author: Stefan Krawczyk <sk...@apache.org>
AuthorDate: Fri Sep 25 10:22:25 2026 -0700
Fix CI compatibility with current dependencies (#1723)
* Fix CI compatibility with current dependencies
* Clarify pandas whitespace delimiter compatibility
* Address pandas compatibility review feedback
---
.github/workflows/hamilton-main.yml | 3 ---
hamilton/plugins/pandas_extensions.py | 29 +++++++++++++++++++-
pyproject.toml | 2 +-
tests/plugins/test_pandas_extensions.py | 31 +++++++++++++++++++++-
tests/test_base.py | 12 +++++++--
tests/test_default_data_quality.py | 2 +-
.../tests/test_db_methods/test_permissions.py | 4 +--
.../tests/test_lifecycle/test_run_tracking.py | 24 +++++------------
8 files changed, 77 insertions(+), 30 deletions(-)
diff --git a/.github/workflows/hamilton-main.yml b/.github/workflows/hamilton-main.yml
index aecae09b..fb055c5b 100644
--- a/.github/workflows/hamilton-main.yml
+++ b/.github/workflows/hamilton-main.yml
@@ -18,8 +18,6 @@ jobs:
static-checks:
name: "Static Checks"
runs-on: ubuntu-latest
- env:
- UV_PRERELEASE: "allow"
steps:
- name: Checkout repository
uses: actions/checkout@v4
@@ -52,7 +50,6 @@ jobs:
- '3.13'
- '3.14'
env:
- UV_PRERELEASE: "allow"
HAMILTON_TELEMETRY_ENABLED: false
steps:
diff --git a/hamilton/plugins/pandas_extensions.py b/hamilton/plugins/pandas_extensions.py
index 77cdc30f..72c569c1 100644
--- a/hamilton/plugins/pandas_extensions.py
+++ b/hamilton/plugins/pandas_extensions.py
@@ -1600,7 +1600,7 @@ class PandasTableReader(DataLoader):
verbose: bool | None = None
skip_blank_lines: bool = True
parse_dates: list[int | str] | dict[str, list[int | str]] | bool = False
- infer_datetime_format: bool = False
+ infer_datetime_format: bool | None = None
keep_date_col: bool | None = None
date_parser: Callable | None = None
date_format: str | str | None = None
@@ -1640,6 +1640,33 @@ class PandasTableReader(DataLoader):
# but we send it separately
del kwargs["filepath_or_buffer"]
+ # pandas deprecated delim_whitespace in 2.2 and removed it in 3.0. Translate the
+ # enabled case to its cross-version equivalent; explicit False is rejected on pandas 3.
+ if self.delim_whitespace:
+ if self.sep is not None or self.delimiter is not None:
+ raise ValueError("delim_whitespace cannot be combined with sep or delimiter")
+ kwargs["sep"] = r"\s+"
+ del kwargs["delim_whitespace"]
+
+ if Version(pd.__version__) >= Version("3.0"):
+ removed_parameters = {
+ "verbose": self.verbose,
+ "infer_datetime_format": self.infer_datetime_format,
+ "keep_date_col": self.keep_date_col,
+ "date_parser": self.date_parser,
+ "delim_whitespace": False if self.delim_whitespace is False else None,
+ }
+ requested_parameters = [
+ name for name, value in removed_parameters.items() if value is not None
+ ]
+ if requested_parameters:
+ raise ValueError(
+ f"pandas 3.0 removed these read_table parameters: "
+ f"{', '.join(requested_parameters)}"
+ )
+ for parameter in removed_parameters:
+ kwargs.pop(parameter, None)
+
return kwargs
def load_data(self, type_: type) -> tuple[DATAFRAME_TYPE, dict[str, Any]]:
diff --git a/pyproject.toml b/pyproject.toml
index 9ad9d332..b9b1279c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -133,7 +133,7 @@ test = [
"pytest-cov",
"PyYAML",
"scikit-learn",
- "sqlalchemy[asyncio]",
+ "sqlalchemy[asyncio]>=2.0,<2.1",
"typer>=0.16.0",
"xgboost; python_version < '3.14'",
"xlsx2csv", # for excel data loader
diff --git a/tests/plugins/test_pandas_extensions.py b/tests/plugins/test_pandas_extensions.py
index 784919d0..e4380ce7 100644
--- a/tests/plugins/test_pandas_extensions.py
+++ b/tests/plugins/test_pandas_extensions.py
@@ -129,7 +129,7 @@ def test_pandas_sql(df: pd.DataFrame, conn: str | sqlite3.Connection) -> None:
assert metadata1["sql_metadata"]["rows"] == 1
assert metadata2["sql_metadata"]["rows"] == 1
- assert metadata1["dataframe_metadata"]["datatypes"] == ["object"]
+ assert metadata1["dataframe_metadata"]["datatypes"] == [str(df["foo"].dtype)]
if hasattr(conn, "close"):
conn.close()
@@ -321,6 +321,35 @@ def test_pandas_table_reader(tmp_path: pathlib.Path) -> None:
]
+def test_pandas_table_reader_translates_delim_whitespace_for_pandas_3(tmp_path) -> None:
+ path = tmp_path / "table.txt"
+ path.write_text("name value\nfoo 1\n")
+
+ reader = PandasTableReader(filepath_or_buffer=path, delim_whitespace=True)
+ df, _ = reader.load_data(pd.DataFrame)
+
+ assert df.to_dict(orient="records") == [{"name": "foo", "value": 1}]
+
+
+@pytest.mark.parametrize(
+ ("parameter", "value"),
+ [
+ ("verbose", True),
+ ("infer_datetime_format", True),
+ ("infer_datetime_format", False),
+ ("delim_whitespace", False),
+ ],
+)
+def test_pandas_table_reader_rejects_removed_parameter_on_pandas_3(parameter, value) -> None:
+ reader = PandasTableReader(filepath_or_buffer="unused", **{parameter: value})
+
+ if int(pd.__version__.split(".", maxsplit=1)[0]) >= 3:
+ with pytest.raises(ValueError, match=f"pandas 3.0 removed.*{parameter}"):
+ reader._get_loading_kwargs()
+ else:
+ assert reader._get_loading_kwargs()[parameter] is value
+
+
@pytest.mark.skipif(sys.version_info >= (3, 14), reason="pyreadstat not available on Python 3.14")
def test_pandas_spss_reader(tmp_path: pathlib.Path) -> None:
import pyreadstat
diff --git a/tests/test_base.py b/tests/test_base.py
index 6c9d6406..617cb838 100644
--- a/tests/test_base.py
+++ b/tests/test_base.py
@@ -311,7 +311,11 @@ PD_VERSION = version.parse(pd.__version__)
(
{
"b": pd.Series(
- [3, 4, 5], index=pd.DatetimeIndex(["2022-01", "2022-02", "2022-03"], freq="MS")
+ [3, 4, 5],
+ index=pd.DatetimeIndex(
+ np.array(["2022-01", "2022-02", "2022-03"], dtype="datetime64[ns]"),
+ freq="MS",
+ ),
)
},
(
@@ -326,7 +330,11 @@ PD_VERSION = version.parse(pd.__version__)
"a": pd.Series([1, 2, 3]),
"b": 1,
"c": pd.Series(
- [3, 4, 5], index=pd.DatetimeIndex(["2022-01", "2022-02", "2022-03"], freq="MS")
+ [3, 4, 5],
+ index=pd.DatetimeIndex(
+ np.array(["2022-01", "2022-02", "2022-03"], dtype="datetime64[ns]"),
+ freq="MS",
+ ),
),
},
(
diff --git a/tests/test_default_data_quality.py b/tests/test_default_data_quality.py
index 924442e1..64a6097a 100644
--- a/tests/test_default_data_quality.py
+++ b/tests/test_default_data_quality.py
@@ -198,7 +198,7 @@ def test_resolve_default_validators_error(output_type, kwargs, importance):
(
default_validators.DataTypeValidatorPandasSeries,
numpy.dtype("object"),
- pd.Series(["hello", "goodbye"]),
+ pd.Series(["hello", "goodbye"], dtype="object"),
True,
),
(
diff --git a/ui/backend/server/tests/test_db_methods/test_permissions.py b/ui/backend/server/tests/test_db_methods/test_permissions.py
index 40b81349..b029a7e2 100644
--- a/ui/backend/server/tests/test_db_methods/test_permissions.py
+++ b/ui/backend/server/tests/test_db_methods/test_permissions.py
@@ -657,9 +657,7 @@ async def test_visible_project_ids_for_user_only_returns_member_projects(db):
"user_individual@no_team.com", "write"
)
# Project B is accessible by user B only.
- project_b, user_b = await _setup_project_accessible_only_by(
- "user_1_team_1@team1.com", "write"
- )
+ project_b, user_b = await _setup_project_accessible_only_by("user_1_team_1@team1.com", "write")
request_a = await _get_authenticated_request("user_individual@no_team.com")
request_b = await _get_authenticated_request("user_1_team_1@team1.com")
user_a_obj, teams_a = request_a.auth
diff --git a/ui/backend/server/tests/test_lifecycle/test_run_tracking.py b/ui/backend/server/tests/test_lifecycle/test_run_tracking.py
index 6d76fc46..7534ca12 100644
--- a/ui/backend/server/tests/test_lifecycle/test_run_tracking.py
+++ b/ui/backend/server/tests/test_lifecycle/test_run_tracking.py
@@ -342,17 +342,11 @@ async def test_get_latest_dag_runs_without_params_excludes_other_users_runs(
dag_template_a, _ = await _setup_dag_template(async_client, user_a)
dag_template_b, _ = await _setup_dag_template(async_client, user_b)
- run_a_id = await _create_dag_run_via_api(
- async_client, user_a, dag_template_a, {"owner": "a"}
- )
- run_b_id = await _create_dag_run_via_api(
- async_client, user_b, dag_template_b, {"owner": "b"}
- )
+ run_a_id = await _create_dag_run_via_api(async_client, user_a, dag_template_a, {"owner": "a"})
+ run_b_id = await _create_dag_run_via_api(async_client, user_b, dag_template_b, {"owner": "b"})
# User A asks for latest runs with no filter -- must not see user B's run.
- response = await async_client.get(
- "/api/v1/dag_runs/latest/", headers={"test_username": user_a}
- )
+ response = await async_client.get("/api/v1/dag_runs/latest/", headers={"test_username": user_a})
assert response.status_code == 200, response.content
returned_ids = {item["id"] for item in response.json()}
assert run_a_id in returned_ids
@@ -390,17 +384,13 @@ async def test_get_latest_dag_runs_with_other_users_project_id_is_rejected(
@pytest.mark.asyncio
-async def test_get_latest_dag_runs_with_own_project_id_still_works(
- async_client: AsyncClient, db
-):
+async def test_get_latest_dag_runs_with_own_project_id_still_works(async_client: AsyncClient, db):
"""The legitimate case -- user asking for runs in a project they own --
must keep returning their runs after the visibility filter is applied.
"""
user_a = "user_individual@no_team.com"
dag_template_a, _ = await _setup_dag_template(async_client, user_a)
- run_id = await _create_dag_run_via_api(
- async_client, user_a, dag_template_a, {"owner": "a"}
- )
+ run_id = await _create_dag_run_via_api(async_client, user_a, dag_template_a, {"owner": "a"})
# By dag_template_id.
response = await async_client.get(
@@ -415,9 +405,7 @@ async def test_get_latest_dag_runs_with_own_project_id_still_works(
project_a_id, *_ = await _setup_sample_project(async_client, user_a)
# New template in this project, new run.
dag_template_a2, _ = await _setup_dag_template(async_client, user_a)
- run_id_2 = await _create_dag_run_via_api(
- async_client, user_a, dag_template_a2, {"owner": "a2"}
- )
+ run_id_2 = await _create_dag_run_via_api(async_client, user_a, dag_template_a2, {"owner": "a2"})
response = await async_client.get(
f"/api/v1/dag_runs/latest/?{urlencode({'project_id': project_a_id})}",
headers={"test_username": user_a},