You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ue...@apache.org on 2026/08/04 21:19:46 UTC
(spark) branch branch-4.x updated: [SPARK-58015][INFRA][FOLLOWUP] Make dependency group names compliant with the PyPA specification
This is an automated email from the ASF dual-hosted git repository.
ueshin pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new b234ff268965 [SPARK-58015][INFRA][FOLLOWUP] Make dependency group names compliant with the PyPA specification
b234ff268965 is described below
commit b234ff2689651ddd4de09f72d9cfc7cecff6baa8
Author: Takuya UESHIN <ue...@happy-camper.st>
AuthorDate: Tue Aug 4 14:19:24 2026 -0700
[SPARK-58015][INFRA][FOLLOWUP] Make dependency group names compliant with the PyPA specification
### What changes were proposed in this pull request?
This is a follow-up of apache/spark#57093.
This PR renames internal dependency groups in `pyproject.toml` from names beginning with `_` to names beginning with `internal_`. It also updates all corresponding `include-group` references.
No package requirements or dependency group contents are changed.
### Why are the changes needed?
The PyPA Dependency Groups specification requires group names to be valid non-normalized names. Such names must begin and end with an ASCII letter or number, so the existing names beginning with `_` are invalid.
- https://packaging.python.org/en/latest/specifications/dependency-groups/#the-dependency-groups-table
> [dependency-groups] keys, sometimes also called “group names”, must be [valid non-normalized names](https://packaging.python.org/en/latest/specifications/name-normalization/#name-format). Tools which handle Dependency Groups MUST [normalize](https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization) these names before comparisons.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
The existing tests should pass. All dependency group names and `include-group` references were validated, including checks for missing references and duplicate normalized names.
Also manually:
```sh
$ uv pip install --dry-run --group pyproject.toml:dev
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-5)
Closes #57729 from ueshin/dependency_group.
Authored-by: Takuya UESHIN <ue...@happy-camper.st>
Signed-off-by: Takuya Ueshin <ue...@happy-camper.st>
(cherry picked from commit a38b12273e25940ea837aea2e8ca70a60fe772f2)
Signed-off-by: Takuya Ueshin <ue...@happy-camper.st>
---
pyproject.toml | 78 +++++++++++++++++++++++++++++-----------------------------
1 file changed, 39 insertions(+), 39 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index 790a3268daf8..d761759bb199 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -64,14 +64,14 @@ ignore = [
"python/pyspark/sql/tests/df_golden/scripts/**/*.py" = ["F821"]
[dependency-groups]
-_py4j = ["py4j>=0.10.9.9"]
+internal_py4j = ["py4j>=0.10.9.9"]
-_numpy = ["numpy>=1.23.2"]
-_pandas = ["pandas>=2.2.0,<3.0.0"]
-_protobuf = ["protobuf==6.33.5"]
-_pyarrow = ["pyarrow>=18.0.0"]
+internal_numpy = ["numpy>=1.23.2"]
+internal_pandas = ["pandas>=2.2.0,<3.0.0"]
+internal_protobuf = ["protobuf==6.33.5"]
+internal_pyarrow = ["pyarrow>=18.0.0"]
-_pyspark_devtool = [
+internal_pyspark_devtool = [
# Optional tools that are used by pyspark directly.
"memory-profiler>=0.61.0",
"flameprof==0.4",
@@ -80,10 +80,10 @@ _pyspark_devtool = [
]
connect = [
- {include-group = "_py4j"},
- {include-group = "_pandas"},
- {include-group = "_protobuf"},
- {include-group = "_pyarrow"},
+ {include-group = "internal_py4j"},
+ {include-group = "internal_pandas"},
+ {include-group = "internal_protobuf"},
+ {include-group = "internal_pyarrow"},
"grpcio>=1.76.0",
"grpcio-status>=1.76.0",
"googleapis-common-protos>=1.71.0",
@@ -92,15 +92,15 @@ connect = [
]
sql = [
- {include-group = "_py4j"},
- {include-group = "_pandas"},
- {include-group = "_pyarrow"},
+ {include-group = "internal_py4j"},
+ {include-group = "internal_pandas"},
+ {include-group = "internal_pyarrow"},
]
pandas_on_spark = [
- {include-group = "_py4j"},
- {include-group = "_pandas"},
- {include-group = "_pyarrow"},
+ {include-group = "internal_py4j"},
+ {include-group = "internal_pandas"},
+ {include-group = "internal_pyarrow"},
]
pandas_on_spark_extra = [
@@ -111,13 +111,13 @@ pandas_on_spark_extra = [
]
ml = [
- {include-group = "_py4j"},
- {include-group = "_numpy"},
+ {include-group = "internal_py4j"},
+ {include-group = "internal_numpy"},
]
mllib = [
- {include-group = "_py4j"},
- {include-group = "_numpy"},
+ {include-group = "internal_py4j"},
+ {include-group = "internal_numpy"},
]
ml_extra_base = [
@@ -145,7 +145,7 @@ pipelines = [
"pyyaml>=3.11",
]
-_lint = [
+internal_lint = [
"ruff==0.14.8",
"mypy==1.19.1",
"pytest",
@@ -159,7 +159,7 @@ _lint = [
"types-PyYAML"
]
-_test = [
+internal_test = [
# Dependency for running tests
"unittest-xml-reporting",
# Coverage tool
@@ -175,7 +175,7 @@ _test = [
"hypothesis>=6,<7",
]
-_docs = [
+internal_docs = [
"sphinx==8.2.3",
"sphinx-plotly-directive",
"sphinx-copybutton",
@@ -188,7 +188,7 @@ _docs = [
"mkdocs",
]
-_devtools = [
+internal_devtools = [
# Devtools that are used by pyspark developers.
"debugpy",
"viztracer",
@@ -198,11 +198,11 @@ _devtools = [
]
dev = [
- {include-group = "_pyspark_devtool"},
- {include-group = "_lint"},
- {include-group = "_test"},
- {include-group = "_docs"},
- {include-group = "_devtools"},
+ {include-group = "internal_pyspark_devtool"},
+ {include-group = "internal_lint"},
+ {include-group = "internal_test"},
+ {include-group = "internal_docs"},
+ {include-group = "internal_devtools"},
{include-group = "sql"},
{include-group = "connect"},
{include-group = "ml"},
@@ -213,7 +213,7 @@ dev = [
{include-group = "pipelines"},
]
-_ci_connect = [
+internal_ci_connect = [
{include-group = "connect"},
"grpcio==1.76.0",
"grpcio-status==1.76.0",
@@ -223,24 +223,24 @@ _ci_connect = [
]
ci_connect_standard = [
- {include-group = "_ci_connect"},
+ {include-group = "internal_ci_connect"},
"graphviz==0.20.3",
]
ci_connect_minimum = [
- {include-group = "_ci_connect"},
+ {include-group = "internal_ci_connect"},
"graphviz==0.20",
]
ci_classic_standard = [
- {include-group = "_pyspark_devtool"},
+ {include-group = "internal_pyspark_devtool"},
{include-group = "sql"},
{include-group = "pandas_on_spark"},
{include-group = "pandas_on_spark_extra"},
{include-group = "ml"},
# torch should be installed with ml_torch group
{include-group = "ml_extra_base"},
- {include-group = "_test"},
+ {include-group = "internal_test"},
"pyarrow>=23.0.0",
"pandas==2.3.3",
]
@@ -260,10 +260,10 @@ ci_classic_minimum = [
]
ci_lint = [
- {include-group = "_lint"},
+ {include-group = "internal_lint"},
{include-group = "ml_extra_base"},
{include-group = "pandas_on_spark_extra"},
- {include-group = "_ci_connect"},
+ {include-group = "internal_ci_connect"},
"pyarrow>=23.0.0",
"pandas==2.3.3",
"pandas-stubs==2.3.3.260113",
@@ -271,9 +271,9 @@ ci_lint = [
]
ci_docs = [
- {include-group = "_docs"},
- {include-group = "_ci_connect"},
- {include-group = "_numpy"},
+ {include-group = "internal_docs"},
+ {include-group = "internal_ci_connect"},
+ {include-group = "internal_numpy"},
# These are required for sphinx to introspect the code
"pyarrow>=23.0.0",
"pandas==2.3.3",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org