You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by nc...@apache.org on 2026/08/19 15:46:47 UTC
(spark) branch branch-4.x updated: [SPARK-58664][PYTHON] Factor out repeated paths in PySpark Classic's setup.py
This is an automated email from the ASF dual-hosted git repository.
nchammas pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 2d7e1ca4827c [SPARK-58664][PYTHON] Factor out repeated paths in PySpark Classic's setup.py
2d7e1ca4827c is described below
commit 2d7e1ca4827c550fcb0579ab20bc33625592e439
Author: Nicholas Chammas <ni...@gmail.com>
AuthorDate: Wed Aug 19 11:46:21 2026 -0400
[SPARK-58664][PYTHON] Factor out repeated paths in PySpark Classic's setup.py
### What changes were proposed in this pull request?
Factor out repeated paths into a single list of install paths that is used throughout the PySpark Classic install script.
During cleanup, change the check to look at the actual filesystem object -- is it a symlink or not? -- rather than repeat the general check on OS capabilities.
I preserved the order in which the paths are installed and then cleaned up.
### Why are the changes needed?
General code hygiene. We shouldn't have to manually touch multiple blocks of code that are supposed to always be in sync.
Checking the file's type directly during cleanup is also more robust than checking a general capability.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
```sh
dev/make-distribution.sh --pip
dev/run-pip-tests
```
### Was this patch authored or co-authored using generative AI tooling?
I wrote this with assistance from GitHub Copilot.
Closes #57871 from nchammas/pyspark-classic-install-paths.
Authored-by: Nicholas Chammas <ni...@gmail.com>
Signed-off-by: Nicholas Chammas <ni...@gmail.com>
(cherry picked from commit 9d2f0b382b27c2c2dd64f0f92ecc4f5f6bb8e9a2)
Signed-off-by: Nicholas Chammas <ni...@gmail.com>
---
python/packaging/classic/setup.py | 68 ++++++++++++++++++---------------------
1 file changed, 31 insertions(+), 37 deletions(-)
diff --git a/python/packaging/classic/setup.py b/python/packaging/classic/setup.py
index 23c20d2c54b7..c711edcd59b0 100755
--- a/python/packaging/classic/setup.py
+++ b/python/packaging/classic/setup.py
@@ -25,6 +25,7 @@ from setuptools import setup
from setuptools.command.install import install
from shutil import copyfile, copytree, rmtree
from pathlib import Path
+from dataclasses import dataclass
if (
# When we package, the parent directory 'classic' dir
@@ -100,10 +101,6 @@ elif len(JARS_PATH) == 0 and not os.path.exists(TEMP_PATH):
print(incorrect_invocation_message, file=sys.stderr)
sys.exit(-1)
-EXAMPLES_PATH = os.path.join(SPARK_HOME, "examples/src/main/python")
-SCRIPTS_PATH = os.path.join(SPARK_HOME, "bin")
-USER_SCRIPTS_PATH = os.path.join(SPARK_HOME, "sbin")
-DATA_PATH = os.path.join(SPARK_HOME, "data")
# The classic PySpark package bundles the assembly jars, so it ships the binary
# license texts (licenses-binary), which enumerate those jars' licenses, mirroring
# the binary distribution. The connect/client packages bundle no jars.
@@ -113,12 +110,26 @@ if not os.path.isdir(LICENSES_PATH):
# were already copied to licenses/ (see dev/make-distribution.sh).
LICENSES_PATH = os.path.join(SPARK_HOME, "licenses")
-SCRIPTS_TARGET = os.path.join(TEMP_PATH, "bin")
-USER_SCRIPTS_TARGET = os.path.join(TEMP_PATH, "sbin")
+
+@dataclass(frozen=True)
+class InstallPath:
+ source: str
+ target: str
+
+
JARS_TARGET = os.path.join(TEMP_PATH, "jars")
-EXAMPLES_TARGET = os.path.join(TEMP_PATH, "examples")
-DATA_TARGET = os.path.join(TEMP_PATH, "data")
-LICENSES_TARGET = os.path.join(TEMP_PATH, "licenses")
+SCRIPTS_TARGET = os.path.join(TEMP_PATH, "bin")
+INSTALL_PATHS = [
+ InstallPath(JARS_PATH, JARS_TARGET),
+ InstallPath(os.path.join(SPARK_HOME, "bin"), SCRIPTS_TARGET),
+ InstallPath(os.path.join(SPARK_HOME, "sbin"), os.path.join(TEMP_PATH, "sbin")),
+ InstallPath(
+ os.path.join(SPARK_HOME, "examples/src/main/python"),
+ os.path.join(TEMP_PATH, "examples"),
+ ),
+ InstallPath(os.path.join(SPARK_HOME, "data"), os.path.join(TEMP_PATH, "data")),
+ InstallPath(LICENSES_PATH, os.path.join(TEMP_PATH, "licenses")),
+]
# Check and see if we are under the spark path in which case we need to build the symlink farm.
# This is important because we only want to build the symlink farm while under Spark otherwise we
@@ -217,24 +228,16 @@ try:
copyfile("packaging/classic/setup.py", "setup.py")
copyfile("packaging/classic/setup.cfg", "setup.cfg")
- # Construct the symlink farm - this is nein_sparkcessary since we can't refer to
+ # Construct the symlink farm - this is necessary since we can't refer to
# the path above the package root and we need to copy the jars and scripts which
# are up above the python root.
if _supports_symlinks():
- os.symlink(JARS_PATH, JARS_TARGET)
- os.symlink(SCRIPTS_PATH, SCRIPTS_TARGET)
- os.symlink(USER_SCRIPTS_PATH, USER_SCRIPTS_TARGET)
- os.symlink(EXAMPLES_PATH, EXAMPLES_TARGET)
- os.symlink(DATA_PATH, DATA_TARGET)
- os.symlink(LICENSES_PATH, LICENSES_TARGET)
+ for path in INSTALL_PATHS:
+ os.symlink(path.source, path.target)
else:
# For windows fall back to the slower copytree
- copytree(JARS_PATH, JARS_TARGET)
- copytree(SCRIPTS_PATH, SCRIPTS_TARGET)
- copytree(USER_SCRIPTS_PATH, USER_SCRIPTS_TARGET)
- copytree(EXAMPLES_PATH, EXAMPLES_TARGET)
- copytree(DATA_PATH, DATA_TARGET)
- copytree(LICENSES_PATH, LICENSES_TARGET)
+ for path in INSTALL_PATHS:
+ copytree(path.source, path.target)
else:
# If we are not inside of SPARK_HOME verify we have the required symlink farm
if not os.path.exists(JARS_TARGET):
@@ -411,19 +414,10 @@ finally:
if in_spark:
os.remove("setup.py")
os.remove("setup.cfg")
- # Depending on cleaning up the symlink farm or copied version
- if _supports_symlinks():
- os.remove(os.path.join(TEMP_PATH, "jars"))
- os.remove(os.path.join(TEMP_PATH, "bin"))
- os.remove(os.path.join(TEMP_PATH, "sbin"))
- os.remove(os.path.join(TEMP_PATH, "examples"))
- os.remove(os.path.join(TEMP_PATH, "data"))
- os.remove(os.path.join(TEMP_PATH, "licenses"))
- else:
- rmtree(os.path.join(TEMP_PATH, "jars"))
- rmtree(os.path.join(TEMP_PATH, "bin"))
- rmtree(os.path.join(TEMP_PATH, "sbin"))
- rmtree(os.path.join(TEMP_PATH, "examples"))
- rmtree(os.path.join(TEMP_PATH, "data"))
- rmtree(os.path.join(TEMP_PATH, "licenses"))
+ for path in INSTALL_PATHS:
+ if os.path.islink(path.target):
+ # Remove the link and not the real source trees under SPARK_HOME.
+ os.remove(path.target)
+ else:
+ rmtree(path.target)
os.rmdir(TEMP_PATH)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org