You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ho...@apache.org on 2026/09/08 05:30:43 UTC
(spark) branch branch-3.5 updated: [PYTHON][MINOR] Harden Python release installation
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new bb76812cce10 [PYTHON][MINOR] Harden Python release installation
bb76812cce10 is described below
commit bb76812cce10c0c7663572a5c1bfe6bd9ec0c817
Author: Holden Karau <ho...@snowflake.com>
AuthorDate: Mon Aug 31 18:31:40 2026 +0000
[PYTHON][MINOR] Harden Python release installation
Co-authored-by: Cursor <cu...@cursor.com>
Co-Authored-By: Holden Karau <ho...@pigscanfly.ca>
---
python/pyspark/install.py | 106 +++++++++++++++++++++++++++--
python/pyspark/tests/test_install_spark.py | 35 ++++++++++
2 files changed, 137 insertions(+), 4 deletions(-)
diff --git a/python/pyspark/install.py b/python/pyspark/install.py
index dd69f5a53f78..3fd17ef0194e 100644
--- a/python/pyspark/install.py
+++ b/python/pyspark/install.py
@@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+import hashlib
import os
import re
import tarfile
@@ -30,6 +31,14 @@ SUPPORTED_HADOOP_VERSIONS = ["hadoop3", "without-hadoop"]
SUPPORTED_HIVE_VERSIONS = ["hive2.3"]
UNSUPPORTED_COMBINATIONS = [] # type: ignore
+# Official ASF-controlled hosts serving release checksums. Integrity metadata
+# must come from these origins, never from a (community) mirror, so that a
+# tampered tarball served by a mirror fails verification.
+CHECKSUM_ORIGINS = [
+ "https://downloads.apache.org/spark",
+ "https://archive.apache.org/dist/spark",
+]
+
def checked_package_name(spark_version, hadoop_version, hive_version):
"""
@@ -125,10 +134,12 @@ def install_spark(dest, spark_version, hadoop_version, hive_version):
package_name = checked_package_name(spark_version, hadoop_version, hive_version)
package_local_path = os.path.join(dest, "%s.tgz" % package_name)
- if "PYSPARK_RELEASE_MIRROR" in os.environ:
+ explicit_mirror = "PYSPARK_RELEASE_MIRROR" in os.environ
+ if explicit_mirror:
sites = [os.environ["PYSPARK_RELEASE_MIRROR"]]
else:
- sites = get_preferred_mirrors()
+ # Plain-http mirrors are trivially tampered with in transit.
+ sites = [site for site in get_preferred_mirrors() if site.startswith("https://")]
print("Trying to download Spark %s from [%s]" % (spark_version, ", ".join(sites)))
pretty_pkg_name = "%s for Hadoop %s" % (
@@ -136,6 +147,27 @@ def install_spark(dest, spark_version, hadoop_version, hive_version):
"Free build" if hadoop_version == "without" else hadoop_version,
)
+ # Verify every downloaded tarball against the SHA-512 digest published on
+ # the official Apache hosts. Setting PYSPARK_UNVERIFIED_DOWNLOAD=1 opts out
+ # (for example for air-gapped setups with no route to downloads.apache.org).
+ expected_digest = None
+ if os.environ.get("PYSPARK_UNVERIFIED_DOWNLOAD", "") != "1":
+ try:
+ expected_digest = _fetch_official_checksum(spark_version, package_name)
+ except Exception:
+ if explicit_mirror:
+ # An explicitly configured mirror (e.g. an internal mirror or a
+ # release-candidate staging area) may host packages that have no
+ # published official checksum; defer to the user's explicit
+ # trust in that mirror.
+ print(
+ "Warning: no official checksum is available for %s; skipping "
+ "verification for the explicitly configured mirror." % package_name
+ )
+ else:
+ raise
+
+ last_error = None
for site in sites:
os.makedirs(dest, exist_ok=True)
url = "%s/spark/%s/%s.tgz" % (site, spark_version, package_name)
@@ -145,11 +177,16 @@ def install_spark(dest, spark_version, hadoop_version, hive_version):
print("Downloading %s from:\n- %s" % (pretty_pkg_name, url))
download_to_file(urllib.request.urlopen(url), package_local_path)
+ if expected_digest is not None:
+ print("Verifying the SHA-512 checksum of %s" % package_local_path)
+ _verify_checksum(package_local_path, expected_digest)
+
print("Installing to %s" % dest)
tar = tarfile.open(package_local_path, "r:gz")
_extract_tar(tar, package_name, dest)
return
- except Exception:
+ except Exception as e:
+ last_error = e
print("Failed to download %s from %s:" % (pretty_pkg_name, url))
traceback.print_exc()
rmtree(dest, ignore_errors=True)
@@ -158,7 +195,7 @@ def install_spark(dest, spark_version, hadoop_version, hive_version):
tar.close()
if os.path.exists(package_local_path):
os.remove(package_local_path)
- raise IOError("Unable to download %s." % pretty_pkg_name)
+ raise IOError("Unable to download %s." % pretty_pkg_name) from last_error
def _extract_tar(tar: tarfile.TarFile, package_name: str, dest: str) -> None:
@@ -191,6 +228,67 @@ def _extract_tar(tar: tarfile.TarFile, package_name: str, dest: str) -> None:
tar.extract(member, dest)
+def _parse_sha512_checksum(content: str, package_file_name: str) -> str:
+ """
+ Parse the contents of an Apache release ``.sha512`` file into a lowercase
+ hex digest. Both formats published for Spark releases are accepted:
+ ``sha512sum`` style (``<hex digest> <file name>``) and GPG
+ ``--print-md`` style (``<file name>: ABCD 1234 ...``).
+ """
+ digest = (
+ content.replace(package_file_name, "")
+ .replace(":", "")
+ .replace("\n", "")
+ .replace("\t", "")
+ .replace(" ", "")
+ .strip()
+ .lower()
+ )
+ if not re.match("^[0-9a-f]{128}$", digest):
+ raise ValueError(
+ "Could not parse a SHA-512 digest for %s from the checksum file." % package_file_name
+ )
+ return digest
+
+
+def _fetch_official_checksum(spark_version: str, package_name: str) -> str:
+ """
+ Fetch the SHA-512 checksum for the given package from the official Apache
+ distribution hosts (never from a mirror), so that a tarball served by an
+ untrusted mirror can be verified against it.
+ """
+ file_name = "%s.tgz" % package_name
+ last_error = None
+ for origin in CHECKSUM_ORIGINS:
+ url = "%s/%s/%s.sha512" % (origin, spark_version, file_name)
+ try:
+ response = urllib.request.urlopen(url, timeout=60)
+ return _parse_sha512_checksum(response.read().decode("utf-8"), file_name)
+ except Exception as e:
+ last_error = e
+ raise OSError(
+ "Unable to fetch the checksum for %s: %s. Set PYSPARK_UNVERIFIED_DOWNLOAD=1 "
+ "to skip verification." % (file_name, last_error)
+ )
+
+
+def _verify_checksum(path: str, expected_digest: str) -> None:
+ """
+ Verify that the SHA-512 digest of the file at ``path`` matches
+ ``expected_digest``, raising ``ValueError`` otherwise.
+ """
+ sha512 = hashlib.sha512()
+ with open(path, mode="rb") as f:
+ for chunk in iter(lambda: f.read(1024 * 1024), b""):
+ sha512.update(chunk)
+ actual_digest = sha512.hexdigest()
+ if actual_digest != expected_digest:
+ raise ValueError(
+ "SHA-512 mismatch for %s: expected %s but got %s. The downloaded file is "
+ "corrupt or has been tampered with." % (path, expected_digest, actual_digest)
+ )
+
+
def get_preferred_mirrors():
mirror_urls = []
for _ in range(3):
diff --git a/python/pyspark/tests/test_install_spark.py b/python/pyspark/tests/test_install_spark.py
index aebec186df8b..5e59022d42c3 100644
--- a/python/pyspark/tests/test_install_spark.py
+++ b/python/pyspark/tests/test_install_spark.py
@@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+import hashlib
import io
import os
import tarfile
@@ -26,6 +27,8 @@ from pyspark.install import (
DEFAULT_HADOOP,
DEFAULT_HIVE,
UNSUPPORTED_COMBINATIONS,
+ _parse_sha512_checksum,
+ _verify_checksum,
checked_versions,
checked_package_name,
)
@@ -83,6 +86,38 @@ class SparkInstallationTestCase(unittest.TestCase):
_extract_tar(tar, package_name, evil_dest)
self.assertEqual([], os.listdir(evil_dest))
+ def test_parse_sha512_checksum(self):
+ file_name = "spark-3.5.0-bin-hadoop3.tgz"
+ digest = "ab" * 64
+
+ # sha512sum style: "<digest> <file name>"
+ self.assertEqual(
+ digest, _parse_sha512_checksum("%s %s\n" % (digest, file_name), file_name)
+ )
+
+ # gpg --print-md style: "<file name>: ABCD 1234 ..."
+ grouped = " ".join(digest.upper()[i : i + 8] for i in range(0, len(digest), 8))
+ self.assertEqual(
+ digest, _parse_sha512_checksum("%s: %s\n" % (file_name, grouped), file_name)
+ )
+
+ # Anything that does not contain exactly one SHA-512 digest is rejected.
+ with self.assertRaises(ValueError):
+ _parse_sha512_checksum("not a checksum file", file_name)
+ with self.assertRaises(ValueError):
+ _parse_sha512_checksum("", file_name)
+
+ def test_verify_checksum(self):
+ with tempfile.TemporaryDirectory(prefix="test_install_spark") as tmp_dir:
+ path = os.path.join(tmp_dir, "pkg.tgz")
+ with open(path, "wb") as f:
+ f.write(b"content")
+
+ _verify_checksum(path, hashlib.sha512(b"content").hexdigest())
+
+ with self.assertRaisesRegex(ValueError, "SHA-512 mismatch"):
+ _verify_checksum(path, "0" * 128)
+
def test_package_name(self):
self.assertEqual(
"spark-3.0.0-bin-hadoop3.2", checked_package_name("spark-3.0.0", "hadoop3.2", "hive2.3")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org