You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2026/03/26 06:11:05 UTC
(spark) branch master updated: [SPARK-56066][PYTHON] Lazy import numpy to improve import speed
This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 533670b57870 [SPARK-56066][PYTHON] Lazy import numpy to improve import speed
533670b57870 is described below
commit 533670b578702e0570f46eb98cc144d187eb993a
Author: Tian Gao <ga...@hotmail.com>
AuthorDate: Thu Mar 26 14:10:49 2026 +0800
[SPARK-56066][PYTHON] Lazy import numpy to improve import speed
### What changes were proposed in this pull request?
In `StatCounter`, only import `numpy` when needed.
### Why are the changes needed?
This is the only place that triggers `import numpy` when we `import pyspark`. importing `numpy` takes about 30 ms. We can reduce the bootstrap time significantly.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
CI
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #54896 from gaogaotiantian/lazy-load-numpy.
Authored-by: Tian Gao <ga...@hotmail.com>
Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
python/pyspark/statcounter.py | 29 ++++++++++++++++-------------
1 file changed, 16 insertions(+), 13 deletions(-)
diff --git a/python/pyspark/statcounter.py b/python/pyspark/statcounter.py
index 758d0a6fb8b0..92bf0b11a3e1 100644
--- a/python/pyspark/statcounter.py
+++ b/python/pyspark/statcounter.py
@@ -21,16 +21,19 @@ import copy
import math
from typing import Dict, Iterable, Optional
-try:
- from numpy import maximum, minimum, sqrt
-except ImportError:
- maximum = max # type: ignore[assignment]
- minimum = min # type: ignore[assignment]
- sqrt = math.sqrt # type: ignore[assignment]
-
class StatCounter:
def __init__(self, values: Optional[Iterable[float]] = None):
+ try:
+ from numpy import maximum, minimum, sqrt
+ except ImportError:
+ maximum = max # type: ignore[assignment]
+ minimum = min # type: ignore[assignment]
+ sqrt = math.sqrt # type: ignore[assignment]
+
+ self.maximum = maximum
+ self.minimum = minimum
+ self.sqrt = sqrt
if values is None:
values = list()
self.n = 0 # Running count of our values
@@ -48,8 +51,8 @@ class StatCounter:
self.n += 1
self.mu += delta / self.n
self.m2 += delta * (value - self.mu)
- self.maxValue = maximum(self.maxValue, value)
- self.minValue = minimum(self.minValue, value)
+ self.maxValue = self.maximum(self.maxValue, value)
+ self.minValue = self.minimum(self.minValue, value)
return self
@@ -77,8 +80,8 @@ class StatCounter:
else:
self.mu = (self.mu * self.n + other.mu * other.n) / (self.n + other.n)
- self.maxValue = maximum(self.maxValue, other.maxValue)
- self.minValue = minimum(self.minValue, other.minValue)
+ self.maxValue = self.maximum(self.maxValue, other.maxValue)
+ self.minValue = self.minimum(self.minValue, other.minValue)
self.m2 += other.m2 + (delta * delta * self.n * other.n) / (self.n + other.n)
self.n += other.n
@@ -122,14 +125,14 @@ class StatCounter:
# Return the standard deviation of the values.
def stdev(self) -> float:
- return sqrt(self.variance())
+ return self.sqrt(self.variance())
#
# Return the sample standard deviation of the values, which corrects for bias in estimating the
# variance by dividing by N-1 instead of N.
#
def sampleStdev(self) -> float:
- return sqrt(self.sampleVariance())
+ return self.sqrt(self.sampleVariance())
def asDict(self, sample: bool = False) -> Dict[str, float]:
"""Returns the :class:`StatCounter` members as a ``dict``.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org