You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ze...@apache.org on 2022/02/08 11:47:45 UTC
[spark] branch master updated: [SPARK-37412][PYTHON][ML] Inline typehints for pyspark.ml.stat
This is an automated email from the ASF dual-hosted git repository.
zero323 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 3d736d9 [SPARK-37412][PYTHON][ML] Inline typehints for pyspark.ml.stat
3d736d9 is described below
commit 3d736d978cdaf345d81833a6216d41464fa86c69
Author: zero323 <ms...@gmail.com>
AuthorDate: Tue Feb 8 12:46:30 2022 +0100
[SPARK-37412][PYTHON][ML] Inline typehints for pyspark.ml.stat
### What changes were proposed in this pull request?
This PR migrates type `pyspark.ml.stat` annotations from stub file to inline type hints.
(second take, after issue resulting in reversion of #35401)
### Why are the changes needed?
Part of ongoing migration of type hints.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing tests.
Closes #35437 from zero323/SPARK-37412.
Authored-by: zero323 <ms...@gmail.com>
Signed-off-by: zero323 <ms...@gmail.com>
---
python/pyspark/ml/stat.py | 62 ++++++++++++++++++++++++++-------------
python/pyspark/ml/stat.pyi | 73 ----------------------------------------------
2 files changed, 41 insertions(+), 94 deletions(-)
diff --git a/python/pyspark/ml/stat.py b/python/pyspark/ml/stat.py
index 15bb6ca..3b3588c 100644
--- a/python/pyspark/ml/stat.py
+++ b/python/pyspark/ml/stat.py
@@ -17,12 +17,20 @@
import sys
+from typing import Optional, Tuple, TYPE_CHECKING
+
+
from pyspark import since, SparkContext
from pyspark.ml.common import _java2py, _py2java
-from pyspark.ml.wrapper import JavaWrapper, _jvm
+from pyspark.ml.linalg import Matrix, Vector
+from pyspark.ml.wrapper import JavaWrapper, _jvm # type: ignore[attr-defined]
from pyspark.sql.column import Column, _to_seq
+from pyspark.sql.dataframe import DataFrame
from pyspark.sql.functions import lit
+if TYPE_CHECKING:
+ from py4j.java_gateway import JavaObject # type: ignore[import]
+
class ChiSquareTest:
"""
@@ -37,7 +45,9 @@ class ChiSquareTest:
"""
@staticmethod
- def test(dataset, featuresCol, labelCol, flatten=False):
+ def test(
+ dataset: DataFrame, featuresCol: str, labelCol: str, flatten: bool = False
+ ) -> DataFrame:
"""
Perform a Pearson's independence test using dataset.
@@ -95,6 +105,8 @@ class ChiSquareTest:
4.0
"""
sc = SparkContext._active_spark_context
+ assert sc is not None
+
javaTestObj = _jvm().org.apache.spark.ml.stat.ChiSquareTest
args = [_py2java(sc, arg) for arg in (dataset, featuresCol, labelCol, flatten)]
return _java2py(sc, javaTestObj.test(*args))
@@ -116,7 +128,7 @@ class Correlation:
"""
@staticmethod
- def corr(dataset, column, method="pearson"):
+ def corr(dataset: DataFrame, column: str, method: str = "pearson") -> DataFrame:
"""
Compute the correlation matrix with specified method using dataset.
@@ -162,6 +174,8 @@ class Correlation:
[ 0.4 , 0.9486... , NaN, 1. ]])
"""
sc = SparkContext._active_spark_context
+ assert sc is not None
+
javaCorrObj = _jvm().org.apache.spark.ml.stat.Correlation
args = [_py2java(sc, arg) for arg in (dataset, column, method)]
return _java2py(sc, javaCorrObj.corr(*args))
@@ -181,7 +195,7 @@ class KolmogorovSmirnovTest:
"""
@staticmethod
- def test(dataset, sampleCol, distName, *params):
+ def test(dataset: DataFrame, sampleCol: str, distName: str, *params: float) -> DataFrame:
"""
Conduct a one-sample, two-sided Kolmogorov-Smirnov test for probability distribution
equality. Currently supports the normal distribution, taking as parameters the mean and
@@ -228,9 +242,11 @@ class KolmogorovSmirnovTest:
0.175
"""
sc = SparkContext._active_spark_context
+ assert sc is not None
+
javaTestObj = _jvm().org.apache.spark.ml.stat.KolmogorovSmirnovTest
dataset = _py2java(sc, dataset)
- params = [float(param) for param in params]
+ params = [float(param) for param in params] # type: ignore[assignment]
return _java2py(
sc, javaTestObj.test(dataset, sampleCol, distName, _jvm().PythonUtils.toSeq(params))
)
@@ -284,7 +300,7 @@ class Summarizer:
@staticmethod
@since("2.4.0")
- def mean(col, weightCol=None):
+ def mean(col: Column, weightCol: Optional[Column] = None) -> Column:
"""
return a column of mean summary
"""
@@ -292,7 +308,7 @@ class Summarizer:
@staticmethod
@since("3.0.0")
- def sum(col, weightCol=None):
+ def sum(col: Column, weightCol: Optional[Column] = None) -> Column:
"""
return a column of sum summary
"""
@@ -300,7 +316,7 @@ class Summarizer:
@staticmethod
@since("2.4.0")
- def variance(col, weightCol=None):
+ def variance(col: Column, weightCol: Optional[Column] = None) -> Column:
"""
return a column of variance summary
"""
@@ -308,7 +324,7 @@ class Summarizer:
@staticmethod
@since("3.0.0")
- def std(col, weightCol=None):
+ def std(col: Column, weightCol: Optional[Column] = None) -> Column:
"""
return a column of std summary
"""
@@ -316,7 +332,7 @@ class Summarizer:
@staticmethod
@since("2.4.0")
- def count(col, weightCol=None):
+ def count(col: Column, weightCol: Optional[Column] = None) -> Column:
"""
return a column of count summary
"""
@@ -324,7 +340,7 @@ class Summarizer:
@staticmethod
@since("2.4.0")
- def numNonZeros(col, weightCol=None):
+ def numNonZeros(col: Column, weightCol: Optional[Column] = None) -> Column:
"""
return a column of numNonZero summary
"""
@@ -332,7 +348,7 @@ class Summarizer:
@staticmethod
@since("2.4.0")
- def max(col, weightCol=None):
+ def max(col: Column, weightCol: Optional[Column] = None) -> Column:
"""
return a column of max summary
"""
@@ -340,7 +356,7 @@ class Summarizer:
@staticmethod
@since("2.4.0")
- def min(col, weightCol=None):
+ def min(col: Column, weightCol: Optional[Column] = None) -> Column:
"""
return a column of min summary
"""
@@ -348,7 +364,7 @@ class Summarizer:
@staticmethod
@since("2.4.0")
- def normL1(col, weightCol=None):
+ def normL1(col: Column, weightCol: Optional[Column] = None) -> Column:
"""
return a column of normL1 summary
"""
@@ -356,14 +372,14 @@ class Summarizer:
@staticmethod
@since("2.4.0")
- def normL2(col, weightCol=None):
+ def normL2(col: Column, weightCol: Optional[Column] = None) -> Column:
"""
return a column of normL2 summary
"""
return Summarizer._get_single_metric(col, weightCol, "normL2")
@staticmethod
- def _check_param(featuresCol, weightCol):
+ def _check_param(featuresCol: Column, weightCol: Optional[Column]) -> Tuple[Column, Column]:
if weightCol is None:
weightCol = lit(1.0)
if not isinstance(featuresCol, Column) or not isinstance(weightCol, Column):
@@ -371,7 +387,7 @@ class Summarizer:
return featuresCol, weightCol
@staticmethod
- def _get_single_metric(col, weightCol, metric):
+ def _get_single_metric(col: Column, weightCol: Optional[Column], metric: str) -> Column:
col, weightCol = Summarizer._check_param(col, weightCol)
return Column(
JavaWrapper._new_java_obj(
@@ -380,7 +396,7 @@ class Summarizer:
)
@staticmethod
- def metrics(*metrics):
+ def metrics(*metrics: str) -> "SummaryBuilder":
"""
Given a list of metrics, provides a builder that it turns computes metrics from a column.
@@ -415,6 +431,8 @@ class Summarizer:
:py:class:`pyspark.ml.stat.SummaryBuilder`
"""
sc = SparkContext._active_spark_context
+ assert sc is not None
+
js = JavaWrapper._new_java_obj(
"org.apache.spark.ml.stat.Summarizer.metrics", _to_seq(sc, metrics)
)
@@ -432,10 +450,10 @@ class SummaryBuilder(JavaWrapper):
"""
- def __init__(self, jSummaryBuilder):
+ def __init__(self, jSummaryBuilder: "JavaObject"):
super(SummaryBuilder, self).__init__(jSummaryBuilder)
- def summary(self, featuresCol, weightCol=None):
+ def summary(self, featuresCol: Column, weightCol: Optional[Column] = None) -> Column:
"""
Returns an aggregate object that contains the summary of the column with the requested
metrics.
@@ -456,6 +474,8 @@ class SummaryBuilder(JavaWrapper):
structure is determined during the creation of the builder.
"""
featuresCol, weightCol = Summarizer._check_param(featuresCol, weightCol)
+ assert self._java_obj is not None
+
return Column(self._java_obj.summary(featuresCol._jc, weightCol._jc))
@@ -474,7 +494,7 @@ class MultivariateGaussian:
[ 3., 2.]]))
"""
- def __init__(self, mean, cov):
+ def __init__(self, mean: Vector, cov: Matrix):
self.mean = mean
self.cov = cov
diff --git a/python/pyspark/ml/stat.pyi b/python/pyspark/ml/stat.pyi
deleted file mode 100644
index 90b0686..0000000
--- a/python/pyspark/ml/stat.pyi
+++ /dev/null
@@ -1,73 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-from typing import Optional
-
-from pyspark.ml.linalg import Matrix, Vector
-from pyspark.ml.wrapper import JavaWrapper
-from pyspark.sql.column import Column
-from pyspark.sql.dataframe import DataFrame
-
-from py4j.java_gateway import JavaObject # type: ignore[import]
-
-class ChiSquareTest:
- @staticmethod
- def test(
- dataset: DataFrame, featuresCol: str, labelCol: str, flatten: bool = ...
- ) -> DataFrame: ...
-
-class Correlation:
- @staticmethod
- def corr(dataset: DataFrame, column: str, method: str = ...) -> DataFrame: ...
-
-class KolmogorovSmirnovTest:
- @staticmethod
- def test(dataset: DataFrame, sampleCol: str, distName: str, *params: float) -> DataFrame: ...
-
-class Summarizer:
- @staticmethod
- def mean(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
- @staticmethod
- def sum(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
- @staticmethod
- def variance(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
- @staticmethod
- def std(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
- @staticmethod
- def count(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
- @staticmethod
- def numNonZeros(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
- @staticmethod
- def max(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
- @staticmethod
- def min(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
- @staticmethod
- def normL1(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
- @staticmethod
- def normL2(col: Column, weightCol: Optional[Column] = ...) -> Column: ...
- @staticmethod
- def metrics(*metrics: str) -> SummaryBuilder: ...
-
-class SummaryBuilder(JavaWrapper):
- def __init__(self, jSummaryBuilder: JavaObject) -> None: ...
- def summary(self, featuresCol: Column, weightCol: Optional[Column] = ...) -> Column: ...
-
-class MultivariateGaussian:
- mean: Vector
- cov: Matrix
- def __init__(self, mean: Vector, cov: Matrix) -> None: ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org