You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/03/16 16:20:17 UTC
(spark) branch master updated: [SPARK-55986][PYTHON] Upgrade black to 26.3.1
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new cbcee8c2f6c2 [SPARK-55986][PYTHON] Upgrade black to 26.3.1
cbcee8c2f6c2 is described below
commit cbcee8c2f6c2c2af5e934575a57ae1cd7528f415
Author: yangjie01 <ya...@baidu.com>
AuthorDate: Mon Mar 16 09:20:05 2026 -0700
[SPARK-55986][PYTHON] Upgrade black to 26.3.1
### What changes were proposed in this pull request?
This pr aims to upgrade black from 23.12.1 to 26.3.1
### Why are the changes needed?
To fix https://github.com/apache/spark/security/dependabot/172
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Pass Github Actions
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #54782 from LuciferYang/black-26.3.1.
Authored-by: yangjie01 <ya...@baidu.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.github/workflows/build_and_test.yml | 4 +-
.github/workflows/pages.yml | 2 +-
dev/create-release/spark-rm/Dockerfile | 2 +-
dev/merge_spark_pr.py | 4 +-
dev/reformat-python | 2 +-
dev/requirements.txt | 2 +-
dev/spark-test-image/connect-gen-protos/Dockerfile | 2 +-
dev/spark-test-image/docs/Dockerfile | 2 +-
dev/spark-test-image/lint/Dockerfile | 2 +-
dev/sparktestsupport/modules.py | 18 +-
dev/sparktestsupport/toposort.py | 1 -
pyproject.toml | 2 +-
python/benchmarks/bench_eval_type.py | 1 -
python/pyspark/accumulators.py | 8 +-
python/pyspark/conf.py | 17 +-
python/pyspark/core/broadcast.py | 16 +-
python/pyspark/core/context.py | 21 +-
python/pyspark/core/files.py | 4 +-
python/pyspark/core/rdd.py | 66 ++--
python/pyspark/errors/__init__.py | 2 +-
python/pyspark/errors/exceptions/connect.py | 24 +-
python/pyspark/errors/tests/test_traceback.py | 5 +-
python/pyspark/errors/utils.py | 8 +-
python/pyspark/instrumentation_utils.py | 1 -
python/pyspark/logger/__init__.py | 1 +
python/pyspark/logger/logger.py | 2 +-
python/pyspark/logger/worker_io.py | 2 +-
python/pyspark/ml/__init__.py | 1 +
python/pyspark/ml/base.py | 6 +-
python/pyspark/ml/classification.py | 14 +-
python/pyspark/ml/clustering.py | 2 +-
python/pyspark/ml/connect/functions.py | 3 +-
python/pyspark/ml/connect/io_utils.py | 1 -
python/pyspark/ml/connect/pipeline.py | 1 -
python/pyspark/ml/connect/readwrite.py | 4 +-
python/pyspark/ml/connect/tuning.py | 1 -
.../deepspeed/tests/test_deepspeed_distributor.py | 8 +-
python/pyspark/ml/dl_util.py | 6 +-
python/pyspark/ml/evaluation.py | 2 +-
python/pyspark/ml/feature.py | 99 ++----
python/pyspark/ml/fpm.py | 3 +-
python/pyspark/ml/functions.py | 10 +-
python/pyspark/ml/image.py | 2 +-
python/pyspark/ml/linalg/__init__.py | 49 +--
python/pyspark/ml/param/__init__.py | 7 +-
python/pyspark/ml/recommendation.py | 2 +-
python/pyspark/ml/regression.py | 2 +-
.../tests/connect/test_parity_torch_distributor.py | 6 +-
python/pyspark/ml/tests/test_dl_util.py | 6 +-
python/pyspark/ml/torch/distributor.py | 8 +-
python/pyspark/ml/torch/tests/test_distributor.py | 6 +-
python/pyspark/ml/tuning.py | 15 +-
python/pyspark/ml/util.py | 6 +-
python/pyspark/ml/wrapper.py | 1 -
python/pyspark/mllib/__init__.py | 1 +
python/pyspark/mllib/classification.py | 29 +-
python/pyspark/mllib/clustering.py | 30 +-
python/pyspark/mllib/evaluation.py | 2 +-
python/pyspark/mllib/feature.py | 51 ++-
python/pyspark/mllib/fpm.py | 2 +-
python/pyspark/mllib/linalg/__init__.py | 48 +--
python/pyspark/mllib/linalg/distributed.py | 2 +-
python/pyspark/mllib/random.py | 3 +-
python/pyspark/mllib/recommendation.py | 3 +-
python/pyspark/mllib/regression.py | 27 +-
python/pyspark/mllib/stat/_statistics.py | 18 +-
python/pyspark/mllib/stat/distribution.py | 1 -
python/pyspark/mllib/stat/test.py | 1 -
python/pyspark/mllib/tests/test_algorithms.py | 1 -
python/pyspark/mllib/tests/test_linalg.py | 1 -
python/pyspark/mllib/tree.py | 14 +-
python/pyspark/mllib/util.py | 3 +-
python/pyspark/pandas/_typing.py | 1 -
python/pyspark/pandas/accessors.py | 3 +-
python/pyspark/pandas/base.py | 11 +-
python/pyspark/pandas/categorical.py | 2 +-
python/pyspark/pandas/config.py | 4 +-
python/pyspark/pandas/correlation.py | 1 -
python/pyspark/pandas/datetimes.py | 3 +-
python/pyspark/pandas/exceptions.py | 3 +-
python/pyspark/pandas/extensions.py | 2 +-
python/pyspark/pandas/frame.py | 23 +-
python/pyspark/pandas/generic.py | 3 +-
python/pyspark/pandas/groupby.py | 3 +-
python/pyspark/pandas/indexes/base.py | 18 +-
python/pyspark/pandas/indexes/category.py | 2 +-
python/pyspark/pandas/indexes/datetimes.py | 2 +-
python/pyspark/pandas/indexes/multi.py | 2 +-
python/pyspark/pandas/indexes/timedelta.py | 1 -
python/pyspark/pandas/indexing.py | 35 +--
python/pyspark/pandas/internal.py | 45 ++-
python/pyspark/pandas/mlflow.py | 3 +-
python/pyspark/pandas/namespace.py | 21 +-
python/pyspark/pandas/numpy_compat.py | 3 +-
python/pyspark/pandas/plot/matplotlib.py | 16 +-
python/pyspark/pandas/resample.py | 5 +-
python/pyspark/pandas/series.py | 23 +-
python/pyspark/pandas/spark/accessors.py | 3 +-
python/pyspark/pandas/spark/utils.py | 27 +-
python/pyspark/pandas/sql_formatter.py | 3 +-
python/pyspark/pandas/sql_processor.py | 3 +-
python/pyspark/pandas/strings.py | 4 +-
python/pyspark/pandas/supported_api_gen.py | 9 +-
python/pyspark/pandas/testing.py | 1 +
.../pandas/tests/data_type_ops/test_as_type.py | 10 +-
.../tests/diff_frames_ops/test_dot_series.py | 10 +-
python/pyspark/pandas/tests/groupby/test_stat.py | 1 -
.../pyspark/pandas/tests/indexes/test_default.py | 7 +-
python/pyspark/pandas/tests/io/test_csv.py | 37 +--
.../pandas/tests/io/test_dataframe_conversion.py | 12 +-
python/pyspark/pandas/tests/test_typedef.py | 40 ++-
python/pyspark/pandas/typedef/typehints.py | 20 +-
python/pyspark/pandas/utils.py | 11 +-
python/pyspark/pandas/window.py | 2 +-
python/pyspark/pipelines/api.py | 18 +-
python/pyspark/pipelines/cli.py | 1 +
python/pyspark/pipelines/spark_connect_pipeline.py | 2 +-
python/pyspark/pipelines/tests/test_cli.py | 72 ++---
python/pyspark/pipelines/tests/test_init_cli.py | 1 -
python/pyspark/profiler.py | 4 +-
python/pyspark/resource/__init__.py | 1 +
python/pyspark/resource/information.py | 1 -
python/pyspark/resource/profile.py | 10 +-
python/pyspark/resource/requests.py | 14 +-
python/pyspark/resultiterable.py | 1 -
python/pyspark/serializers.py | 19 +-
python/pyspark/shell.py | 7 +-
python/pyspark/shuffle.py | 8 +-
python/pyspark/sql/__init__.py | 2 +-
python/pyspark/sql/avro/functions.py | 3 +-
python/pyspark/sql/catalog.py | 2 +-
python/pyspark/sql/classic/column.py | 10 +-
python/pyspark/sql/classic/dataframe.py | 92 ++----
python/pyspark/sql/classic/table_arg.py | 1 -
python/pyspark/sql/classic/window.py | 4 +-
python/pyspark/sql/column.py | 93 ++----
python/pyspark/sql/conf.py | 2 +-
python/pyspark/sql/connect/_typing.py | 13 +-
python/pyspark/sql/connect/avro/functions.py | 2 +-
python/pyspark/sql/connect/catalog.py | 2 +-
python/pyspark/sql/connect/client/artifact.py | 1 -
python/pyspark/sql/connect/client/core.py | 12 +-
python/pyspark/sql/connect/client/reattach.py | 6 +-
python/pyspark/sql/connect/column.py | 3 +-
python/pyspark/sql/connect/conf.py | 2 +-
python/pyspark/sql/connect/dataframe.py | 43 +--
python/pyspark/sql/connect/datasource.py | 1 -
python/pyspark/sql/connect/expressions.py | 3 +-
python/pyspark/sql/connect/functions/__init__.py | 1 +
python/pyspark/sql/connect/functions/builtin.py | 69 ++---
.../pyspark/sql/connect/functions/partitioning.py | 3 +-
python/pyspark/sql/connect/group.py | 8 +-
python/pyspark/sql/connect/merge.py | 2 +-
python/pyspark/sql/connect/observation.py | 3 +-
python/pyspark/sql/connect/plan.py | 16 +-
python/pyspark/sql/connect/proto/base_pb2.py | 32 +-
python/pyspark/sql/connect/proto/base_pb2.pyi | 228 ++++++++++----
python/pyspark/sql/connect/proto/base_pb2_grpc.py | 1 +
python/pyspark/sql/connect/proto/catalog_pb2.py | 8 +-
python/pyspark/sql/connect/proto/catalog_pb2.pyi | 10 +-
python/pyspark/sql/connect/proto/commands_pb2.py | 14 +-
python/pyspark/sql/connect/proto/commands_pb2.pyi | 188 ++++++-----
python/pyspark/sql/connect/proto/common_pb2.py | 7 +-
python/pyspark/sql/connect/proto/common_pb2.pyi | 15 +-
.../sql/connect/proto/example_plugins_pb2.py | 8 +-
.../sql/connect/proto/example_plugins_pb2.pyi | 1 +
.../pyspark/sql/connect/proto/expressions_pb2.py | 8 +-
.../pyspark/sql/connect/proto/expressions_pb2.pyi | 60 +++-
python/pyspark/sql/connect/proto/ml_common_pb2.py | 8 +-
python/pyspark/sql/connect/proto/ml_common_pb2.pyi | 12 +-
python/pyspark/sql/connect/proto/ml_pb2.py | 8 +-
python/pyspark/sql/connect/proto/ml_pb2.pyi | 20 +-
python/pyspark/sql/connect/proto/pipelines_pb2.py | 26 +-
python/pyspark/sql/connect/proto/pipelines_pb2.pyi | 60 ++--
python/pyspark/sql/connect/proto/relations_pb2.py | 8 +-
python/pyspark/sql/connect/proto/relations_pb2.pyi | 344 ++++++++++++++-------
python/pyspark/sql/connect/proto/types_pb2.py | 7 +-
python/pyspark/sql/connect/proto/types_pb2.pyi | 14 +-
python/pyspark/sql/connect/protobuf/functions.py | 2 +-
python/pyspark/sql/connect/readwriter.py | 35 +--
python/pyspark/sql/connect/session.py | 18 +-
python/pyspark/sql/connect/shell/progress.py | 1 +
python/pyspark/sql/connect/streaming/query.py | 6 +-
python/pyspark/sql/connect/streaming/readwriter.py | 37 +--
.../streaming/worker/foreach_batch_worker.py | 1 +
.../connect/streaming/worker/listener_worker.py | 1 +
python/pyspark/sql/connect/tvf.py | 2 +-
python/pyspark/sql/connect/udf.py | 1 +
python/pyspark/sql/connect/udtf.py | 6 +-
python/pyspark/sql/connect/window.py | 2 +-
python/pyspark/sql/context.py | 17 +-
python/pyspark/sql/conversion.py | 16 +-
python/pyspark/sql/dataframe.py | 126 +++-----
python/pyspark/sql/datasource_internal.py | 2 +-
python/pyspark/sql/functions/builtin.py | 124 +++-----
python/pyspark/sql/functions/partitioning.py | 2 +-
python/pyspark/sql/group.py | 8 +-
python/pyspark/sql/merge.py | 2 +-
python/pyspark/sql/metrics.py | 13 +-
python/pyspark/sql/observation.py | 2 +-
python/pyspark/sql/pandas/conversion.py | 22 +-
python/pyspark/sql/pandas/functions.py | 2 +-
python/pyspark/sql/pandas/group_ops.py | 2 +-
python/pyspark/sql/pandas/map_ops.py | 2 +-
python/pyspark/sql/pandas/serializers.py | 24 +-
python/pyspark/sql/pandas/typehints.py | 4 +-
python/pyspark/sql/pandas/types.py | 19 +-
python/pyspark/sql/plot/__init__.py | 1 +
python/pyspark/sql/profiler.py | 11 +-
python/pyspark/sql/protobuf/functions.py | 3 +-
python/pyspark/sql/readwriter.py | 35 +--
python/pyspark/sql/session.py | 43 +--
python/pyspark/sql/streaming/listener.py | 22 +-
.../sql/streaming/proto/StateMessage_pb2.py | 1 +
.../sql/streaming/proto/StateMessage_pb2.pyi | 13 +-
.../streaming/python_streaming_source_runner.py | 1 -
python/pyspark/sql/streaming/query.py | 2 +-
python/pyspark/sql/streaming/readwriter.py | 34 +-
python/pyspark/sql/table_arg.py | 1 -
.../sql/tests/arrow/test_arrow_python_udf.py | 13 +-
.../sql/tests/arrow/test_arrow_udf_grouped_agg.py | 6 +-
.../sql/tests/arrow/test_arrow_udf_scalar.py | 54 ++--
.../sql/tests/arrow/test_arrow_udf_window.py | 6 +-
python/pyspark/sql/tests/arrow/test_arrow_udtf.py | 114 +++----
.../sql/tests/connect/test_connect_basic.py | 1 -
.../sql/tests/connect/test_connect_session.py | 12 +-
.../tests/connect/test_parity_python_datasource.py | 3 +-
.../helper/helper_pandas_transform_with_state.py | 6 +-
.../streaming/test_pandas_transform_with_state.py | 3 +-
...t_pandas_transform_with_state_state_variable.py | 3 +-
.../sql/tests/pandas/test_pandas_cogrouped_map.py | 21 +-
.../sql/tests/pandas/test_pandas_grouped_map.py | 5 +-
python/pyspark/sql/tests/pandas/test_pandas_map.py | 5 +-
.../tests/pandas/test_pandas_udf_grouped_agg.py | 1 -
.../test_streaming_offline_state_repartition.py | 5 +-
python/pyspark/sql/tests/test_dataframe.py | 6 +-
python/pyspark/sql/tests/test_functions.py | 4 +-
python/pyspark/sql/tests/test_observation.py | 10 +-
python/pyspark/sql/tests/test_python_datasource.py | 11 +-
.../sql/tests/test_python_streaming_datasource.py | 3 +-
python/pyspark/sql/tests/test_subquery.py | 126 +++-----
python/pyspark/sql/tests/test_tvf.py | 102 ++----
python/pyspark/sql/tests/test_types.py | 10 +-
python/pyspark/sql/tests/test_udf.py | 12 +-
python/pyspark/sql/tests/test_udtf.py | 77 ++---
python/pyspark/sql/tests/test_unified_udf.py | 12 +-
python/pyspark/sql/tvf.py | 2 +-
python/pyspark/sql/types.py | 35 +--
python/pyspark/sql/udf.py | 5 +-
python/pyspark/sql/udtf.py | 3 +-
python/pyspark/sql/utils.py | 8 +-
python/pyspark/sql/worker/analyze_udtf.py | 48 +--
python/pyspark/storagelevel.py | 1 -
python/pyspark/streaming/dstream.py | 32 +-
python/pyspark/streaming/kinesis.py | 7 +-
python/pyspark/streaming/listener.py | 1 -
python/pyspark/streaming/util.py | 2 +-
python/pyspark/taskcontext.py | 7 +-
python/pyspark/testing/connectutils.py | 1 -
python/pyspark/testing/pandasutils.py | 1 -
python/pyspark/testing/sqlutils.py | 1 -
python/pyspark/testing/streamingutils.py | 1 -
python/pyspark/testing/utils.py | 3 +-
python/pyspark/tests/test_appsubmit.py | 7 +-
python/pyspark/tests/test_daemon.py | 2 +-
python/pyspark/tests/test_rdd.py | 7 +-
python/pyspark/tests/test_serializers.py | 2 -
.../pyarrow/test_pyarrow_ignore_timezone.py | 1 -
python/pyspark/traceback_utils.py | 1 -
python/pyspark/util.py | 14 +-
python/pyspark/worker.py | 15 +-
python/pyspark/worker_util.py | 5 +-
sql/connect/common/src/main/protobuf/README.md | 2 +-
273 files changed, 1970 insertions(+), 2350 deletions(-)
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index bfa8c1864226..9fec6330ffd3 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -842,7 +842,7 @@ jobs:
python-version: '3.12'
- name: Install dependencies for Python CodeGen check
run: |
- python3.12 -m pip install 'black==23.12.1' 'protobuf==6.33.5' 'mypy==1.8.0' 'mypy-protobuf==3.3.0'
+ python3.12 -m pip install 'black==26.3.1' 'protobuf==6.33.5' 'mypy==1.8.0' 'mypy-protobuf==3.3.0'
python3.12 -m pip list
- name: Python CodeGen check for branch-3.5
if: inputs.branch == 'branch-3.5'
@@ -944,7 +944,7 @@ jobs:
run: |
# SPARK-45212: Copy from https://github.com/apache/spark/blob/555c8def51e5951c7bf5165a332795e9e330ec9d/.github/workflows/build_and_test.yml#L631-L638
# Should delete this section after SPARK 3.5 EOL.
- python3.9 -m pip install 'flake8==3.9.0' pydata_sphinx_theme 'mypy==0.982' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' numpydoc 'jinja2<3.0.0' 'black==22.6.0'
+ python3.9 -m pip install 'flake8==3.9.0' pydata_sphinx_theme 'mypy==0.982' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' numpydoc 'jinja2<3.0.0' 'black==26.3.1'
python3.9 -m pip install 'pandas-stubs==1.2.0.53' ipython 'grpcio==1.56.0' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0'
- name: List Python packages
shell: 'script -q -e -c "bash {0}"'
diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml
index 276b822b4dc1..a17da0ef5aca 100644
--- a/.github/workflows/pages.yml
+++ b/.github/workflows/pages.yml
@@ -62,7 +62,7 @@ jobs:
run: |
pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' \
ipython ipython_genutils sphinx_plotly_directive 'numpy>=1.22' pyarrow 'pandas==2.3.3' 'plotly>=4.8' 'docutils<0.18.0' \
- 'flake8==3.9.0' 'mypy==1.8.0' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'black==23.12.1' \
+ 'flake8==3.9.0' 'mypy==1.8.0' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'black==26.3.1' \
'pandas-stubs==1.2.0.53' 'grpcio==1.76.0' 'grpcio-status==1.76.0' 'protobuf==6.33.5' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0' \
'sphinxcontrib-applehelp==1.0.4' 'sphinxcontrib-devhelp==1.0.2' 'sphinxcontrib-htmlhelp==2.0.1' 'sphinxcontrib-qthelp==1.0.3' 'sphinxcontrib-serializinghtml==1.1.5'
- name: Install Ruby for documentation generation
diff --git a/dev/create-release/spark-rm/Dockerfile b/dev/create-release/spark-rm/Dockerfile
index 72bc13739749..63b27db3b95d 100644
--- a/dev/create-release/spark-rm/Dockerfile
+++ b/dev/create-release/spark-rm/Dockerfile
@@ -61,7 +61,7 @@ RUN python3.10 -m pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13'
sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' \
ipython ipython_genutils sphinx_plotly_directive 'numpy>=1.22' pyarrow pandas \
'plotly>=4.8' 'docutils<0.18.0' 'flake8==3.9.0' 'mypy==1.19.1' 'pytest==7.1.3' \
- 'pytest-mypy-plugins==1.9.3' 'black==23.12.1' 'pandas-stubs==1.2.0.53' \
+ 'pytest-mypy-plugins==1.9.3' 'black==26.3.1' 'pandas-stubs==1.2.0.53' \
'grpcio==1.76.0' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0' \
'sphinxcontrib-applehelp==1.0.4' 'sphinxcontrib-devhelp==1.0.2' \
'sphinxcontrib-htmlhelp==2.0.1' 'sphinxcontrib-qthelp==1.0.3' \
diff --git a/dev/merge_spark_pr.py b/dev/merge_spark_pr.py
index 1eef16aa229f..ff7a95fbaf53 100755
--- a/dev/merge_spark_pr.py
+++ b/dev/merge_spark_pr.py
@@ -345,7 +345,7 @@ def resolve_jira_issue(merge_branches, comment, default_jira_id=""):
# In this case, if the PR is committed to the master branch and the release branch, we
# only consider the release branch to be the fix version. E.g. it is not valid to have
# both 1.1.0 and 1.0.0 as fix versions.
- (major, minor, patch) = v.split(".")
+ major, minor, patch = v.split(".")
if patch == "0":
previous = "%s.%s.%s" % (major, int(minor) - 1, 0)
if previous in default_fix_versions:
@@ -737,7 +737,7 @@ def main():
if __name__ == "__main__":
import doctest
- (failure_count, test_count) = doctest.testmod()
+ failure_count, test_count = doctest.testmod()
if failure_count:
sys.exit(-1)
try:
diff --git a/dev/reformat-python b/dev/reformat-python
index 19f0ffa74cc7..a8e6c25817ac 100755
--- a/dev/reformat-python
+++ b/dev/reformat-python
@@ -22,7 +22,7 @@ FWDIR="$( cd "$DIR"/.. && pwd )"
cd "$FWDIR"
BLACK_BUILD="${PYTHON_EXECUTABLE} -m black"
-BLACK_VERSION="23.12.1"
+BLACK_VERSION="26.3.1"
$PYTHON_EXECUTABLE -c 'import black' 2> /dev/null
if [ $? -ne 0 ]; then
echo "The Python library providing the 'black' module was not found. Please install Black, for example, via 'pip install black==$BLACK_VERSION'."
diff --git a/dev/requirements.txt b/dev/requirements.txt
index a219a017cf92..8656a22b10cf 100644
--- a/dev/requirements.txt
+++ b/dev/requirements.txt
@@ -59,7 +59,7 @@ jira>=3.5.2
PyGithub
# pandas API on Spark Code formatter.
-black==23.12.1
+black==26.3.1
py
# Spark Connect (required)
diff --git a/dev/spark-test-image/connect-gen-protos/Dockerfile b/dev/spark-test-image/connect-gen-protos/Dockerfile
index 28c072ece678..7d1ca14c4762 100644
--- a/dev/spark-test-image/connect-gen-protos/Dockerfile
+++ b/dev/spark-test-image/connect-gen-protos/Dockerfile
@@ -53,7 +53,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
RUN python3.12 -m pip install \
'mypy==1.19.1' \
'mypy-protobuf==3.3.0' \
- 'black==23.12.1'
+ 'black==26.3.1'
# Mount the Spark repo at /spark
WORKDIR /spark
diff --git a/dev/spark-test-image/docs/Dockerfile b/dev/spark-test-image/docs/Dockerfile
index 1cdfde2d046c..a4c7f41dcd08 100644
--- a/dev/spark-test-image/docs/Dockerfile
+++ b/dev/spark-test-image/docs/Dockerfile
@@ -89,7 +89,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
# See 'docutils<0.18.0' in SPARK-39421
RUN python3.12 -m pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe \
ipython ipython_genutils sphinx_plotly_directive 'numpy>=1.22' 'pyarrow>=23.0.0' 'pandas==2.3.3' 'plotly>=4.8' 'docutils<0.18.0' \
- 'flake8==3.9.0' 'mypy==1.19.1' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'black==23.12.1' \
+ 'flake8==3.9.0' 'mypy==1.19.1' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'black==26.3.1' \
'pandas-stubs==1.2.0.53' 'grpcio==1.76.0' 'grpcio-status==1.76.0' 'protobuf==6.33.5' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0' \
'sphinxcontrib-applehelp==1.0.4' 'sphinxcontrib-devhelp==1.0.2' 'sphinxcontrib-htmlhelp==2.0.1' 'sphinxcontrib-qthelp==1.0.3' 'sphinxcontrib-serializinghtml==1.1.5' \
&& python3.12 -m pip cache purge
diff --git a/dev/spark-test-image/lint/Dockerfile b/dev/spark-test-image/lint/Dockerfile
index ac95c002f491..751988ffb3ea 100644
--- a/dev/spark-test-image/lint/Dockerfile
+++ b/dev/spark-test-image/lint/Dockerfile
@@ -77,7 +77,7 @@ RUN python3.12 -m venv $VIRTUAL_ENV
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
RUN python3.12 -m pip install \
- 'black==23.12.1' \
+ 'black==26.3.1' \
'flake8==3.9.0' \
'ruff==0.14.8' \
'googleapis-common-protos-stubs==2.2.0' \
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 1b6c2eb83a57..c24fd2c08e4a 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -253,9 +253,9 @@ catalyst = Module(
sbt_test_goals=[
"catalyst/test",
],
- environ=None
- if "GITHUB_ACTIONS" not in os.environ
- else {"ENABLE_DOCKER_INTEGRATION_TESTS": "1"},
+ environ=(
+ None if "GITHUB_ACTIONS" not in os.environ else {"ENABLE_DOCKER_INTEGRATION_TESTS": "1"}
+ ),
)
sql = Module(
@@ -268,9 +268,9 @@ sql = Module(
sbt_test_goals=[
"sql/test",
],
- environ=None
- if "GITHUB_ACTIONS" not in os.environ
- else {"ENABLE_DOCKER_INTEGRATION_TESTS": "1"},
+ environ=(
+ None if "GITHUB_ACTIONS" not in os.environ else {"ENABLE_DOCKER_INTEGRATION_TESTS": "1"}
+ ),
)
hive = Module(
@@ -1688,9 +1688,9 @@ docker_integration_tests = Module(
build_profile_flags=["-Pdocker-integration-tests"],
source_file_regexes=["connector/docker-integration-tests"],
sbt_test_goals=["docker-integration-tests/test"],
- environ=None
- if "GITHUB_ACTIONS" not in os.environ
- else {"ENABLE_DOCKER_INTEGRATION_TESTS": "1"},
+ environ=(
+ None if "GITHUB_ACTIONS" not in os.environ else {"ENABLE_DOCKER_INTEGRATION_TESTS": "1"}
+ ),
test_tags=["org.apache.spark.tags.DockerTest"],
)
diff --git a/dev/sparktestsupport/toposort.py b/dev/sparktestsupport/toposort.py
index dcc8d4a83ec8..64d36faca92d 100644
--- a/dev/sparktestsupport/toposort.py
+++ b/dev/sparktestsupport/toposort.py
@@ -34,7 +34,6 @@
from functools import reduce as _reduce
-
__all__ = ["toposort", "toposort_flatten"]
diff --git a/pyproject.toml b/pyproject.toml
index bb8685f89005..ca90a0319b9f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -64,7 +64,7 @@ ignore = [
[tool.black]
# When changing the version, we have to update
# GitHub workflow version and dev/reformat-python
-required-version = "23.12.1"
+required-version = "26.3.1"
line-length = 100
target-version = ['py39']
include = '\.pyi?$'
diff --git a/python/benchmarks/bench_eval_type.py b/python/benchmarks/bench_eval_type.py
index 9569eeacf23f..7a2a5ddc0357 100644
--- a/python/benchmarks/bench_eval_type.py
+++ b/python/benchmarks/bench_eval_type.py
@@ -50,7 +50,6 @@ from pyspark.sql.types import (
from pyspark.util import PythonEvalType
from pyspark.worker import main as worker_main
-
# ---------------------------------------------------------------------------
# Wire-format helpers
# ---------------------------------------------------------------------------
diff --git a/python/pyspark/accumulators.py b/python/pyspark/accumulators.py
index 39791f4bbd01..7584a840a1c7 100644
--- a/python/pyspark/accumulators.py
+++ b/python/pyspark/accumulators.py
@@ -63,7 +63,6 @@ class SpecialAccumulatorIds:
class Accumulator(Generic[T]):
-
"""
A shared variable that can be accumulated, i.e., has a commutative and associative "add"
operation. Worker tasks on a Spark cluster can add values to an Accumulator with the `+=`
@@ -186,7 +185,6 @@ class Accumulator(Generic[T]):
class AccumulatorParam(Generic[T]):
-
"""
Helper object that defines how to accumulate values of a given type.
@@ -229,7 +227,6 @@ class AccumulatorParam(Generic[T]):
class AddingAccumulatorParam(AccumulatorParam[U]):
-
"""
An AccumulatorParam that uses the + operators to add values. Designed for simple types
such as integers, floats, and lists. Requires the zero value for the underlying type
@@ -254,7 +251,6 @@ COMPLEX_ACCUMULATOR_PARAM = AddingAccumulatorParam(0.0j) # type: ignore[type-va
class UpdateRequestHandler(socketserver.StreamRequestHandler):
-
"""
This handler will keep polling updates from the same socket until the
server is shutdown.
@@ -299,7 +295,7 @@ class UpdateRequestHandler(socketserver.StreamRequestHandler):
def accum_updates() -> bool:
num_updates = read_int(self.rfile)
for _ in range(num_updates):
- (aid, update) = pickleSer._read_with_length(self.rfile)
+ aid, update = pickleSer._read_with_length(self.rfile)
_accumulatorRegistry[aid] += update
# Write a byte in acknowledgement
self.wfile.write(struct.pack("!b", 1))
@@ -406,7 +402,7 @@ if __name__ == "__main__":
# The small batch size here ensures that we see multiple batches,
# even in these small test examples:
globs["sc"] = SparkContext("local", "test")
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
globs["sc"].stop()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/conf.py b/python/pyspark/conf.py
index 850754a75789..2f109b007b9c 100644
--- a/python/pyspark/conf.py
+++ b/python/pyspark/conf.py
@@ -172,12 +172,10 @@ class SparkConf:
return self
@overload
- def setExecutorEnv(self, key: str, value: str) -> "SparkConf":
- ...
+ def setExecutorEnv(self, key: str, value: str) -> "SparkConf": ...
@overload
- def setExecutorEnv(self, *, pairs: List[Tuple[str, str]]) -> "SparkConf":
- ...
+ def setExecutorEnv(self, *, pairs: List[Tuple[str, str]]) -> "SparkConf": ...
def setExecutorEnv(
self,
@@ -212,16 +210,13 @@ class SparkConf:
return self
@overload
- def get(self, key: str) -> Optional[str]:
- ...
+ def get(self, key: str) -> Optional[str]: ...
@overload
- def get(self, key: str, defaultValue: None) -> Optional[str]:
- ...
+ def get(self, key: str, defaultValue: None) -> Optional[str]: ...
@overload
- def get(self, key: str, defaultValue: str) -> str:
- ...
+ def get(self, key: str, defaultValue: str) -> str: ...
def get(self, key: str, defaultValue: Optional[str] = None) -> Optional[str]:
"""Get the configured value for some key, or return a default otherwise."""
@@ -273,7 +268,7 @@ class SparkConf:
def _test() -> None:
import doctest
- (failure_count, test_count) = doctest.testmod(optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(optionflags=doctest.ELLIPSIS)
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/core/broadcast.py b/python/pyspark/core/broadcast.py
index 2d5658284be8..9b0002bdccfe 100644
--- a/python/pyspark/core/broadcast.py
+++ b/python/pyspark/core/broadcast.py
@@ -68,7 +68,6 @@ def _from_id(bid: int) -> "Broadcast[Any]":
class Broadcast(Generic[T]):
-
"""
A broadcast variable created with :meth:`SparkContext.broadcast`.
Access its value through :attr:`value`.
@@ -91,16 +90,13 @@ class Broadcast(Generic[T]):
sc: "SparkContext",
value: T,
pickle_registry: "BroadcastPickleRegistry",
- ):
- ...
+ ): ...
@overload # On worker without decryption server
- def __init__(self: "Broadcast[Any]", *, path: str):
- ...
+ def __init__(self: "Broadcast[Any]", *, path: str): ...
@overload # On worker with decryption server
- def __init__(self: "Broadcast[Any]", *, sock_file: str):
- ...
+ def __init__(self: "Broadcast[Any]", *, sock_file: str): ...
def __init__( # type: ignore[misc]
self,
@@ -126,7 +122,7 @@ class Broadcast(Generic[T]):
# with encryption, we ask the jvm to do the encryption for us, we send it data
# over a socket
conn_info, auth_secret = self._python_broadcast.setupEncryptionServer()
- (encryption_sock_file, _) = local_connect_and_auth(conn_info, auth_secret)
+ encryption_sock_file, _ = local_connect_and_auth(conn_info, auth_secret)
broadcast_out = ChunkedStream(encryption_sock_file, 8192)
else:
# no encryption, we can just write pickled data directly to the file from python
@@ -271,7 +267,7 @@ class Broadcast(Generic[T]):
# if its on the driver, since executor decryption is handled already
if self._sc is not None and self._sc._encryption_enabled:
conn_info, auth_secret = self._python_broadcast.setupDecryptionServer()
- (decrypted_sock_file, _) = local_connect_and_auth(conn_info, auth_secret)
+ decrypted_sock_file, _ = local_connect_and_auth(conn_info, auth_secret)
self._python_broadcast.waitTillBroadcastDataSent()
return self.load(decrypted_sock_file)
else:
@@ -372,7 +368,7 @@ def _test() -> None:
spark = SparkSession.builder.master("local[4]").appName("broadcast tests").getOrCreate()
globs["spark"] = spark
- (failure_count, test_count) = doctest.testmod(pyspark.core.broadcast, globs=globs)
+ failure_count, test_count = doctest.testmod(pyspark.core.broadcast, globs=globs)
spark.stop()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/core/context.py b/python/pyspark/core/context.py
index 9bab9d1afe96..63260f8f3cd9 100644
--- a/python/pyspark/core/context.py
+++ b/python/pyspark/core/context.py
@@ -95,7 +95,6 @@ U = TypeVar("U")
class SparkContext:
-
"""
Main entry point for Spark functionality. A SparkContext represents the
connection to a Spark cluster, and can be used to create :class:`RDD` and
@@ -162,9 +161,9 @@ class SparkContext:
_next_accum_id = 0
_active_spark_context: ClassVar[Optional["SparkContext"]] = None
_lock = RLock()
- _python_includes: Optional[
- List[str]
- ] = None # zip and egg files that need to be added to PYTHONPATH
+ _python_includes: Optional[List[str]] = (
+ None # zip and egg files that need to be added to PYTHONPATH
+ )
serializer: Serializer
profiler_collector: ProfilerCollector
@@ -327,7 +326,7 @@ class SparkContext:
self._accumulatorServer.server_address
)
else:
- (host, port) = self._accumulatorServer.server_address # type: ignore[misc]
+ host, port = self._accumulatorServer.server_address # type: ignore[misc]
self._javaAccumulator = self._jvm.PythonAccumulatorV2(host, port, auth_token)
self._jsc.sc().register(self._javaAccumulator)
@@ -362,7 +361,7 @@ class SparkContext:
# with SparkContext.addFile, so we just need to add them to the PYTHONPATH
for path in self._conf.get("spark.submit.pyFiles", "").split(","):
if path != "":
- (dirname, filename) = os.path.split(path)
+ dirname, filename = os.path.split(path)
try:
filepath = os.path.join(SparkFiles.getRootDirectory(), filename)
if not os.path.exists(filepath):
@@ -438,9 +437,7 @@ class SparkContext:
<dd><code>{sc.appName}</code></dd>
</dl>
</div>
- """.format(
- sc=self
- )
+ """.format(sc=self)
def _initialize_context(self, jconf: JavaObject) -> JavaObject:
"""
@@ -901,7 +898,7 @@ class SparkContext:
if self._encryption_enabled:
# with encryption, we open a server in java and send the data directly
server = server_func()
- (sock_file, _) = local_connect_and_auth(server.connInfo(), server.secret())
+ sock_file, _ = local_connect_and_auth(server.connInfo(), server.secret())
chunked_out = ChunkedStream(sock_file, 8192)
serializer.dump_stream(data, chunked_out)
chunked_out.close()
@@ -1987,7 +1984,7 @@ class SparkContext:
A path can be added only once. Subsequent additions of the same path are ignored.
"""
self.addFile(path)
- (dirname, filename) = os.path.split(path) # dirname may be directory or HDFS/S3 prefix
+ dirname, filename = os.path.split(path) # dirname may be directory or HDFS/S3 prefix
if filename[-4:].lower() in self.PACKAGE_EXTENSIONS:
assert self._python_includes is not None
self._python_includes.append(filename)
@@ -2663,7 +2660,7 @@ def _test() -> None:
globs = globals().copy()
conf = SparkConf().set("spark.ui.enabled", "True")
globs["sc"] = SparkContext("local[4]", "context tests", conf=conf)
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
globs["sc"].stop()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/core/files.py b/python/pyspark/core/files.py
index a2544425af0c..aec67a0fe31c 100644
--- a/python/pyspark/core/files.py
+++ b/python/pyspark/core/files.py
@@ -17,7 +17,6 @@
import os
-
__all__ = ["SparkFiles"]
from typing import cast, ClassVar, Optional, TYPE_CHECKING
@@ -27,7 +26,6 @@ if TYPE_CHECKING:
class SparkFiles:
-
"""
Resolves paths to files added through :meth:`SparkContext.addFile`.
@@ -155,7 +153,7 @@ def _test() -> None:
globs = globals().copy()
globs["sc"] = SparkContext("local[2]", "files tests")
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
globs["sc"].stop()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/core/rdd.py b/python/pyspark/core/rdd.py
index 131d9fc5dc8c..18e0d286fc7b 100644
--- a/python/pyspark/core/rdd.py
+++ b/python/pyspark/core/rdd.py
@@ -94,7 +94,6 @@ from pyspark.errors import PySparkRuntimeError
# for backward compatibility references.
from pyspark.util import PythonEvalType # noqa: F401
-
if TYPE_CHECKING:
from py4j.java_gateway import JavaObject
@@ -196,7 +195,6 @@ class Partitioner:
class RDD(Generic[T_co]):
-
"""
A Resilient Distributed Dataset (RDD), the basic abstraction in Spark.
Represents an immutable, partitioned collection of elements that can be
@@ -1205,8 +1203,7 @@ class RDD(Generic[T_co]):
numPartitions: Optional[int] = ...,
partitionFunc: Callable[["S"], int] = ...,
ascending: bool = ...,
- ) -> "RDD[Tuple[S, V]]":
- ...
+ ) -> "RDD[Tuple[S, V]]": ...
@overload
def repartitionAndSortWithinPartitions(
@@ -1215,8 +1212,7 @@ class RDD(Generic[T_co]):
partitionFunc: Callable[[K], int],
ascending: bool,
keyfunc: Callable[[K], "S"],
- ) -> "RDD[Tuple[K, V]]":
- ...
+ ) -> "RDD[Tuple[K, V]]": ...
@overload
def repartitionAndSortWithinPartitions(
@@ -1226,8 +1222,7 @@ class RDD(Generic[T_co]):
ascending: bool = ...,
*,
keyfunc: Callable[[K], "S"],
- ) -> "RDD[Tuple[K, V]]":
- ...
+ ) -> "RDD[Tuple[K, V]]": ...
def repartitionAndSortWithinPartitions(
self: "RDD[Tuple[Any, Any]]",
@@ -1289,8 +1284,7 @@ class RDD(Generic[T_co]):
self: "RDD[Tuple[S, V]]",
ascending: bool = ...,
numPartitions: Optional[int] = ...,
- ) -> "RDD[Tuple[K, V]]":
- ...
+ ) -> "RDD[Tuple[K, V]]": ...
@overload
def sortByKey(
@@ -1298,8 +1292,7 @@ class RDD(Generic[T_co]):
ascending: bool,
numPartitions: int,
keyfunc: Callable[[K], "S"],
- ) -> "RDD[Tuple[K, V]]":
- ...
+ ) -> "RDD[Tuple[K, V]]": ...
@overload
def sortByKey(
@@ -1308,8 +1301,7 @@ class RDD(Generic[T_co]):
numPartitions: Optional[int] = ...,
*,
keyfunc: Callable[[K], "S"],
- ) -> "RDD[Tuple[K, V]]":
- ...
+ ) -> "RDD[Tuple[K, V]]": ...
def sortByKey(
self: "RDD[Tuple[K, V]]",
@@ -2064,12 +2056,10 @@ class RDD(Generic[T_co]):
return partiallyAggregated.reduce(combOp)
@overload
- def max(self: "RDD[S]") -> "S":
- ...
+ def max(self: "RDD[S]") -> "S": ...
@overload
- def max(self: "RDD[T]", key: Callable[[T], "S"]) -> T:
- ...
+ def max(self: "RDD[T]", key: Callable[[T], "S"]) -> T: ...
def max(self: "RDD[T]", key: Optional[Callable[[T], "S"]] = None) -> T:
"""
@@ -2104,12 +2094,10 @@ class RDD(Generic[T_co]):
return self.reduce(lambda a, b: max(a, b, key=key))
@overload
- def min(self: "RDD[S]") -> "S":
- ...
+ def min(self: "RDD[S]") -> "S": ...
@overload
- def min(self: "RDD[T]", key: Callable[[T], "S"]) -> T:
- ...
+ def min(self: "RDD[T]", key: Callable[[T], "S"]) -> T: ...
def min(self: "RDD[T]", key: Optional[Callable[[T], "S"]] = None) -> T:
"""
@@ -2538,12 +2526,10 @@ class RDD(Generic[T_co]):
return self.mapPartitions(countPartition).reduce(mergeMaps)
@overload
- def top(self: "RDD[S]", num: int) -> List["S"]:
- ...
+ def top(self: "RDD[S]", num: int) -> List["S"]: ...
@overload
- def top(self: "RDD[T]", num: int, key: Callable[[T], "S"]) -> List[T]:
- ...
+ def top(self: "RDD[T]", num: int, key: Callable[[T], "S"]) -> List[T]: ...
def top(self: "RDD[T]", num: int, key: Optional[Callable[[T], "S"]] = None) -> List[T]:
"""
@@ -2595,12 +2581,10 @@ class RDD(Generic[T_co]):
return self.mapPartitions(topIterator).reduce(merge)
@overload
- def takeOrdered(self: "RDD[S]", num: int) -> List["S"]:
- ...
+ def takeOrdered(self: "RDD[S]", num: int) -> List["S"]: ...
@overload
- def takeOrdered(self: "RDD[T]", num: int, key: Callable[[T], "S"]) -> List[T]:
- ...
+ def takeOrdered(self: "RDD[T]", num: int, key: Callable[[T], "S"]) -> List[T]: ...
def takeOrdered(self: "RDD[T]", num: int, key: Optional[Callable[[T], "S"]] = None) -> List[T]:
"""
@@ -4143,14 +4127,12 @@ class RDD(Generic[T_co]):
@overload
def groupWith(
self: "RDD[Tuple[K, V]]", other: "RDD[Tuple[K, V1]]"
- ) -> "RDD[Tuple[K, Tuple[ResultIterable[V], ResultIterable[V1]]]]":
- ...
+ ) -> "RDD[Tuple[K, Tuple[ResultIterable[V], ResultIterable[V1]]]]": ...
@overload
def groupWith(
self: "RDD[Tuple[K, V]]", other: "RDD[Tuple[K, V1]]", __o1: "RDD[Tuple[K, V2]]"
- ) -> "RDD[Tuple[K, Tuple[ResultIterable[V], ResultIterable[V1], ResultIterable[V2]]]]":
- ...
+ ) -> "RDD[Tuple[K, Tuple[ResultIterable[V], ResultIterable[V1], ResultIterable[V2]]]]": ...
@overload
def groupWith(
@@ -4168,8 +4150,7 @@ class RDD(Generic[T_co]):
ResultIterable[V3],
],
]
- ]""":
- ...
+ ]""": ...
def groupWith( # type: ignore[misc]
self: "RDD[Tuple[Any, Any]]", other: "RDD[Tuple[Any, Any]]", *others: "RDD[Tuple[Any, Any]]"
@@ -5096,21 +5077,18 @@ class RDD(Generic[T_co]):
self: "RDD[RowLike]",
schema: Optional[Union[List[str], Tuple[str, ...]]] = None,
sampleRatio: Optional[float] = None,
- ) -> "DataFrame":
- ...
+ ) -> "DataFrame": ...
@overload
def toDF(
self: "RDD[RowLike]", schema: Optional[Union["StructType", str]] = None
- ) -> "DataFrame":
- ...
+ ) -> "DataFrame": ...
@overload
def toDF(
self: "RDD[AtomicValue]",
schema: Union["AtomicType", str],
- ) -> "DataFrame":
- ...
+ ) -> "DataFrame": ...
def toDF(
self: "RDD[Any]", schema: Optional[Any] = None, sampleRatio: Optional[float] = None
@@ -5158,7 +5136,6 @@ def _wrap_function(
class RDDBarrier(Generic[T]):
-
"""
Wraps an RDD in a barrier stage, which forces Spark to launch tasks of this stage together.
:class:`RDDBarrier` instances are created by :meth:`RDD.barrier`.
@@ -5274,7 +5251,6 @@ class RDDBarrier(Generic[T]):
class PipelinedRDD(RDD[U], Generic[T, U]):
-
"""
Examples
--------
@@ -5399,7 +5375,7 @@ def _test() -> None:
# even in these small test examples:
globs["sc"] = SparkContext("local[4]", "PythonTest")
globs["sc"].setCheckpointDir(tmp_dir.name)
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
globs["sc"].stop()
tmp_dir.cleanup()
if failure_count:
diff --git a/python/pyspark/errors/__init__.py b/python/pyspark/errors/__init__.py
index 3293f1fe699b..cde12949c9ca 100644
--- a/python/pyspark/errors/__init__.py
+++ b/python/pyspark/errors/__init__.py
@@ -18,6 +18,7 @@
"""
PySpark exceptions.
"""
+
from pyspark.errors.exceptions.base import (
PySparkException,
AnalysisException,
@@ -53,7 +54,6 @@ from pyspark.errors.exceptions.base import (
PickleException,
)
-
__all__ = [
"PySparkException",
"AnalysisException",
diff --git a/python/pyspark/errors/exceptions/connect.py b/python/pyspark/errors/exceptions/connect.py
index 88acc1a9d5b9..90537c2cc364 100644
--- a/python/pyspark/errors/exceptions/connect.py
+++ b/python/pyspark/errors/exceptions/connect.py
@@ -104,12 +104,16 @@ def convert_observation_errors(
return _convert_exception(
classes=list(root_error.error_type_hierarchy),
- sql_state=root_error.spark_throwable.sql_state
- if root_error.spark_throwable.HasField("sql_state")
- else None,
- error_class=root_error.spark_throwable.error_class
- if root_error.spark_throwable.HasField("error_class")
- else None,
+ sql_state=(
+ root_error.spark_throwable.sql_state
+ if root_error.spark_throwable.HasField("sql_state")
+ else None
+ ),
+ error_class=(
+ root_error.spark_throwable.error_class
+ if root_error.spark_throwable.HasField("error_class")
+ else None
+ ),
reason=None,
root_error_idx=root_error_idx,
errors=errors,
@@ -150,9 +154,11 @@ def _convert_exception(
error_class = root_error.spark_throwable.error_class
message_parameters = dict(root_error.spark_throwable.message_parameters)
contexts = [
- SQLQueryContext(c)
- if c.context_type == pb2.FetchErrorDetailsResponse.QueryContext.SQL
- else DataFrameQueryContext(c)
+ (
+ SQLQueryContext(c)
+ if c.context_type == pb2.FetchErrorDetailsResponse.QueryContext.SQL
+ else DataFrameQueryContext(c)
+ )
for c in root_error.spark_throwable.query_contexts
]
# Extract breaking change info if present
diff --git a/python/pyspark/errors/tests/test_traceback.py b/python/pyspark/errors/tests/test_traceback.py
index 070ce666700a..c3803369a439 100644
--- a/python/pyspark/errors/tests/test_traceback.py
+++ b/python/pyspark/errors/tests/test_traceback.py
@@ -167,8 +167,9 @@ class BaseTracebackSqlTestsMixin:
def test_udf(self):
for jvm_stack_trace in [False, True]:
- with self.subTest(jvm_stack_trace=jvm_stack_trace), self.sql_conf(
- {"spark.sql.pyspark.jvmStacktrace.enabled": jvm_stack_trace}
+ with (
+ self.subTest(jvm_stack_trace=jvm_stack_trace),
+ self.sql_conf({"spark.sql.pyspark.jvmStacktrace.enabled": jvm_stack_trace}),
):
@sf.udf()
diff --git a/python/pyspark/errors/utils.py b/python/pyspark/errors/utils.py
index e420aea86921..c796e37a28c5 100644
--- a/python/pyspark/errors/utils.py
+++ b/python/pyspark/errors/utils.py
@@ -347,15 +347,15 @@ def _with_origin(func: FuncT) -> FuncT:
@overload
-def with_origin_to_class(cls_or_ignores: Type[T], ignores: Optional[List[str]] = None) -> Type[T]:
- ...
+def with_origin_to_class(
+ cls_or_ignores: Type[T], ignores: Optional[List[str]] = None
+) -> Type[T]: ...
@overload
def with_origin_to_class(
cls_or_ignores: Optional[List[str]] = None,
-) -> Callable[[Type[T]], Type[T]]:
- ...
+) -> Callable[[Type[T]], Type[T]]: ...
def with_origin_to_class(
diff --git a/python/pyspark/instrumentation_utils.py b/python/pyspark/instrumentation_utils.py
index ac971c7543fe..9ad76c4209f8 100644
--- a/python/pyspark/instrumentation_utils.py
+++ b/python/pyspark/instrumentation_utils.py
@@ -24,7 +24,6 @@ import time
from types import ModuleType
from typing import Tuple, Union, List, Callable, Any, Type
-
__all__: List[str] = []
_local = threading.local()
diff --git a/python/pyspark/logger/__init__.py b/python/pyspark/logger/__init__.py
index 3e5034cfe248..9e9548e91983 100644
--- a/python/pyspark/logger/__init__.py
+++ b/python/pyspark/logger/__init__.py
@@ -18,6 +18,7 @@
"""
PySpark logging
"""
+
from pyspark.logger.logger import PySparkLogger, SPARK_LOG_SCHEMA
__all__ = ["PySparkLogger", "SPARK_LOG_SCHEMA"]
diff --git a/python/pyspark/logger/logger.py b/python/pyspark/logger/logger.py
index 72179b033bb3..8f5cfb8628cc 100644
--- a/python/pyspark/logger/logger.py
+++ b/python/pyspark/logger/logger.py
@@ -318,7 +318,7 @@ def _test() -> None:
import pyspark.logger.logger
globs = pyspark.logger.logger.__dict__.copy()
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.logger.logger, globs=globs, optionflags=doctest.ELLIPSIS
)
diff --git a/python/pyspark/logger/worker_io.py b/python/pyspark/logger/worker_io.py
index 7843b43e10ee..4b8a7cf15912 100644
--- a/python/pyspark/logger/worker_io.py
+++ b/python/pyspark/logger/worker_io.py
@@ -266,7 +266,7 @@ def context_provider() -> dict[str, str]:
@contextmanager
def capture_outputs(
- context_provider: Callable[[], dict[str, str]] = context_provider
+ context_provider: Callable[[], dict[str, str]] = context_provider,
) -> Generator[None, None, None]:
if "PYSPARK_SPARK_SESSION_UUID" in os.environ:
marker: str = "PYTHON_WORKER_LOGGING"
diff --git a/python/pyspark/ml/__init__.py b/python/pyspark/ml/__init__.py
index 167cf33a1665..2f692ee2e569 100644
--- a/python/pyspark/ml/__init__.py
+++ b/python/pyspark/ml/__init__.py
@@ -19,6 +19,7 @@
DataFrame-based machine learning APIs to let users quickly assemble and configure practical
machine learning pipelines.
"""
+
from pyspark.ml.base import (
Estimator,
Model,
diff --git a/python/pyspark/ml/base.py b/python/pyspark/ml/base.py
index 224ef34fd5ed..882c9b1f65a0 100644
--- a/python/pyspark/ml/base.py
+++ b/python/pyspark/ml/base.py
@@ -156,14 +156,12 @@ class Estimator(Params, Generic[M], metaclass=ABCMeta):
return _FitMultipleIterator(fitSingleModel, len(paramMaps))
@overload
- def fit(self, dataset: DataFrame, params: Optional["ParamMap"] = ...) -> M:
- ...
+ def fit(self, dataset: DataFrame, params: Optional["ParamMap"] = ...) -> M: ...
@overload
def fit(
self, dataset: DataFrame, params: Union[List["ParamMap"], Tuple["ParamMap"]]
- ) -> List[M]:
- ...
+ ) -> List[M]: ...
def fit(
self,
diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py
index 2590cc9c5097..de279ab03aa9 100644
--- a/python/pyspark/ml/classification.py
+++ b/python/pyspark/ml/classification.py
@@ -1261,8 +1261,7 @@ class LogisticRegression(
lowerBoundsOnIntercepts: Optional[Vector] = ...,
upperBoundsOnIntercepts: Optional[Vector] = ...,
maxBlockSizeInMB: float = ...,
- ):
- ...
+ ): ...
@overload
def __init__(
@@ -1288,8 +1287,7 @@ class LogisticRegression(
lowerBoundsOnIntercepts: Optional[Vector] = ...,
upperBoundsOnIntercepts: Optional[Vector] = ...,
maxBlockSizeInMB: float = ...,
- ):
- ...
+ ): ...
@keyword_only
def __init__(
@@ -1360,8 +1358,7 @@ class LogisticRegression(
lowerBoundsOnIntercepts: Optional[Vector] = ...,
upperBoundsOnIntercepts: Optional[Vector] = ...,
maxBlockSizeInMB: float = ...,
- ) -> "LogisticRegression":
- ...
+ ) -> "LogisticRegression": ...
@overload
def setParams(
@@ -1387,8 +1384,7 @@ class LogisticRegression(
lowerBoundsOnIntercepts: Optional[Vector] = ...,
upperBoundsOnIntercepts: Optional[Vector] = ...,
maxBlockSizeInMB: float = ...,
- ) -> "LogisticRegression":
- ...
+ ) -> "LogisticRegression": ...
@keyword_only
@since("1.3.0")
@@ -4351,7 +4347,7 @@ if __name__ == "__main__":
temp_path = tempfile.mkdtemp()
globs["temp_path"] = temp_path
try:
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
spark.stop()
finally:
from shutil import rmtree
diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py
index f6543e707680..740ccd6b546b 100644
--- a/python/pyspark/ml/clustering.py
+++ b/python/pyspark/ml/clustering.py
@@ -2193,7 +2193,7 @@ if __name__ == "__main__":
temp_path = tempfile.mkdtemp()
globs["temp_path"] = temp_path
try:
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
spark.stop()
finally:
from shutil import rmtree
diff --git a/python/pyspark/ml/connect/functions.py b/python/pyspark/ml/connect/functions.py
index 8df40a400c5f..5f2738cb2b3e 100644
--- a/python/pyspark/ml/connect/functions.py
+++ b/python/pyspark/ml/connect/functions.py
@@ -20,7 +20,6 @@ from typing import Any, TYPE_CHECKING
from pyspark.ml import functions as PyMLFunctions
from pyspark.sql.column import Column
-
if TYPE_CHECKING:
from pyspark.sql._typing import UserDefinedFunctionLike
@@ -76,7 +75,7 @@ def _test() -> None:
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.ml.connect.functions,
globs=globs,
optionflags=doctest.ELLIPSIS
diff --git a/python/pyspark/ml/connect/io_utils.py b/python/pyspark/ml/connect/io_utils.py
index e28b0c7d9dec..fcc2e8d93c0f 100644
--- a/python/pyspark/ml/connect/io_utils.py
+++ b/python/pyspark/ml/connect/io_utils.py
@@ -28,7 +28,6 @@ from pyspark.sql import SparkSession
from pyspark.sql.utils import is_remote
from pyspark import __version__ as pyspark_version
-
_META_DATA_FILE_NAME = "metadata.json"
diff --git a/python/pyspark/ml/connect/pipeline.py b/python/pyspark/ml/connect/pipeline.py
index daba4e6fa218..afc0fa78a9db 100644
--- a/python/pyspark/ml/connect/pipeline.py
+++ b/python/pyspark/ml/connect/pipeline.py
@@ -28,7 +28,6 @@ from pyspark.ml.param import Param, Params
from pyspark.ml.common import inherit_doc
from pyspark.sql.dataframe import DataFrame
-
if TYPE_CHECKING:
from pyspark.ml._typing import ParamMap
diff --git a/python/pyspark/ml/connect/readwrite.py b/python/pyspark/ml/connect/readwrite.py
index efe6d60e0e2c..08802a0fceae 100644
--- a/python/pyspark/ml/connect/readwrite.py
+++ b/python/pyspark/ml/connect/readwrite.py
@@ -255,7 +255,7 @@ class RemoteMLReader(MLReader[RL]):
operator=pb2.MlOperator(name=java_qualified_class_name, type=ml_type), path=path
)
)
- (_, properties, _) = session.client.execute_command(command)
+ _, properties, _ = session.client.execute_command(command)
result = deserialize(properties)
# Get the python type
@@ -353,7 +353,7 @@ class RemoteMLReader(MLReader[RL]):
path=path,
)
)
- (_, properties, _) = session.client.execute_command(command)
+ _, properties, _ = session.client.execute_command(command)
result = deserialize(properties)
instance = PowerIterationClustering()
diff --git a/python/pyspark/ml/connect/tuning.py b/python/pyspark/ml/connect/tuning.py
index 39aae6f17cc5..ac393a3daf1d 100644
--- a/python/pyspark/ml/connect/tuning.py
+++ b/python/pyspark/ml/connect/tuning.py
@@ -46,7 +46,6 @@ from pyspark.sql.dataframe import DataFrame
from pyspark.sql import SparkSession
from pyspark.sql.utils import is_remote
-
if TYPE_CHECKING:
from pyspark.ml._typing import ParamMap
diff --git a/python/pyspark/ml/deepspeed/tests/test_deepspeed_distributor.py b/python/pyspark/ml/deepspeed/tests/test_deepspeed_distributor.py
index bb7ba0e4464c..11be9f051d15 100644
--- a/python/pyspark/ml/deepspeed/tests/test_deepspeed_distributor.py
+++ b/python/pyspark/ml/deepspeed/tests/test_deepspeed_distributor.py
@@ -192,14 +192,12 @@ def _create_pytorch_training_test_file():
# Note: when Deepspeed CPU support becomes better,
# switch in more realistic training files using Deepspeed
# optimizations + constructs
- str_to_write = textwrap.dedent(
- """
+ str_to_write = textwrap.dedent("""
import sys
def pythagorean_thm(x : int, y: int): # type: ignore
import deepspeed # type: ignore
return (x*x + y*y)**0.5 # type: ignore
- print(pythagorean_thm(int(sys.argv[1]), int(sys.argv[2])))"""
- )
+ print(pythagorean_thm(int(sys.argv[1]), int(sys.argv[2])))""")
cp_path = "/tmp/test_deepspeed_training_file.py"
with open(cp_path, "w") as f:
f.write(str_to_write)
@@ -218,7 +216,7 @@ def _create_pytorch_training_test_file():
class DeepspeedTorchDistributorDistributedEndToEnd(unittest.TestCase):
@classmethod
def setUpClass(cls) -> None:
- (cls.gpu_discovery_script_file_name, cls.mnist_dir_path) = set_up_test_dirs() # noqa
+ cls.gpu_discovery_script_file_name, cls.mnist_dir_path = set_up_test_dirs() # noqa
# "loadDefaults" is set to False because if not, the SparkConf will
# use contain configurations from the LocalEndToEnd test,
# which causes the test to break.
diff --git a/python/pyspark/ml/dl_util.py b/python/pyspark/ml/dl_util.py
index 3b87049ef277..26538c328353 100644
--- a/python/pyspark/ml/dl_util.py
+++ b/python/pyspark/ml/dl_util.py
@@ -108,8 +108,7 @@ class FunctionPickler:
The path to the location of the newly created pytorch file.
"""
- code_snippet = textwrap.dedent(
- f"""
+ code_snippet = textwrap.dedent(f"""
from pyspark import cloudpickle
import os
@@ -119,8 +118,7 @@ class FunctionPickler:
output = fn(*args, **kwargs)
with open("{fn_output_path}", "wb") as f:
cloudpickle.dump(output, f)
- """
- )
+ """)
with open(script_path, "w") as f:
if prefix_code != "":
f.write(prefix_code)
diff --git a/python/pyspark/ml/evaluation.py b/python/pyspark/ml/evaluation.py
index 719282d54143..801d23db4eef 100644
--- a/python/pyspark/ml/evaluation.py
+++ b/python/pyspark/ml/evaluation.py
@@ -1182,7 +1182,7 @@ if __name__ == "__main__":
temp_path = tempfile.mkdtemp()
globs["temp_path"] = temp_path
try:
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
spark.stop()
finally:
from shutil import rmtree
diff --git a/python/pyspark/ml/feature.py b/python/pyspark/ml/feature.py
index c5a5033cd8f0..ce008bc2661a 100755
--- a/python/pyspark/ml/feature.py
+++ b/python/pyspark/ml/feature.py
@@ -220,8 +220,7 @@ class Binarizer(
threshold: float = ...,
inputCol: Optional[str] = ...,
outputCol: Optional[str] = ...,
- ):
- ...
+ ): ...
@overload
def __init__(
@@ -230,8 +229,7 @@ class Binarizer(
thresholds: Optional[List[float]] = ...,
inputCols: Optional[List[str]] = ...,
outputCols: Optional[List[str]] = ...,
- ):
- ...
+ ): ...
@keyword_only
def __init__(
@@ -261,8 +259,7 @@ class Binarizer(
threshold: float = ...,
inputCol: Optional[str] = ...,
outputCol: Optional[str] = ...,
- ) -> "Binarizer":
- ...
+ ) -> "Binarizer": ...
@overload
def setParams(
@@ -271,8 +268,7 @@ class Binarizer(
thresholds: Optional[List[float]] = ...,
inputCols: Optional[List[str]] = ...,
outputCols: Optional[List[str]] = ...,
- ) -> "Binarizer":
- ...
+ ) -> "Binarizer": ...
@keyword_only
@since("1.4.0")
@@ -790,8 +786,7 @@ class Bucketizer(
inputCol: Optional[str] = ...,
outputCol: Optional[str] = ...,
handleInvalid: str = ...,
- ):
- ...
+ ): ...
@overload
def __init__(
@@ -801,8 +796,7 @@ class Bucketizer(
splitsArray: Optional[List[List[float]]] = ...,
inputCols: Optional[List[str]] = ...,
outputCols: Optional[List[str]] = ...,
- ):
- ...
+ ): ...
@keyword_only
def __init__(
@@ -834,8 +828,7 @@ class Bucketizer(
inputCol: Optional[str] = ...,
outputCol: Optional[str] = ...,
handleInvalid: str = ...,
- ) -> "Bucketizer":
- ...
+ ) -> "Bucketizer": ...
@overload
def setParams(
@@ -845,8 +838,7 @@ class Bucketizer(
splitsArray: Optional[List[List[float]]] = ...,
inputCols: Optional[List[str]] = ...,
outputCols: Optional[List[str]] = ...,
- ) -> "Bucketizer":
- ...
+ ) -> "Bucketizer": ...
@keyword_only
@since("1.4.0")
@@ -2121,8 +2113,7 @@ class Imputer(
inputCols: Optional[List[str]] = ...,
outputCols: Optional[List[str]] = ...,
relativeError: float = ...,
- ):
- ...
+ ): ...
@overload
def __init__(
@@ -2133,8 +2124,7 @@ class Imputer(
inputCol: Optional[str] = ...,
outputCol: Optional[str] = ...,
relativeError: float = ...,
- ):
- ...
+ ): ...
@keyword_only
def __init__(
@@ -2166,8 +2156,7 @@ class Imputer(
inputCols: Optional[List[str]] = ...,
outputCols: Optional[List[str]] = ...,
relativeError: float = ...,
- ) -> "Imputer":
- ...
+ ) -> "Imputer": ...
@overload
def setParams(
@@ -2178,8 +2167,7 @@ class Imputer(
inputCol: Optional[str] = ...,
outputCol: Optional[str] = ...,
relativeError: float = ...,
- ) -> "Imputer":
- ...
+ ) -> "Imputer": ...
@keyword_only
@since("2.2.0")
@@ -2524,7 +2512,6 @@ class MinHashLSH(
JavaMLReadable["MinHashLSH"],
JavaMLWritable,
):
-
"""
LSH class for Jaccard distance.
The input can be dense or sparse vectors, but it is more efficient if it is sparse.
@@ -3192,8 +3179,7 @@ class OneHotEncoder(
outputCols: Optional[List[str]] = ...,
handleInvalid: str = ...,
dropLast: bool = ...,
- ):
- ...
+ ): ...
@overload
def __init__(
@@ -3203,8 +3189,7 @@ class OneHotEncoder(
dropLast: bool = ...,
inputCol: Optional[str] = ...,
outputCol: Optional[str] = ...,
- ):
- ...
+ ): ...
@keyword_only
def __init__(
@@ -3234,8 +3219,7 @@ class OneHotEncoder(
outputCols: Optional[List[str]] = ...,
handleInvalid: str = ...,
dropLast: bool = ...,
- ) -> "OneHotEncoder":
- ...
+ ) -> "OneHotEncoder": ...
@overload
def setParams(
@@ -3245,8 +3229,7 @@ class OneHotEncoder(
dropLast: bool = ...,
inputCol: Optional[str] = ...,
outputCol: Optional[str] = ...,
- ) -> "OneHotEncoder":
- ...
+ ) -> "OneHotEncoder": ...
@keyword_only
@since("2.3.0")
@@ -3626,8 +3609,7 @@ class QuantileDiscretizer(
outputCol: Optional[str] = ...,
relativeError: float = ...,
handleInvalid: str = ...,
- ):
- ...
+ ): ...
@overload
def __init__(
@@ -3638,8 +3620,7 @@ class QuantileDiscretizer(
numBucketsArray: Optional[List[int]] = ...,
inputCols: Optional[List[str]] = ...,
outputCols: Optional[List[str]] = ...,
- ):
- ...
+ ): ...
@keyword_only
def __init__(
@@ -3675,8 +3656,7 @@ class QuantileDiscretizer(
outputCol: Optional[str] = ...,
relativeError: float = ...,
handleInvalid: str = ...,
- ) -> "QuantileDiscretizer":
- ...
+ ) -> "QuantileDiscretizer": ...
@overload
def setParams(
@@ -3687,8 +3667,7 @@ class QuantileDiscretizer(
numBucketsArray: Optional[List[int]] = ...,
inputCols: Optional[List[str]] = ...,
outputCols: Optional[List[str]] = ...,
- ) -> "QuantileDiscretizer":
- ...
+ ) -> "QuantileDiscretizer": ...
@keyword_only
@since("2.0.0")
@@ -4670,8 +4649,7 @@ class StringIndexer(
outputCol: Optional[str] = ...,
handleInvalid: str = ...,
stringOrderType: str = ...,
- ):
- ...
+ ): ...
@overload
def __init__(
@@ -4681,8 +4659,7 @@ class StringIndexer(
outputCols: Optional[List[str]] = ...,
handleInvalid: str = ...,
stringOrderType: str = ...,
- ):
- ...
+ ): ...
@keyword_only
def __init__(
@@ -4712,8 +4689,7 @@ class StringIndexer(
outputCol: Optional[str] = ...,
handleInvalid: str = ...,
stringOrderType: str = ...,
- ) -> "StringIndexer":
- ...
+ ) -> "StringIndexer": ...
@overload
def setParams(
@@ -4723,8 +4699,7 @@ class StringIndexer(
outputCols: Optional[List[str]] = ...,
handleInvalid: str = ...,
stringOrderType: str = ...,
- ) -> "StringIndexer":
- ...
+ ) -> "StringIndexer": ...
@keyword_only
@since("1.4.0")
@@ -5109,8 +5084,7 @@ class StopWordsRemover(
stopWords: Optional[List[str]] = ...,
caseSensitive: bool = ...,
locale: Optional[str] = ...,
- ):
- ...
+ ): ...
@overload
def __init__(
@@ -5121,8 +5095,7 @@ class StopWordsRemover(
locale: Optional[str] = ...,
inputCols: Optional[List[str]] = ...,
outputCols: Optional[List[str]] = ...,
- ):
- ...
+ ): ...
@keyword_only
def __init__(
@@ -5163,8 +5136,7 @@ class StopWordsRemover(
stopWords: Optional[List[str]] = ...,
caseSensitive: bool = ...,
locale: Optional[str] = ...,
- ) -> "StopWordsRemover":
- ...
+ ) -> "StopWordsRemover": ...
@overload
def setParams(
@@ -5175,8 +5147,7 @@ class StopWordsRemover(
locale: Optional[str] = ...,
inputCols: Optional[List[str]] = ...,
outputCols: Optional[List[str]] = ...,
- ) -> "StopWordsRemover":
- ...
+ ) -> "StopWordsRemover": ...
@keyword_only
@since("1.6.0")
@@ -5376,8 +5347,7 @@ class TargetEncoder(
handleInvalid: str = ...,
targetType: str = ...,
smoothing: float = ...,
- ):
- ...
+ ): ...
@overload
def __init__(
@@ -5389,8 +5359,7 @@ class TargetEncoder(
smoothing: float = ...,
inputCol: Optional[str] = ...,
outputCol: Optional[str] = ...,
- ):
- ...
+ ): ...
@keyword_only
def __init__(
@@ -5424,8 +5393,7 @@ class TargetEncoder(
handleInvalid: str = ...,
targetType: str = ...,
smoothing: float = ...,
- ) -> "TargetEncoder":
- ...
+ ) -> "TargetEncoder": ...
@overload
def setParams(
@@ -5437,8 +5405,7 @@ class TargetEncoder(
smoothing: float = ...,
inputCol: Optional[str] = ...,
outputCol: Optional[str] = ...,
- ) -> "TargetEncoder":
- ...
+ ) -> "TargetEncoder": ...
@keyword_only
@since("4.0.0")
@@ -7855,7 +7822,7 @@ if __name__ == "__main__":
temp_path = tempfile.mkdtemp()
globs["temp_path"] = temp_path
try:
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
spark.stop()
finally:
from shutil import rmtree
diff --git a/python/pyspark/ml/fpm.py b/python/pyspark/ml/fpm.py
index 857e9a97b154..7ad28f69b7c7 100644
--- a/python/pyspark/ml/fpm.py
+++ b/python/pyspark/ml/fpm.py
@@ -240,6 +240,7 @@ class FPGrowth(
>>> fpm.transform(data).take(1) == model2.transform(data).take(1)
True
"""
+
_input_kwargs: Dict[str, Any]
@keyword_only
@@ -547,7 +548,7 @@ if __name__ == "__main__":
temp_path = tempfile.mkdtemp()
globs["temp_path"] = temp_path
try:
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
spark.stop()
finally:
from shutil import rmtree
diff --git a/python/pyspark/ml/functions.py b/python/pyspark/ml/functions.py
index c72e343b73f3..34edc32ecbcc 100644
--- a/python/pyspark/ml/functions.py
+++ b/python/pyspark/ml/functions.py
@@ -217,9 +217,11 @@ def _validate_and_transform_multiple_inputs(
if input_shapes:
if len(input_shapes) == num_input_cols:
multi_inputs = [
- np.vstack(v).reshape([-1] + input_shapes[i]) # type: ignore
- if input_shapes[i]
- else v
+ (
+ np.vstack(v).reshape([-1] + input_shapes[i]) # type: ignore
+ if input_shapes[i]
+ else v
+ )
for i, v in enumerate(multi_inputs)
]
if not all([len(x) == len(batch) for x in multi_inputs]):
@@ -858,7 +860,7 @@ def _test() -> None:
globs["sc"] = sc
globs["spark"] = spark
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.ml.functions,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/ml/image.py b/python/pyspark/ml/image.py
index 325992c08580..d4f59d2d0273 100644
--- a/python/pyspark/ml/image.py
+++ b/python/pyspark/ml/image.py
@@ -237,7 +237,7 @@ def _test() -> None:
spark = SparkSession.builder.master("local[2]").appName("ml.image tests").getOrCreate()
globs["spark"] = spark
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.ml.image, globs=globs, optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE
)
spark.stop()
diff --git a/python/pyspark/ml/linalg/__init__.py b/python/pyspark/ml/linalg/__init__.py
index ee6fc8d21818..508df369dbde 100644
--- a/python/pyspark/ml/linalg/__init__.py
+++ b/python/pyspark/ml/linalg/__init__.py
@@ -55,7 +55,6 @@ from pyspark.sql.types import (
BooleanType,
)
-
__all__ = [
"Vector",
"DenseVector",
@@ -461,12 +460,10 @@ class DenseVector(Vector):
return self.array
@overload
- def __getitem__(self, item: int) -> np.float64:
- ...
+ def __getitem__(self, item: int) -> np.float64: ...
@overload
- def __getitem__(self, item: slice) -> np.ndarray:
- ...
+ def __getitem__(self, item: slice) -> np.ndarray: ...
def __getitem__(self, item: Union[int, slice]) -> Union[np.float64, np.ndarray]:
return self.array[item]
@@ -541,24 +538,19 @@ class SparseVector(Vector):
"""
@overload
- def __init__(self, size: int, __indices: bytes, __values: bytes):
- ...
+ def __init__(self, size: int, __indices: bytes, __values: bytes): ...
@overload
- def __init__(self, size: int, *args: Tuple[int, float]):
- ...
+ def __init__(self, size: int, *args: Tuple[int, float]): ...
@overload
- def __init__(self, size: int, __indices: Iterable[int], __values: Iterable[float]):
- ...
+ def __init__(self, size: int, __indices: Iterable[int], __values: Iterable[float]): ...
@overload
- def __init__(self, size: int, __pairs: Iterable[Tuple[int, float]]):
- ...
+ def __init__(self, size: int, __pairs: Iterable[Tuple[int, float]]): ...
@overload
- def __init__(self, size: int, __map: Dict[int, float]):
- ...
+ def __init__(self, size: int, __map: Dict[int, float]): ...
def __init__(
self,
@@ -867,7 +859,6 @@ class SparseVector(Vector):
class Vectors:
-
"""
Factory methods for working with vectors.
@@ -881,28 +872,23 @@ class Vectors:
@staticmethod
@overload
- def sparse(size: int, __indices: bytes, __values: bytes) -> SparseVector:
- ...
+ def sparse(size: int, __indices: bytes, __values: bytes) -> SparseVector: ...
@staticmethod
@overload
- def sparse(size: int, *args: Tuple[int, float]) -> SparseVector:
- ...
+ def sparse(size: int, *args: Tuple[int, float]) -> SparseVector: ...
@staticmethod
@overload
- def sparse(size: int, __indices: Iterable[int], __values: Iterable[float]) -> SparseVector:
- ...
+ def sparse(size: int, __indices: Iterable[int], __values: Iterable[float]) -> SparseVector: ...
@staticmethod
@overload
- def sparse(size: int, __pairs: Iterable[Tuple[int, float]]) -> SparseVector:
- ...
+ def sparse(size: int, __pairs: Iterable[Tuple[int, float]]) -> SparseVector: ...
@staticmethod
@overload
- def sparse(size: int, __map: Dict[int, float]) -> SparseVector:
- ...
+ def sparse(size: int, __map: Dict[int, float]) -> SparseVector: ...
@staticmethod
def sparse(
@@ -937,18 +923,15 @@ class Vectors:
@overload
@staticmethod
- def dense(*elements: float) -> DenseVector:
- ...
+ def dense(*elements: float) -> DenseVector: ...
@overload
@staticmethod
- def dense(__arr: bytes) -> DenseVector:
- ...
+ def dense(__arr: bytes) -> DenseVector: ...
@overload
@staticmethod
- def dense(__arr: Iterable[float]) -> DenseVector:
- ...
+ def dense(__arr: Iterable[float]) -> DenseVector: ...
@staticmethod
def dense(*elements: Union[float, bytes, np.ndarray, Iterable[float]]) -> DenseVector:
@@ -1375,7 +1358,7 @@ def _test() -> None:
np.set_printoptions(legacy="1.13")
except TypeError:
pass
- (failure_count, test_count) = doctest.testmod(optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(optionflags=doctest.ELLIPSIS)
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/ml/param/__init__.py b/python/pyspark/ml/param/__init__.py
index 6fce132561db..942008405779 100644
--- a/python/pyspark/ml/param/__init__.py
+++ b/python/pyspark/ml/param/__init__.py
@@ -35,7 +35,6 @@ from pyspark.util import is_remote_only
from pyspark.ml.linalg import DenseVector, Vector, Matrix
from pyspark.ml.util import Identifiable
-
if TYPE_CHECKING:
from pyspark.ml._typing import ParamMap
@@ -365,12 +364,10 @@ class Params(Identifiable, metaclass=ABCMeta):
raise TypeError("hasParam(): paramName must be a string")
@overload
- def getOrDefault(self, param: str) -> Any:
- ...
+ def getOrDefault(self, param: str) -> Any: ...
@overload
- def getOrDefault(self, param: Param[T]) -> T:
- ...
+ def getOrDefault(self, param: Param[T]) -> T: ...
def getOrDefault(self, param: Union[str, Param[T]]) -> Union[Any, T]:
"""
diff --git a/python/pyspark/ml/recommendation.py b/python/pyspark/ml/recommendation.py
index d40b231224dc..97a638ed6743 100644
--- a/python/pyspark/ml/recommendation.py
+++ b/python/pyspark/ml/recommendation.py
@@ -741,7 +741,7 @@ if __name__ == "__main__":
temp_path = tempfile.mkdtemp()
globs["temp_path"] = temp_path
try:
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
spark.stop()
finally:
from shutil import rmtree
diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py
index 3873d83b47c8..d910db6c6d30 100644
--- a/python/pyspark/ml/regression.py
+++ b/python/pyspark/ml/regression.py
@@ -3329,7 +3329,7 @@ if __name__ == "__main__":
temp_path = tempfile.mkdtemp()
globs["temp_path"] = temp_path
try:
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
spark.stop()
finally:
from shutil import rmtree
diff --git a/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py b/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py
index 0d4b660d7e14..ce0d64c66f88 100644
--- a/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py
+++ b/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py
@@ -64,7 +64,7 @@ if not is_remote_only() and should_test_connect:
):
@classmethod
def setUpClass(cls):
- (cls.gpu_discovery_script_file_name, cls.mnist_dir_path) = set_up_test_dirs()
+ cls.gpu_discovery_script_file_name, cls.mnist_dir_path = set_up_test_dirs()
builder = SparkSession.builder.appName(cls.__name__)
for k, v in get_local_mode_conf().items():
builder = builder.config(k, v)
@@ -97,7 +97,7 @@ if not is_remote_only() and should_test_connect:
):
@classmethod
def setUpClass(cls):
- (cls.gpu_discovery_script_file_name, cls.mnist_dir_path) = set_up_test_dirs()
+ cls.gpu_discovery_script_file_name, cls.mnist_dir_path = set_up_test_dirs()
builder = SparkSession.builder.appName(cls.__name__)
for k, v in get_local_mode_conf().items():
builder = builder.config(k, v)
@@ -131,7 +131,7 @@ if not is_remote_only() and should_test_connect:
):
@classmethod
def setUpClass(cls):
- (cls.gpu_discovery_script_file_name, cls.mnist_dir_path) = set_up_test_dirs()
+ cls.gpu_discovery_script_file_name, cls.mnist_dir_path = set_up_test_dirs()
builder = SparkSession.builder.appName(cls.__name__)
for k, v in get_distributed_mode_conf().items():
builder = builder.config(k, v)
diff --git a/python/pyspark/ml/tests/test_dl_util.py b/python/pyspark/ml/tests/test_dl_util.py
index 8c4caa3c3ac0..ba9dc31b204e 100644
--- a/python/pyspark/ml/tests/test_dl_util.py
+++ b/python/pyspark/ml/tests/test_dl_util.py
@@ -103,8 +103,7 @@ class TestFunctionPickler(unittest.TestCase):
os.remove(fname)
def _create_code_snippet_body(self, pickled_fn_path: str, fn_output_save_path: str) -> str:
- code_snippet = textwrap.dedent(
- f"""
+ code_snippet = textwrap.dedent(f"""
from pyspark import cloudpickle
import os
@@ -114,8 +113,7 @@ class TestFunctionPickler(unittest.TestCase):
output = fn(*args, **kwargs)
with open("{fn_output_save_path}", "wb") as f:
cloudpickle.dump(output, f)
- """
- )
+ """)
return code_snippet
def _are_two_files_identical(self, fpath1: str, fpath2: str) -> bool:
diff --git a/python/pyspark/ml/torch/distributor.py b/python/pyspark/ml/torch/distributor.py
index 9fec7782cc67..d0eb250d3913 100644
--- a/python/pyspark/ml/torch/distributor.py
+++ b/python/pyspark/ml/torch/distributor.py
@@ -276,9 +276,7 @@ class Distributor:
)
)
return
- raise RuntimeError(
- textwrap.dedent(
- f"""
+ raise RuntimeError(textwrap.dedent(f"""
This cluster has TLS encryption enabled;
however, {name} does not support
data encryption in transit. To override
@@ -287,9 +285,7 @@ class Distributor:
to 'true' in the Spark configuration. Please note this
will cause model parameters and possibly training
data to be sent between nodes unencrypted.
- """
- )
- )
+ """))
class TorchDistributor(Distributor):
diff --git a/python/pyspark/ml/torch/tests/test_distributor.py b/python/pyspark/ml/torch/tests/test_distributor.py
index 63fdc4e3be55..62399e2db68d 100644
--- a/python/pyspark/ml/torch/tests/test_distributor.py
+++ b/python/pyspark/ml/torch/tests/test_distributor.py
@@ -427,7 +427,7 @@ class TorchDistributorLocalUnitTestsMixin:
class TorchDistributorLocalUnitTests(TorchDistributorLocalUnitTestsMixin, unittest.TestCase):
@classmethod
def setUpClass(cls):
- (cls.gpu_discovery_script_file_name, cls.mnist_dir_path) = set_up_test_dirs()
+ cls.gpu_discovery_script_file_name, cls.mnist_dir_path = set_up_test_dirs()
conf = SparkConf()
for k, v in get_local_mode_conf().items():
conf = conf.set(k, v)
@@ -451,7 +451,7 @@ class TorchDistributorLocalUnitTests(TorchDistributorLocalUnitTestsMixin, unitte
class TorchDistributorLocalUnitTestsII(TorchDistributorLocalUnitTestsMixin, unittest.TestCase):
@classmethod
def setUpClass(cls):
- (cls.gpu_discovery_script_file_name, cls.mnist_dir_path) = set_up_test_dirs()
+ cls.gpu_discovery_script_file_name, cls.mnist_dir_path = set_up_test_dirs()
conf = SparkConf()
for k, v in get_local_mode_conf().items():
conf = conf.set(k, v)
@@ -526,7 +526,7 @@ class TorchDistributorDistributedUnitTests(
):
@classmethod
def setUpClass(cls):
- (cls.gpu_discovery_script_file_name, cls.mnist_dir_path) = set_up_test_dirs()
+ cls.gpu_discovery_script_file_name, cls.mnist_dir_path = set_up_test_dirs()
conf = SparkConf()
for k, v in get_distributed_mode_conf().items():
conf = conf.set(k, v)
diff --git a/python/pyspark/ml/tuning.py b/python/pyspark/ml/tuning.py
index b4d9747e4249..00e317375c40 100644
--- a/python/pyspark/ml/tuning.py
+++ b/python/pyspark/ml/tuning.py
@@ -169,12 +169,10 @@ class ParamGridBuilder:
return self
@overload
- def baseOn(self, __args: "ParamMap") -> "ParamGridBuilder":
- ...
+ def baseOn(self, __args: "ParamMap") -> "ParamGridBuilder": ...
@overload
- def baseOn(self, *args: Tuple[Param, Any]) -> "ParamGridBuilder":
- ...
+ def baseOn(self, *args: Tuple[Param, Any]) -> "ParamGridBuilder": ...
@since("1.4.0")
def baseOn(self, *args: Union["ParamMap", Tuple[Param, Any]]) -> "ParamGridBuilder":
@@ -853,9 +851,10 @@ class CrossValidator(
validation = datasets[i][1]
train = datasets[i][0]
- with _cache_spark_dataset(train) as train, _cache_spark_dataset(
- validation
- ) as validation:
+ with (
+ _cache_spark_dataset(train) as train,
+ _cache_spark_dataset(validation) as validation,
+ ):
tasks = map(
inheritable_thread_target(dataset.sparkSession),
_parallelFitTasks(est, train, eva, validation, epm, collectSubModelsParam),
@@ -1758,7 +1757,7 @@ if __name__ == "__main__":
sc = spark.sparkContext
globs["sc"] = sc
globs["spark"] = spark
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
spark.stop()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/ml/util.py b/python/pyspark/ml/util.py
index 5849f4a6dad8..c289078cc43e 100644
--- a/python/pyspark/ml/util.py
+++ b/python/pyspark/ml/util.py
@@ -188,7 +188,7 @@ def try_remote_fit(f: FuncT) -> FuncT:
dataset=input,
)
)
- (_, properties, _) = client.execute_command(command)
+ _, properties, _ = client.execute_command(command)
model_info = deserialize(properties)
if warning_msg := getattr(model_info, "warning_message", None):
_logger.warning(warning_msg)
@@ -305,7 +305,7 @@ def try_remote_call(f: FuncT) -> FuncT:
command.ml_command.fetch.CopyFrom(
pb2.Fetch(obj_ref=pb2.ObjectRef(id=obj_ref), methods=methods)
)
- (_, properties, _) = session.client.execute_command(command)
+ _, properties, _ = session.client.execute_command(command)
ml_command_result = properties["ml_command_result"]
if ml_command_result.HasField("summary"):
summary = ml_command_result.summary
@@ -476,7 +476,7 @@ def try_remote_evaluate(f: FuncT) -> FuncT:
dataset=input,
)
)
- (_, properties, _) = client.execute_command(command)
+ _, properties, _ = client.execute_command(command)
return deserialize(properties)
else:
return f(self, dataset)
diff --git a/python/pyspark/ml/wrapper.py b/python/pyspark/ml/wrapper.py
index abb7c7625f1c..cd8df055daa8 100644
--- a/python/pyspark/ml/wrapper.py
+++ b/python/pyspark/ml/wrapper.py
@@ -34,7 +34,6 @@ from pyspark.ml.param import Param, Params
from pyspark.ml.util import _jvm
from pyspark.ml.common import inherit_doc, _java2py, _py2java
-
if TYPE_CHECKING:
from pyspark.ml._typing import ParamMap
from py4j.java_gateway import JavaObject, JavaClass
diff --git a/python/pyspark/mllib/__init__.py b/python/pyspark/mllib/__init__.py
index 392a693eaefe..fdf35368ade9 100644
--- a/python/pyspark/mllib/__init__.py
+++ b/python/pyspark/mllib/__init__.py
@@ -21,6 +21,7 @@ RDD-based machine learning APIs for Python (in maintenance mode).
The `pyspark.mllib` package is in maintenance mode as of the Spark 2.0.0 release to encourage
migration to the DataFrame-based APIs under the `pyspark.ml` package.
"""
+
# MLlib currently needs NumPy 1.4+, so complain if lower
import numpy
diff --git a/python/pyspark/mllib/classification.py b/python/pyspark/mllib/classification.py
index 3a7c40d15fdf..3d4350d7c3bf 100644
--- a/python/pyspark/mllib/classification.py
+++ b/python/pyspark/mllib/classification.py
@@ -91,12 +91,10 @@ class LinearClassificationModel(LinearModel):
self._threshold = None
@overload
- def predict(self, test: "VectorLike") -> Union[int, float]:
- ...
+ def predict(self, test: "VectorLike") -> Union[int, float]: ...
@overload
- def predict(self, test: RDD["VectorLike"]) -> RDD[Union[int, float]]:
- ...
+ def predict(self, test: RDD["VectorLike"]) -> RDD[Union[int, float]]: ...
def predict(
self, test: Union["VectorLike", RDD["VectorLike"]]
@@ -111,7 +109,6 @@ class LinearClassificationModel(LinearModel):
class LogisticRegressionModel(LinearClassificationModel):
-
"""
Classification model trained using Multinomial/Binary Logistic
Regression.
@@ -231,12 +228,10 @@ class LogisticRegressionModel(LinearClassificationModel):
return self._numClasses
@overload
- def predict(self, x: "VectorLike") -> Union[int, float]:
- ...
+ def predict(self, x: "VectorLike") -> Union[int, float]: ...
@overload
- def predict(self, x: RDD["VectorLike"]) -> RDD[Union[int, float]]:
- ...
+ def predict(self, x: RDD["VectorLike"]) -> RDD[Union[int, float]]: ...
def predict(
self, x: Union["VectorLike", RDD["VectorLike"]]
@@ -526,7 +521,6 @@ class LogisticRegressionWithLBFGS:
class SVMModel(LinearClassificationModel):
-
"""
Model for Support Vector Machines (SVMs).
@@ -588,12 +582,10 @@ class SVMModel(LinearClassificationModel):
self._threshold = 0.0
@overload
- def predict(self, x: "VectorLike") -> Union[int, float]:
- ...
+ def predict(self, x: "VectorLike") -> Union[int, float]: ...
@overload
- def predict(self, x: RDD["VectorLike"]) -> RDD[Union[int, float]]:
- ...
+ def predict(self, x: RDD["VectorLike"]) -> RDD[Union[int, float]]: ...
def predict(
self, x: Union["VectorLike", RDD["VectorLike"]]
@@ -730,7 +722,6 @@ class SVMWithSGD:
@inherit_doc
class NaiveBayesModel(Saveable, Loader["NaiveBayesModel"]):
-
"""
Model for Naive Bayes classifiers.
@@ -793,12 +784,10 @@ class NaiveBayesModel(Saveable, Loader["NaiveBayesModel"]):
self.theta = theta
@overload
- def predict(self, x: "VectorLike") -> numpy.float64:
- ...
+ def predict(self, x: "VectorLike") -> numpy.float64: ...
@overload
- def predict(self, x: RDD["VectorLike"]) -> RDD[numpy.float64]:
- ...
+ def predict(self, x: RDD["VectorLike"]) -> RDD[numpy.float64]: ...
@since("0.9.0")
def predict(
@@ -981,7 +970,7 @@ def _test() -> None:
SparkSession.builder.master("local[4]").appName("mllib.classification tests").getOrCreate()
)
globs["sc"] = spark.sparkContext
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
spark.stop()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/mllib/clustering.py b/python/pyspark/mllib/clustering.py
index 4fac735d7ed7..e3653635f3a6 100644
--- a/python/pyspark/mllib/clustering.py
+++ b/python/pyspark/mllib/clustering.py
@@ -93,12 +93,10 @@ class BisectingKMeansModel(JavaModelWrapper):
return self.call("k")
@overload
- def predict(self, x: "VectorLike") -> int:
- ...
+ def predict(self, x: "VectorLike") -> int: ...
@overload
- def predict(self, x: RDD["VectorLike"]) -> RDD[int]:
- ...
+ def predict(self, x: RDD["VectorLike"]) -> RDD[int]: ...
def predict(self, x: Union["VectorLike", RDD["VectorLike"]]) -> Union[int, RDD[int]]:
"""
@@ -221,7 +219,6 @@ class BisectingKMeans:
@inherit_doc
class KMeansModel(Saveable, Loader["KMeansModel"]):
-
"""A clustering model derived from the k-means method.
.. versionadded:: 0.9.0
@@ -294,12 +291,10 @@ class KMeansModel(Saveable, Loader["KMeansModel"]):
return len(self.centers)
@overload
- def predict(self, x: "VectorLike") -> int:
- ...
+ def predict(self, x: "VectorLike") -> int: ...
@overload
- def predict(self, x: RDD["VectorLike"]) -> RDD[int]:
- ...
+ def predict(self, x: RDD["VectorLike"]) -> RDD[int]: ...
def predict(self, x: Union["VectorLike", RDD["VectorLike"]]) -> Union[int, RDD[int]]:
"""
@@ -464,7 +459,6 @@ class KMeans:
@inherit_doc
class GaussianMixtureModel(JavaModelWrapper, JavaSaveable, JavaLoader["GaussianMixtureModel"]):
-
"""
A clustering model derived from the Gaussian Mixture Model method.
@@ -559,12 +553,10 @@ class GaussianMixtureModel(JavaModelWrapper, JavaSaveable, JavaLoader["GaussianM
return len(self.weights)
@overload
- def predict(self, x: "VectorLike") -> np.int64:
- ...
+ def predict(self, x: "VectorLike") -> np.int64: ...
@overload
- def predict(self, x: RDD["VectorLike"]) -> RDD[int]:
- ...
+ def predict(self, x: RDD["VectorLike"]) -> RDD[int]: ...
def predict(self, x: Union["VectorLike", RDD["VectorLike"]]) -> Union[np.int64, RDD[int]]:
"""
@@ -592,12 +584,10 @@ class GaussianMixtureModel(JavaModelWrapper, JavaSaveable, JavaLoader["GaussianM
return z.argmax()
@overload
- def predictSoft(self, x: "VectorLike") -> np.ndarray:
- ...
+ def predictSoft(self, x: "VectorLike") -> np.ndarray: ...
@overload
- def predictSoft(self, x: RDD["VectorLike"]) -> RDD[pyarray.array]:
- ...
+ def predictSoft(self, x: RDD["VectorLike"]) -> RDD[pyarray.array]: ...
def predictSoft(
self, x: Union["VectorLike", RDD["VectorLike"]]
@@ -724,7 +714,6 @@ class GaussianMixture:
class PowerIterationClusteringModel(
JavaModelWrapper, JavaSaveable, JavaLoader["PowerIterationClusteringModel"]
):
-
"""
Model produced by :py:class:`PowerIterationClustering`.
@@ -1112,7 +1101,6 @@ class StreamingKMeans:
class LDAModel(JavaModelWrapper, JavaSaveable, Loader["LDAModel"]):
-
"""A clustering model derived from the LDA method.
Latent Dirichlet Allocation (LDA), a topic model designed for text documents.
@@ -1308,7 +1296,7 @@ def _test() -> None:
pass
globs = pyspark.mllib.clustering.__dict__.copy()
globs["sc"] = SparkContext("local[4]", "PythonTest", batchSize=2)
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
globs["sc"].stop()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/mllib/evaluation.py b/python/pyspark/mllib/evaluation.py
index 1c744e1ddb98..1990668f852a 100644
--- a/python/pyspark/mllib/evaluation.py
+++ b/python/pyspark/mllib/evaluation.py
@@ -680,7 +680,7 @@ def _test() -> None:
globs = pyspark.mllib.evaluation.__dict__.copy()
spark = SparkSession.builder.master("local[4]").appName("mllib.evaluation tests").getOrCreate()
globs["sc"] = spark.sparkContext
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
spark.stop()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/mllib/feature.py b/python/pyspark/mllib/feature.py
index 0d988dcb065a..685a82cf6693 100644
--- a/python/pyspark/mllib/feature.py
+++ b/python/pyspark/mllib/feature.py
@@ -18,6 +18,7 @@
"""
Python package for feature in MLlib.
"""
+
import sys
import warnings
from typing import Dict, Hashable, Iterable, List, Optional, Tuple, Union, overload, TYPE_CHECKING
@@ -58,12 +59,10 @@ class VectorTransformer:
"""
@overload
- def transform(self, vector: "VectorLike") -> Vector:
- ...
+ def transform(self, vector: "VectorLike") -> Vector: ...
@overload
- def transform(self, vector: RDD["VectorLike"]) -> RDD[Vector]:
- ...
+ def transform(self, vector: RDD["VectorLike"]) -> RDD[Vector]: ...
def transform(
self, vector: Union["VectorLike", RDD["VectorLike"]]
@@ -118,12 +117,10 @@ class Normalizer(VectorTransformer):
self.p = float(p)
@overload
- def transform(self, vector: "VectorLike") -> Vector:
- ...
+ def transform(self, vector: "VectorLike") -> Vector: ...
@overload
- def transform(self, vector: RDD["VectorLike"]) -> RDD[Vector]:
- ...
+ def transform(self, vector: RDD["VectorLike"]) -> RDD[Vector]: ...
def transform(
self, vector: Union["VectorLike", RDD["VectorLike"]]
@@ -157,12 +154,10 @@ class JavaVectorTransformer(JavaModelWrapper, VectorTransformer):
"""
@overload
- def transform(self, vector: "VectorLike") -> Vector:
- ...
+ def transform(self, vector: "VectorLike") -> Vector: ...
@overload
- def transform(self, vector: RDD["VectorLike"]) -> RDD[Vector]:
- ...
+ def transform(self, vector: RDD["VectorLike"]) -> RDD[Vector]: ...
def transform(
self, vector: Union["VectorLike", RDD["VectorLike"]]
@@ -196,12 +191,10 @@ class StandardScalerModel(JavaVectorTransformer):
"""
@overload
- def transform(self, vector: "VectorLike") -> Vector:
- ...
+ def transform(self, vector: "VectorLike") -> Vector: ...
@overload
- def transform(self, vector: RDD["VectorLike"]) -> RDD[Vector]:
- ...
+ def transform(self, vector: RDD["VectorLike"]) -> RDD[Vector]: ...
def transform(
self, vector: Union["VectorLike", RDD["VectorLike"]]
@@ -356,12 +349,10 @@ class ChiSqSelectorModel(JavaVectorTransformer):
"""
@overload
- def transform(self, vector: "VectorLike") -> Vector:
- ...
+ def transform(self, vector: "VectorLike") -> Vector: ...
@overload
- def transform(self, vector: RDD["VectorLike"]) -> RDD[Vector]:
- ...
+ def transform(self, vector: RDD["VectorLike"]) -> RDD[Vector]: ...
def transform(
self, vector: Union["VectorLike", RDD["VectorLike"]]
@@ -627,12 +618,10 @@ class HashingTF:
return hash(term) % self.numFeatures
@overload
- def transform(self, document: Iterable[Hashable]) -> Vector:
- ...
+ def transform(self, document: Iterable[Hashable]) -> Vector: ...
@overload
- def transform(self, document: RDD[Iterable[Hashable]]) -> RDD[Vector]:
- ...
+ def transform(self, document: RDD[Iterable[Hashable]]) -> RDD[Vector]: ...
@since("1.2.0")
def transform(
@@ -661,12 +650,10 @@ class IDFModel(JavaVectorTransformer):
"""
@overload
- def transform(self, x: "VectorLike") -> Vector:
- ...
+ def transform(self, x: "VectorLike") -> Vector: ...
@overload
- def transform(self, x: RDD["VectorLike"]) -> RDD[Vector]:
- ...
+ def transform(self, x: RDD["VectorLike"]) -> RDD[Vector]: ...
def transform(self, x: Union["VectorLike", RDD["VectorLike"]]) -> Union[Vector, RDD[Vector]]:
"""
@@ -1042,12 +1029,10 @@ class ElementwiseProduct(VectorTransformer):
self.scalingVector = _convert_to_vector(scalingVector)
@overload
- def transform(self, vector: "VectorLike") -> Vector:
- ...
+ def transform(self, vector: "VectorLike") -> Vector: ...
@overload
- def transform(self, vector: RDD["VectorLike"]) -> RDD[Vector]:
- ...
+ def transform(self, vector: RDD["VectorLike"]) -> RDD[Vector]: ...
def transform(
self, vector: Union["VectorLike", RDD["VectorLike"]]
@@ -1072,7 +1057,7 @@ def _test() -> None:
globs = globals().copy()
spark = SparkSession.builder.master("local[4]").appName("mllib.feature tests").getOrCreate()
globs["sc"] = spark.sparkContext
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
spark.stop()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/mllib/fpm.py b/python/pyspark/mllib/fpm.py
index 19da87da3c27..3f4d36884d94 100644
--- a/python/pyspark/mllib/fpm.py
+++ b/python/pyspark/mllib/fpm.py
@@ -216,7 +216,7 @@ def _test() -> None:
temp_path = tempfile.mkdtemp()
globs["temp_path"] = temp_path
try:
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
spark.stop()
finally:
from shutil import rmtree
diff --git a/python/pyspark/mllib/linalg/__init__.py b/python/pyspark/mllib/linalg/__init__.py
index 1613e3d2cc8d..a32ddc1b84f5 100644
--- a/python/pyspark/mllib/linalg/__init__.py
+++ b/python/pyspark/mllib/linalg/__init__.py
@@ -526,12 +526,10 @@ class DenseVector(Vector):
return self.array
@overload
- def __getitem__(self, item: int) -> np.float64:
- ...
+ def __getitem__(self, item: int) -> np.float64: ...
@overload
- def __getitem__(self, item: slice) -> np.ndarray:
- ...
+ def __getitem__(self, item: slice) -> np.ndarray: ...
def __getitem__(self, item: Union[int, slice]) -> Union[np.float64, np.ndarray]:
return self.array[item]
@@ -606,24 +604,19 @@ class SparseVector(Vector):
"""
@overload
- def __init__(self, size: int, __indices: bytes, __values: bytes):
- ...
+ def __init__(self, size: int, __indices: bytes, __values: bytes): ...
@overload
- def __init__(self, size: int, *args: Tuple[int, float]):
- ...
+ def __init__(self, size: int, *args: Tuple[int, float]): ...
@overload
- def __init__(self, size: int, __indices: Iterable[int], __values: Iterable[float]):
- ...
+ def __init__(self, size: int, __indices: Iterable[int], __values: Iterable[float]): ...
@overload
- def __init__(self, size: int, __pairs: Iterable[Tuple[int, float]]):
- ...
+ def __init__(self, size: int, __pairs: Iterable[Tuple[int, float]]): ...
@overload
- def __init__(self, size: int, __map: Dict[int, float]):
- ...
+ def __init__(self, size: int, __map: Dict[int, float]): ...
def __init__(
self,
@@ -986,7 +979,6 @@ class SparseVector(Vector):
class Vectors:
-
"""
Factory methods for working with vectors.
@@ -1000,28 +992,23 @@ class Vectors:
@staticmethod
@overload
- def sparse(size: int, __indices: bytes, __values: bytes) -> SparseVector:
- ...
+ def sparse(size: int, __indices: bytes, __values: bytes) -> SparseVector: ...
@staticmethod
@overload
- def sparse(size: int, *args: Tuple[int, float]) -> SparseVector:
- ...
+ def sparse(size: int, *args: Tuple[int, float]) -> SparseVector: ...
@staticmethod
@overload
- def sparse(size: int, __indices: Iterable[int], __values: Iterable[float]) -> SparseVector:
- ...
+ def sparse(size: int, __indices: Iterable[int], __values: Iterable[float]) -> SparseVector: ...
@staticmethod
@overload
- def sparse(size: int, __pairs: Iterable[Tuple[int, float]]) -> SparseVector:
- ...
+ def sparse(size: int, __pairs: Iterable[Tuple[int, float]]) -> SparseVector: ...
@staticmethod
@overload
- def sparse(size: int, __map: Dict[int, float]) -> SparseVector:
- ...
+ def sparse(size: int, __map: Dict[int, float]) -> SparseVector: ...
@staticmethod
def sparse(
@@ -1056,18 +1043,15 @@ class Vectors:
@overload
@staticmethod
- def dense(*elements: float) -> DenseVector:
- ...
+ def dense(*elements: float) -> DenseVector: ...
@overload
@staticmethod
- def dense(__arr: bytes) -> DenseVector:
- ...
+ def dense(__arr: bytes) -> DenseVector: ...
@overload
@staticmethod
- def dense(__arr: Iterable[float]) -> DenseVector:
- ...
+ def dense(__arr: Iterable[float]) -> DenseVector: ...
@staticmethod
def dense(*elements: Union[float, bytes, np.ndarray, Iterable[float]]) -> DenseVector:
@@ -1645,7 +1629,7 @@ def _test() -> None:
numpy.set_printoptions(legacy="1.13")
except TypeError:
pass
- (failure_count, test_count) = doctest.testmod(optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(optionflags=doctest.ELLIPSIS)
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/mllib/linalg/distributed.py b/python/pyspark/mllib/linalg/distributed.py
index aa98f959798c..f5402bdbe96d 100644
--- a/python/pyspark/mllib/linalg/distributed.py
+++ b/python/pyspark/mllib/linalg/distributed.py
@@ -1652,7 +1652,7 @@ def _test() -> None:
)
globs["sc"] = spark.sparkContext
globs["Matrices"] = Matrices
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
spark.stop()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/mllib/random.py b/python/pyspark/mllib/random.py
index dbe1048a64b3..faafb024abb4 100644
--- a/python/pyspark/mllib/random.py
+++ b/python/pyspark/mllib/random.py
@@ -30,7 +30,6 @@ from pyspark.core.context import SparkContext
from pyspark.core.rdd import RDD
from pyspark.mllib.linalg import Vector
-
__all__ = [
"RandomRDDs",
]
@@ -688,7 +687,7 @@ def _test() -> None:
# even in these small test examples:
spark = SparkSession.builder.master("local[2]").appName("mllib.random tests").getOrCreate()
globs["sc"] = spark.sparkContext
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
spark.stop()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/mllib/recommendation.py b/python/pyspark/mllib/recommendation.py
index fa3d1a73f35a..f6a0d39a74c6 100644
--- a/python/pyspark/mllib/recommendation.py
+++ b/python/pyspark/mllib/recommendation.py
@@ -55,7 +55,6 @@ class Rating(NamedTuple):
class MatrixFactorizationModel(
JavaModelWrapper, JavaSaveable, JavaLoader["MatrixFactorizationModel"]
):
-
"""A matrix factorisation model trained by regularized alternating
least-squares.
@@ -379,7 +378,7 @@ def _test() -> None:
sc = SparkContext("local[4]", "PythonTest")
globs["sc"] = sc
globs["sqlContext"] = SQLContext(sc)
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
globs["sc"].stop()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/mllib/regression.py b/python/pyspark/mllib/regression.py
index a69f8c00b221..a5dde25d32fc 100644
--- a/python/pyspark/mllib/regression.py
+++ b/python/pyspark/mllib/regression.py
@@ -65,7 +65,6 @@ __all__ = [
class LabeledPoint:
-
"""
Class that represents the features and labels of a data point.
@@ -99,7 +98,6 @@ class LabeledPoint:
class LinearModel:
-
"""
A linear model that has a vector of coefficients and an intercept.
@@ -135,7 +133,6 @@ class LinearModel:
@inherit_doc
class LinearRegressionModelBase(LinearModel):
-
"""A linear regression model.
.. versionadded:: 0.9.0
@@ -151,12 +148,10 @@ class LinearRegressionModelBase(LinearModel):
"""
@overload
- def predict(self, x: "VectorLike") -> float:
- ...
+ def predict(self, x: "VectorLike") -> float: ...
@overload
- def predict(self, x: RDD["VectorLike"]) -> RDD[float]:
- ...
+ def predict(self, x: RDD["VectorLike"]) -> RDD[float]: ...
def predict(self, x: Union["VectorLike", RDD["VectorLike"]]) -> Union[float, RDD[float]]:
"""
@@ -173,7 +168,6 @@ class LinearRegressionModelBase(LinearModel):
@inherit_doc
class LinearRegressionModel(LinearRegressionModelBase):
-
"""A linear regression model derived from a least-squares fit.
.. versionadded:: 0.9.0
@@ -384,7 +378,6 @@ class LinearRegressionWithSGD:
@inherit_doc
class LassoModel(LinearRegressionModelBase):
-
"""A linear regression model derived from a least-squares fit with
an l_1 penalty term.
@@ -562,7 +555,6 @@ class LassoWithSGD:
@inherit_doc
class RidgeRegressionModel(LinearRegressionModelBase):
-
"""A linear regression model derived from a least-squares fit with
an l_2 penalty term.
@@ -743,7 +735,6 @@ class RidgeRegressionWithSGD:
class IsotonicRegressionModel(Saveable, Loader["IsotonicRegressionModel"]):
-
"""
Regression model for isotonic regression.
@@ -791,20 +782,16 @@ class IsotonicRegressionModel(Saveable, Loader["IsotonicRegressionModel"]):
self.isotonic = isotonic
@overload
- def predict(self, x: float) -> np.float64:
- ...
+ def predict(self, x: float) -> np.float64: ...
@overload
- def predict(self, x: "VectorLike") -> np.ndarray:
- ...
+ def predict(self, x: "VectorLike") -> np.ndarray: ...
@overload
- def predict(self, x: RDD[float]) -> RDD[np.float64]:
- ...
+ def predict(self, x: RDD[float]) -> RDD[np.float64]: ...
@overload
- def predict(self, x: RDD["VectorLike"]) -> RDD[np.ndarray]:
- ...
+ def predict(self, x: RDD["VectorLike"]) -> RDD[np.ndarray]: ...
def predict(
self, x: Union[float, "VectorLike", RDD[float], RDD["VectorLike"]]
@@ -1057,7 +1044,7 @@ def _test() -> None:
globs = pyspark.mllib.regression.__dict__.copy()
spark = SparkSession.builder.master("local[2]").appName("mllib.regression tests").getOrCreate()
globs["sc"] = spark.sparkContext
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
spark.stop()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/mllib/stat/_statistics.py b/python/pyspark/mllib/stat/_statistics.py
index e993ced1a419..34a4a35865fe 100644
--- a/python/pyspark/mllib/stat/_statistics.py
+++ b/python/pyspark/mllib/stat/_statistics.py
@@ -34,7 +34,6 @@ __all__ = ["MultivariateStatisticalSummary", "Statistics"]
class MultivariateStatisticalSummary(JavaModelWrapper):
-
"""
Trait for multivariate statistical summary of a data matrix.
"""
@@ -106,13 +105,11 @@ class Statistics:
@overload
@staticmethod
- def corr(x: RDD[Vector], *, method: Optional["CorrMethodType"] = ...) -> Matrix:
- ...
+ def corr(x: RDD[Vector], *, method: Optional["CorrMethodType"] = ...) -> Matrix: ...
@overload
@staticmethod
- def corr(x: RDD[float], y: RDD[float], method: Optional["CorrMethodType"] = ...) -> float:
- ...
+ def corr(x: RDD[float], y: RDD[float], method: Optional["CorrMethodType"] = ...) -> float: ...
@staticmethod
def corr(
@@ -200,18 +197,15 @@ class Statistics:
@overload
@staticmethod
- def chiSqTest(observed: Matrix) -> ChiSqTestResult:
- ...
+ def chiSqTest(observed: Matrix) -> ChiSqTestResult: ...
@overload
@staticmethod
- def chiSqTest(observed: Vector, expected: Optional[Vector] = ...) -> ChiSqTestResult:
- ...
+ def chiSqTest(observed: Vector, expected: Optional[Vector] = ...) -> ChiSqTestResult: ...
@overload
@staticmethod
- def chiSqTest(observed: RDD[LabeledPoint]) -> List[ChiSqTestResult]:
- ...
+ def chiSqTest(observed: RDD[LabeledPoint]) -> List[ChiSqTestResult]: ...
@staticmethod
def chiSqTest(
@@ -400,7 +394,7 @@ def _test() -> None:
SparkSession.builder.master("local[4]").appName("mllib.stat.statistics tests").getOrCreate()
)
globs["sc"] = spark.sparkContext
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
spark.stop()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/mllib/stat/distribution.py b/python/pyspark/mllib/stat/distribution.py
index 95e94ce1cb94..92dbe2f70f13 100644
--- a/python/pyspark/mllib/stat/distribution.py
+++ b/python/pyspark/mllib/stat/distribution.py
@@ -23,7 +23,6 @@ from pyspark.mllib.linalg import Matrix, Vector
class MultivariateGaussian(NamedTuple):
-
"""Represents a (mu, sigma) tuple
Examples
diff --git a/python/pyspark/mllib/stat/test.py b/python/pyspark/mllib/stat/test.py
index b84ecfc6884f..4afd23f8df06 100644
--- a/python/pyspark/mllib/stat/test.py
+++ b/python/pyspark/mllib/stat/test.py
@@ -19,7 +19,6 @@ from typing import Generic, Tuple, TypeVar
from pyspark.mllib.common import inherit_doc, JavaModelWrapper
-
__all__ = ["ChiSqTestResult", "KolmogorovSmirnovTestResult"]
DF = TypeVar("DF", int, float, Tuple[int, ...], Tuple[float, ...])
diff --git a/python/pyspark/mllib/tests/test_algorithms.py b/python/pyspark/mllib/tests/test_algorithms.py
index 9bae52f0ff70..2cd15a5bf8ab 100644
--- a/python/pyspark/mllib/tests/test_algorithms.py
+++ b/python/pyspark/mllib/tests/test_algorithms.py
@@ -31,7 +31,6 @@ from pyspark.testing.utils import eventually
class ListTests(MLlibTestCase):
-
"""
Test MLlib algorithms on plain lists, to make sure they're passed through
as NumPy arrays.
diff --git a/python/pyspark/mllib/tests/test_linalg.py b/python/pyspark/mllib/tests/test_linalg.py
index 27c4787dd317..634d9c431b6d 100644
--- a/python/pyspark/mllib/tests/test_linalg.py
+++ b/python/pyspark/mllib/tests/test_linalg.py
@@ -501,7 +501,6 @@ class MatrixUDTTests(MLlibTestCase):
@unittest.skipIf(not have_scipy, "SciPy not installed")
class SciPyTests(MLlibTestCase):
-
"""
Test both vector operations and MLlib algorithms with SciPy sparse matrices,
if SciPy is available.
diff --git a/python/pyspark/mllib/tree.py b/python/pyspark/mllib/tree.py
index 9e945d8a1b94..2f3a85d83ad8 100644
--- a/python/pyspark/mllib/tree.py
+++ b/python/pyspark/mllib/tree.py
@@ -47,12 +47,10 @@ class TreeEnsembleModel(JavaModelWrapper, JavaSaveable):
"""
@overload
- def predict(self, x: "VectorLike") -> float:
- ...
+ def predict(self, x: "VectorLike") -> float: ...
@overload
- def predict(self, x: RDD["VectorLike"]) -> RDD[float]:
- ...
+ def predict(self, x: RDD["VectorLike"]) -> RDD[float]: ...
def predict(self, x: Union["VectorLike", RDD["VectorLike"]]) -> Union[float, RDD[float]]:
"""
@@ -105,12 +103,10 @@ class DecisionTreeModel(JavaModelWrapper, JavaSaveable, JavaLoader["DecisionTree
"""
@overload
- def predict(self, x: "VectorLike") -> float:
- ...
+ def predict(self, x: "VectorLike") -> float: ...
@overload
- def predict(self, x: RDD["VectorLike"]) -> RDD[float]:
- ...
+ def predict(self, x: RDD["VectorLike"]) -> RDD[float]: ...
def predict(self, x: Union["VectorLike", RDD["VectorLike"]]) -> Union[float, RDD[float]]:
"""
@@ -876,7 +872,7 @@ def _test() -> None:
spark = SparkSession.builder.master("local[4]").appName("mllib.tree tests").getOrCreate()
globs["sc"] = spark.sparkContext
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
globs=globs, optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE
)
spark.stop()
diff --git a/python/pyspark/mllib/util.py b/python/pyspark/mllib/util.py
index 65b25f8add6d..699909d430ea 100644
--- a/python/pyspark/mllib/util.py
+++ b/python/pyspark/mllib/util.py
@@ -40,7 +40,6 @@ if TYPE_CHECKING:
class MLUtils:
-
"""
Helper methods to load, save and pre-process data used in MLlib.
@@ -645,7 +644,7 @@ def _test() -> None:
spark = SparkSession.builder.master("local[2]").appName("mllib.util tests").getOrCreate()
globs["spark"] = spark
globs["sc"] = spark.sparkContext
- (failure_count, test_count) = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
+ failure_count, test_count = doctest.testmod(globs=globs, optionflags=doctest.ELLIPSIS)
spark.stop()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/pandas/_typing.py b/python/pyspark/pandas/_typing.py
index 0e5b86a53fe0..51d1233fae25 100644
--- a/python/pyspark/pandas/_typing.py
+++ b/python/pyspark/pandas/_typing.py
@@ -21,7 +21,6 @@ from typing import Any, Tuple, TypeVar, Union, TYPE_CHECKING
import numpy as np
from pandas.api.extensions import ExtensionDtype
-
if TYPE_CHECKING:
from pyspark.pandas.base import IndexOpsMixin
from pyspark.pandas.frame import DataFrame
diff --git a/python/pyspark/pandas/accessors.py b/python/pyspark/pandas/accessors.py
index ea6d33c8fa18..a0edf34da585 100644
--- a/python/pyspark/pandas/accessors.py
+++ b/python/pyspark/pandas/accessors.py
@@ -17,6 +17,7 @@
"""
pandas-on-Spark specific features.
"""
+
import inspect
from typing import Any, Callable, Optional, Tuple, Union, TYPE_CHECKING, cast, List
from types import FunctionType
@@ -974,7 +975,7 @@ def _test() -> None:
.appName("pyspark.pandas.accessors tests")
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.accessors,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/base.py b/python/pyspark/pandas/base.py
index 5aa45e1185b6..fe14f2becc92 100644
--- a/python/pyspark/pandas/base.py
+++ b/python/pyspark/pandas/base.py
@@ -18,6 +18,7 @@
"""
Base and utility classes for pandas-on-Spark objects.
"""
+
import warnings
from abc import ABCMeta, abstractmethod
from functools import wraps, partial
@@ -132,9 +133,11 @@ def align_diff_index_ops(
column_op(func)(
this_index_ops.to_series().reset_index(drop=True),
*[
- arg.to_series().reset_index(drop=True)
- if isinstance(arg, Index)
- else arg
+ (
+ arg.to_series().reset_index(drop=True)
+ if isinstance(arg, Index)
+ else arg
+ )
for arg in args
],
).sort_index(),
@@ -1801,7 +1804,7 @@ def _test() -> None:
spark = (
SparkSession.builder.master("local[4]").appName("pyspark.pandas.base tests").getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.base,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/categorical.py b/python/pyspark/pandas/categorical.py
index 024912ebc2eb..36f10f3426a1 100644
--- a/python/pyspark/pandas/categorical.py
+++ b/python/pyspark/pandas/categorical.py
@@ -784,7 +784,7 @@ def _test() -> None:
.appName("pyspark.pandas.categorical tests")
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.categorical,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/config.py b/python/pyspark/pandas/config.py
index 2049e1fcef5f..54c8d3afae39 100644
--- a/python/pyspark/pandas/config.py
+++ b/python/pyspark/pandas/config.py
@@ -18,6 +18,7 @@
"""
Infrastructure of options for pandas-on-Spark.
"""
+
from contextlib import contextmanager
import json
from typing import Any, Callable, Dict, Iterator, List, Tuple, Union, Optional
@@ -26,7 +27,6 @@ from pyspark._globals import _NoValue, _NoValueType
from pyspark.sql.session import SparkSession
from pyspark.pandas.utils import default_session
-
__all__ = ["get_option", "set_option", "reset_option", "options", "option_context"]
@@ -571,7 +571,7 @@ def _test() -> None:
spark = (
SparkSession.builder.master("local[4]").appName("pyspark.pandas.config tests").getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.config,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/correlation.py b/python/pyspark/pandas/correlation.py
index 9327b5eb71a4..dbbbc199d432 100644
--- a/python/pyspark/pandas/correlation.py
+++ b/python/pyspark/pandas/correlation.py
@@ -21,7 +21,6 @@ from pyspark.sql import DataFrame as SparkDataFrame, functions as F
from pyspark.sql.window import Window
from pyspark.pandas.utils import verify_temp_column_name, is_ansi_mode_enabled
-
CORRELATION_VALUE_1_COLUMN = "__correlation_value_1_input__"
CORRELATION_VALUE_2_COLUMN = "__correlation_value_2_input__"
CORRELATION_CORR_OUTPUT_COLUMN = "__correlation_corr_output__"
diff --git a/python/pyspark/pandas/datetimes.py b/python/pyspark/pandas/datetimes.py
index cc07ed278233..aee37e11ab96 100644
--- a/python/pyspark/pandas/datetimes.py
+++ b/python/pyspark/pandas/datetimes.py
@@ -18,6 +18,7 @@
"""
Date/Time related functions on pandas-on-Spark Series
"""
+
from typing import Any, Optional, Union, no_type_check
import numpy as np
@@ -933,7 +934,7 @@ def _test() -> None:
.appName("pyspark.pandas.datetimes tests")
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.datetimes,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/exceptions.py b/python/pyspark/pandas/exceptions.py
index 8644d0306bf0..83283c9a4afe 100644
--- a/python/pyspark/pandas/exceptions.py
+++ b/python/pyspark/pandas/exceptions.py
@@ -18,6 +18,7 @@
"""
Exceptions/Errors used in pandas-on-Spark.
"""
+
from typing import Optional
@@ -126,7 +127,7 @@ def _test() -> None:
.appName("pyspark.pandas.exceptions tests")
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.exceptions,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/extensions.py b/python/pyspark/pandas/extensions.py
index af4d160a40fb..1764f2477065 100644
--- a/python/pyspark/pandas/extensions.py
+++ b/python/pyspark/pandas/extensions.py
@@ -374,7 +374,7 @@ def _test() -> None:
.appName("pyspark.pandas.extensions tests")
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.extensions,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/frame.py b/python/pyspark/pandas/frame.py
index 4bc008efe6de..b50bc726cd30 100644
--- a/python/pyspark/pandas/frame.py
+++ b/python/pyspark/pandas/frame.py
@@ -18,6 +18,7 @@
"""
A wrapper class for Spark DataFrame to behave like pandas DataFrame.
"""
+
from collections import defaultdict, namedtuple
from collections.abc import Mapping
import re
@@ -5717,9 +5718,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
(k if is_name_like_tuple(k) else (k,)): (
(v.spark.column, v._internal.data_fields[0])
if isinstance(v, IndexOpsMixin) and not isinstance(v, MultiIndex)
- else (v, None)
- if isinstance(v, PySparkColumn)
- else (F.lit(v), None)
+ else (v, None) if isinstance(v, PySparkColumn) else (F.lit(v), None)
)
for k, v in kwargs.items()
}
@@ -10135,9 +10134,11 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
# we also need to calculate the `std` for numeric columns
if has_numeric_type:
std_exprs = [
- F.lit(None).alias("stddev_samp({})".format(label[0]))
- if isinstance(spark_data_type, (TimestampType, TimestampNTZType))
- else F.stddev(label[0])
+ (
+ F.lit(None).alias("stddev_samp({})".format(label[0]))
+ if isinstance(spark_data_type, (TimestampType, TimestampNTZType))
+ else F.stddev(label[0])
+ )
for label, spark_data_type in zip(column_labels, spark_data_types)
]
exprs.extend(std_exprs)
@@ -11183,9 +11184,11 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
column_label_names=(
df._internal.column_label_names[:-1]
+ [
- None
- if self._internal.index_names[-1] is None
- else df._internal.column_label_names[-1]
+ (
+ None
+ if self._internal.index_names[-1] is None
+ else df._internal.column_label_names[-1]
+ )
]
),
)
@@ -14333,7 +14336,7 @@ def _test() -> None:
path = tempfile.mkdtemp()
globs["path"] = path
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.frame,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/generic.py b/python/pyspark/pandas/generic.py
index 9e36fb9225eb..92f58b878eda 100644
--- a/python/pyspark/pandas/generic.py
+++ b/python/pyspark/pandas/generic.py
@@ -18,6 +18,7 @@
"""
A base class of DataFrame/Column to behave like pandas DataFrame/Series.
"""
+
from abc import ABCMeta, abstractmethod
from functools import reduce
from typing import (
@@ -3681,7 +3682,7 @@ def _test() -> None:
path = tempfile.mkdtemp()
globs["path"] = path
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.generic,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/groupby.py b/python/pyspark/pandas/groupby.py
index ddb9cb032371..9cd3d464c242 100644
--- a/python/pyspark/pandas/groupby.py
+++ b/python/pyspark/pandas/groupby.py
@@ -18,6 +18,7 @@
"""
A wrapper for GroupedData to behave like pandas GroupBy.
"""
+
from abc import ABCMeta, abstractmethod
import inspect
from collections import defaultdict, namedtuple
@@ -4731,7 +4732,7 @@ def _test() -> None:
.appName("pyspark.pandas.groupby tests")
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.groupby,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/indexes/base.py b/python/pyspark/pandas/indexes/base.py
index 081d2836d701..4bed26bce8da 100644
--- a/python/pyspark/pandas/indexes/base.py
+++ b/python/pyspark/pandas/indexes/base.py
@@ -246,9 +246,11 @@ class Index(IndexOpsMixin):
internal = self._internal.copy(
index_spark_columns=[scol.alias(SPARK_DEFAULT_INDEX_NAME)],
index_fields=[
- field
- if field is None or field.struct_field is None
- else field.copy(name=SPARK_DEFAULT_INDEX_NAME)
+ (
+ field
+ if field is None or field.struct_field is None
+ else field.copy(name=SPARK_DEFAULT_INDEX_NAME)
+ )
],
column_labels=[],
data_spark_columns=[],
@@ -2253,9 +2255,11 @@ class Index(IndexOpsMixin):
for left, right in zip(self._internal.index_fields, other._internal.index_fields)
):
return [
- left.copy(nullable=left.nullable or right.nullable)
- if left.spark_type == right.spark_type
- else InternalField(dtype=left.dtype)
+ (
+ left.copy(nullable=left.nullable or right.nullable)
+ if left.spark_type == right.spark_type
+ else InternalField(dtype=left.dtype)
+ )
for left, right in zip(self._internal.index_fields, other._internal.index_fields)
]
elif any(
@@ -2671,7 +2675,7 @@ def _test() -> None:
.appName("pyspark.pandas.indexes.base tests")
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.indexes.base,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/indexes/category.py b/python/pyspark/pandas/indexes/category.py
index da22c029633b..d4c709fe6284 100644
--- a/python/pyspark/pandas/indexes/category.py
+++ b/python/pyspark/pandas/indexes/category.py
@@ -654,7 +654,7 @@ def _test() -> None:
.appName("pyspark.pandas.indexes.category tests")
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.indexes.category,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/indexes/datetimes.py b/python/pyspark/pandas/indexes/datetimes.py
index 51fec78e89dc..8d679de92f6d 100644
--- a/python/pyspark/pandas/indexes/datetimes.py
+++ b/python/pyspark/pandas/indexes/datetimes.py
@@ -881,7 +881,7 @@ def _test() -> None:
.appName("pyspark.pandas.indexes.datetimes tests")
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.indexes.datetimes,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/indexes/multi.py b/python/pyspark/pandas/indexes/multi.py
index 55819ce59168..d9eb17baf7d5 100644
--- a/python/pyspark/pandas/indexes/multi.py
+++ b/python/pyspark/pandas/indexes/multi.py
@@ -1279,7 +1279,7 @@ def _test() -> None:
.appName("pyspark.pandas.indexes.multi tests")
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.indexes.multi,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/indexes/timedelta.py b/python/pyspark/pandas/indexes/timedelta.py
index 762bc90f46c3..87b57d84b435 100644
--- a/python/pyspark/pandas/indexes/timedelta.py
+++ b/python/pyspark/pandas/indexes/timedelta.py
@@ -30,7 +30,6 @@ from pyspark.pandas.missing.indexes import MissingPandasLikeTimedeltaIndex
from pyspark.pandas.series import Series
from pyspark.sql import functions as F
-
HOURS_PER_DAY = 24
MINUTES_PER_HOUR = 60
SECONDS_PER_MINUTE = 60
diff --git a/python/pyspark/pandas/indexing.py b/python/pyspark/pandas/indexing.py
index 3ef01d72496f..ee73f7bbcdb8 100644
--- a/python/pyspark/pandas/indexing.py
+++ b/python/pyspark/pandas/indexing.py
@@ -18,6 +18,7 @@
"""
A loc indexer for pandas-on-Spark DataFrame/Series.
"""
+
from abc import ABCMeta, abstractmethod
from collections.abc import Iterable
from functools import reduce
@@ -278,9 +279,7 @@ class LocIndexerLike(IndexerLike, metaclass=ABCMeta):
else:
return self._select_rows_else(rows_sel)
- def _select_cols(
- self, cols_sel: Any, missing_keys: Optional[List[Name]] = None
- ) -> Tuple[
+ def _select_cols(self, cols_sel: Any, missing_keys: Optional[List[Name]] = None) -> Tuple[
List[Label],
Optional[List[PySparkColumn]],
Optional[List[InternalField]],
@@ -396,9 +395,7 @@ class LocIndexerLike(IndexerLike, metaclass=ABCMeta):
pass
@abstractmethod
- def _select_cols_by_slice(
- self, cols_sel: slice, missing_keys: Optional[List[Name]]
- ) -> Tuple[
+ def _select_cols_by_slice(self, cols_sel: slice, missing_keys: Optional[List[Name]]) -> Tuple[
List[Label],
Optional[List[PySparkColumn]],
Optional[List[InternalField]],
@@ -422,9 +419,7 @@ class LocIndexerLike(IndexerLike, metaclass=ABCMeta):
pass
@abstractmethod
- def _select_cols_else(
- self, cols_sel: Any, missing_keys: Optional[List[Name]]
- ) -> Tuple[
+ def _select_cols_else(self, cols_sel: Any, missing_keys: Optional[List[Name]]) -> Tuple[
List[Label],
Optional[List[PySparkColumn]],
Optional[List[InternalField]],
@@ -1317,9 +1312,7 @@ class LocIndexer(LocIndexerLike):
data_spark_columns = [cols_sel]
return column_labels, data_spark_columns, None, True, None
- def _select_cols_by_slice(
- self, cols_sel: slice, missing_keys: Optional[List[Name]]
- ) -> Tuple[
+ def _select_cols_by_slice(self, cols_sel: slice, missing_keys: Optional[List[Name]]) -> Tuple[
List[Label],
Optional[List[PySparkColumn]],
Optional[List[InternalField]],
@@ -1425,9 +1418,7 @@ class LocIndexer(LocIndexerLike):
return column_labels, data_spark_columns, data_fields, False, None
- def _select_cols_else(
- self, cols_sel: Any, missing_keys: Optional[List[Name]]
- ) -> Tuple[
+ def _select_cols_else(self, cols_sel: Any, missing_keys: Optional[List[Name]]) -> Tuple[
List[Label],
Optional[List[PySparkColumn]],
Optional[List[InternalField]],
@@ -1776,9 +1767,7 @@ class iLocIndexer(LocIndexerLike):
"listlike of integers, boolean array] types, got {}".format(cols_sel)
)
- def _select_cols_by_slice(
- self, cols_sel: slice, missing_keys: Optional[List[Name]]
- ) -> Tuple[
+ def _select_cols_by_slice(self, cols_sel: slice, missing_keys: Optional[List[Name]]) -> Tuple[
List[Label],
Optional[List[PySparkColumn]],
Optional[List[InternalField]],
@@ -1796,9 +1785,7 @@ class iLocIndexer(LocIndexerLike):
not_none = (
cols_sel.start
if cols_sel.start is not None
- else cols_sel.stop
- if cols_sel.stop is not None
- else cols_sel.step
+ else cols_sel.stop if cols_sel.stop is not None else cols_sel.step
)
raise TypeError(
"cannot do slice indexing with these indexers {} of {}".format(
@@ -1825,9 +1812,7 @@ class iLocIndexer(LocIndexerLike):
else:
raise TypeError("cannot perform reduce with flexible type")
- def _select_cols_else(
- self, cols_sel: Any, missing_keys: Optional[List[Name]]
- ) -> Tuple[
+ def _select_cols_else(self, cols_sel: Any, missing_keys: Optional[List[Name]]) -> Tuple[
List[Label],
Optional[List[PySparkColumn]],
Optional[List[InternalField]],
@@ -1901,7 +1886,7 @@ def _test() -> None:
.appName("pyspark.pandas.indexing tests")
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.indexing,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/internal.py b/python/pyspark/pandas/internal.py
index 61bcd618b460..9ba714dee2d8 100644
--- a/python/pyspark/pandas/internal.py
+++ b/python/pyspark/pandas/internal.py
@@ -18,6 +18,7 @@
"""
An internal immutable DataFrame with some metadata to manage indexes.
"""
+
import re
from typing import Any, Dict, List, Optional, Sequence, Tuple, Union, TYPE_CHECKING, cast
@@ -715,11 +716,15 @@ class InternalFrame:
):
schema = spark_frame.select(index_spark_columns + data_spark_columns).schema
fields = [
- InternalField.from_struct_field(struct_field)
- if field is None
- else InternalField(field.dtype, struct_field)
- if field.struct_field is None
- else field
+ (
+ InternalField.from_struct_field(struct_field)
+ if field is None
+ else (
+ InternalField(field.dtype, struct_field)
+ if field.struct_field is None
+ else field
+ )
+ )
for field, struct_field in zip(index_fields + data_fields, schema.fields)
]
index_fields = fields[: len(index_spark_columns)]
@@ -727,21 +732,29 @@ class InternalFrame:
elif any(field is None or field.struct_field is None for field in index_fields):
schema = spark_frame.select(index_spark_columns).schema
index_fields = [
- InternalField.from_struct_field(struct_field)
- if field is None
- else InternalField(field.dtype, struct_field)
- if field.struct_field is None
- else field
+ (
+ InternalField.from_struct_field(struct_field)
+ if field is None
+ else (
+ InternalField(field.dtype, struct_field)
+ if field.struct_field is None
+ else field
+ )
+ )
for field, struct_field in zip(index_fields, schema.fields)
]
elif any(field is None or field.struct_field is None for field in data_fields):
schema = spark_frame.select(data_spark_columns).schema
data_fields = [
- InternalField.from_struct_field(struct_field)
- if field is None
- else InternalField(field.dtype, struct_field)
- if field.struct_field is None
- else field
+ (
+ InternalField.from_struct_field(struct_field)
+ if field is None
+ else (
+ InternalField(field.dtype, struct_field)
+ if field.struct_field is None
+ else field
+ )
+ )
for field, struct_field in zip(data_fields, schema.fields)
]
@@ -1632,7 +1645,7 @@ def _test() -> None:
.appName("pyspark.pandas.internal tests")
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.internal,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/mlflow.py b/python/pyspark/pandas/mlflow.py
index 9db8d6285274..f15ff83b11cc 100644
--- a/python/pyspark/pandas/mlflow.py
+++ b/python/pyspark/pandas/mlflow.py
@@ -18,6 +18,7 @@
"""
MLflow-related functions to load models and apply them to pandas-on-Spark dataframes.
"""
+
from typing import List, Union
from typing import Any
@@ -216,7 +217,7 @@ def _test() -> None:
spark = (
SparkSession.builder.master("local[4]").appName("pyspark.pandas.mlflow tests").getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.mlflow,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/namespace.py b/python/pyspark/pandas/namespace.py
index 9490e3702523..151928f3ba81 100644
--- a/python/pyspark/pandas/namespace.py
+++ b/python/pyspark/pandas/namespace.py
@@ -18,6 +18,7 @@
"""
Wrappers around spark that correspond to common pandas functions.
"""
+
from typing import (
Any,
Callable,
@@ -2280,11 +2281,11 @@ def get_dummies(
raise KeyError(name_like_string(columns))
if prefix is None:
prefix = [
- str(label[len(columns) :])
- if len(label) > len(columns) + 1
- else label[len(columns)]
- if len(label) == len(columns) + 1
- else ""
+ (
+ str(label[len(columns) :])
+ if len(label) > len(columns) + 1
+ else label[len(columns)] if len(label) == len(columns) + 1 else ""
+ )
for label in column_labels
]
elif any(isinstance(col, tuple) for col in columns) and any(
@@ -2567,9 +2568,11 @@ def concat(
level: int = min(psdf._internal.column_labels_level for psdf in psdfs)
psdfs = [
- DataFrame._index_normalized_frame(level, psdf)
- if psdf._internal.column_labels_level > level
- else psdf
+ (
+ DataFrame._index_normalized_frame(level, psdf)
+ if psdf._internal.column_labels_level > level
+ else psdf
+ )
for psdf in psdfs
]
@@ -3941,7 +3944,7 @@ def _test() -> None:
path = tempfile.mkdtemp()
globs["path"] = path
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.namespace,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/numpy_compat.py b/python/pyspark/pandas/numpy_compat.py
index 6fed89157d29..913089ca56c4 100644
--- a/python/pyspark/pandas/numpy_compat.py
+++ b/python/pyspark/pandas/numpy_compat.py
@@ -23,7 +23,6 @@ from pyspark.sql.pandas.functions import pandas_udf
from pyspark.sql.types import DoubleType, LongType, BooleanType
from pyspark.pandas.base import IndexOpsMixin
-
unary_np_spark_mappings = {
"abs": F.abs,
"absolute": F.abs,
@@ -245,7 +244,7 @@ def _test() -> None:
.appName("pyspark.pandas.numpy_compat tests")
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.numpy_compat,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/plot/matplotlib.py b/python/pyspark/pandas/plot/matplotlib.py
index 06bc7e84cd4d..f68d1f3efdda 100644
--- a/python/pyspark/pandas/plot/matplotlib.py
+++ b/python/pyspark/pandas/plot/matplotlib.py
@@ -451,9 +451,11 @@ class PandasOnSparkHistPlot(PandasHistPlot, HistogramPlotBase):
artists = self._plot(ax, y, column_num=i, stacking_id=stacking_id, **kwds)
# `if hasattr(...)` makes plotting compatible with pandas < 1.3,
# see pandas-dev/pandas#40078.
- self._append_legend_handles_labels(artists[0], label) if hasattr(
- self, "_append_legend_handles_labels"
- ) else self._add_legend_handle(artists[0], label, index=i)
+ (
+ self._append_legend_handles_labels(artists[0], label)
+ if hasattr(self, "_append_legend_handles_labels")
+ else self._add_legend_handle(artists[0], label, index=i)
+ )
@classmethod
def _plot(cls, ax, y, style=None, bins=None, bottom=0, column_num=0, stacking_id=None, **kwds):
@@ -570,9 +572,11 @@ class PandasOnSparkKdePlot(PandasKdePlot, KdePlotBase):
artists = self._plot(ax, y, column_num=i, stacking_id=stacking_id, **kwds)
# `if hasattr(...)` makes plotting compatible with pandas < 1.3,
# see pandas-dev/pandas#40078.
- self._append_legend_handles_labels(artists[0], label) if hasattr(
- self, "_append_legend_handles_labels"
- ) else self._add_legend_handle(artists[0], label, index=i)
+ (
+ self._append_legend_handles_labels(artists[0], label)
+ if hasattr(self, "_append_legend_handles_labels")
+ else self._add_legend_handle(artists[0], label, index=i)
+ )
@staticmethod
def _get_ind(y, ind):
diff --git a/python/pyspark/pandas/resample.py b/python/pyspark/pandas/resample.py
index 709a7c784cc3..bdb939cffd0a 100644
--- a/python/pyspark/pandas/resample.py
+++ b/python/pyspark/pandas/resample.py
@@ -18,6 +18,7 @@
"""
A wrapper for ResampledData to behave like pandas Resampler.
"""
+
from abc import ABCMeta, abstractmethod
from functools import partial
from typing import (
@@ -136,7 +137,7 @@ class Resampler(Generic[FrameLike], metaclass=ABCMeta):
def _bin_timestamp(self, origin: pd.Timestamp, ts_scol: Column) -> Column:
key_type = self._resamplekey_type
origin_scol = F.lit(origin)
- (rule_code, n) = (self._offset.rule_code, getattr(self._offset, "n"))
+ rule_code, n = (self._offset.rule_code, getattr(self._offset, "n"))
left_closed, right_closed = (self._closed == "left", self._closed == "right")
left_labeled, right_labeled = (self._label == "left", self._label == "right")
@@ -780,7 +781,7 @@ def _test() -> None:
.appName("pyspark.pandas.resample tests")
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.resample,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/series.py b/python/pyspark/pandas/series.py
index 79898c6de9dd..8c9f54cff998 100644
--- a/python/pyspark/pandas/series.py
+++ b/python/pyspark/pandas/series.py
@@ -18,6 +18,7 @@
"""
A wrapper class for Spark Column to behave like pandas Series.
"""
+
import datetime
import re
import inspect
@@ -6021,12 +6022,14 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
F.max_by(
spark_column,
F.when(
- (index_scol <= F.lit(index).cast(index_type)) & spark_column.isNotNull()
- if pd.notna(index)
- # If index is nan and the value of the col is not null
- # then return monotonically_increasing_id. This will let max by
- # to return last index value, which is the behaviour of pandas
- else spark_column.isNotNull(),
+ (
+ (index_scol <= F.lit(index).cast(index_type)) & spark_column.isNotNull()
+ if pd.notna(index)
+ # If index is nan and the value of the col is not null
+ # then return monotonically_increasing_id. This will let max by
+ # to return last index value, which is the behaviour of pandas
+ else spark_column.isNotNull()
+ ),
F.col(monotonically_increasing_id_column),
),
)
@@ -7470,13 +7473,11 @@ def unpack_scalar(sdf: SparkDataFrame) -> Any:
@overload
-def first_series(df: DataFrame) -> Series:
- ...
+def first_series(df: DataFrame) -> Series: ...
@overload
-def first_series(df: pd.DataFrame) -> pd.Series:
- ...
+def first_series(df: pd.DataFrame) -> pd.Series: ...
def first_series(df: Union[DataFrame, pd.DataFrame]) -> Union[Series, pd.Series]:
@@ -7505,7 +7506,7 @@ def _test() -> None:
spark = (
SparkSession.builder.master("local[4]").appName("pyspark.pandas.series tests").getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.series,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/spark/accessors.py b/python/pyspark/pandas/spark/accessors.py
index 86070ed841c6..efebdee00166 100644
--- a/python/pyspark/pandas/spark/accessors.py
+++ b/python/pyspark/pandas/spark/accessors.py
@@ -19,6 +19,7 @@
Spark related features. Usually, the features here are missing in pandas
but Spark has it.
"""
+
from abc import ABCMeta, abstractmethod
from typing import TYPE_CHECKING, Callable, Generic, List, Optional, Union
@@ -1257,7 +1258,7 @@ def _test() -> None:
path = tempfile.mkdtemp()
globs["path"] = path
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.spark.accessors,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/spark/utils.py b/python/pyspark/pandas/spark/utils.py
index 9b8b5bb7542a..d8c5aa41debc 100644
--- a/python/pyspark/pandas/spark/utils.py
+++ b/python/pyspark/pandas/spark/utils.py
@@ -17,29 +17,26 @@
"""
Helpers and utilities to deal with PySpark instances
"""
+
from typing import overload
from pyspark.sql.types import DecimalType, StructType, MapType, ArrayType, StructField, DataType
@overload
-def as_nullable_spark_type(dt: StructType) -> StructType:
- ...
+def as_nullable_spark_type(dt: StructType) -> StructType: ...
@overload
-def as_nullable_spark_type(dt: ArrayType) -> ArrayType:
- ...
+def as_nullable_spark_type(dt: ArrayType) -> ArrayType: ...
@overload
-def as_nullable_spark_type(dt: MapType) -> MapType:
- ...
+def as_nullable_spark_type(dt: MapType) -> MapType: ...
@overload
-def as_nullable_spark_type(dt: DataType) -> DataType:
- ...
+def as_nullable_spark_type(dt: DataType) -> DataType: ...
def as_nullable_spark_type(dt: DataType) -> DataType:
@@ -96,29 +93,25 @@ def as_nullable_spark_type(dt: DataType) -> DataType:
@overload
def force_decimal_precision_scale(
dt: StructType, *, precision: int = ..., scale: int = ...
-) -> StructType:
- ...
+) -> StructType: ...
@overload
def force_decimal_precision_scale(
dt: ArrayType, *, precision: int = ..., scale: int = ...
-) -> ArrayType:
- ...
+) -> ArrayType: ...
@overload
def force_decimal_precision_scale(
dt: MapType, *, precision: int = ..., scale: int = ...
-) -> MapType:
- ...
+) -> MapType: ...
@overload
def force_decimal_precision_scale(
dt: DataType, *, precision: int = ..., scale: int = ...
-) -> DataType:
- ...
+) -> DataType: ...
def force_decimal_precision_scale(
@@ -189,7 +182,7 @@ def _test() -> None:
import pyspark.pandas.spark.utils
globs = pyspark.pandas.spark.utils.__dict__.copy()
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.spark.utils,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/sql_formatter.py b/python/pyspark/pandas/sql_formatter.py
index 98f085b39717..d78bff17f9ae 100644
--- a/python/pyspark/pandas/sql_formatter.py
+++ b/python/pyspark/pandas/sql_formatter.py
@@ -33,7 +33,6 @@ from pyspark.pandas.frame import DataFrame
from pyspark.pandas.series import Series
from pyspark.sql.utils import is_remote
-
__all__ = ["sql"]
@@ -320,7 +319,7 @@ def _test() -> None:
.appName("pyspark.pandas.sql_formatter tests")
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.sql_formatter,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/sql_processor.py b/python/pyspark/pandas/sql_processor.py
index 8437ff6e48cf..afac0ca9cb33 100644
--- a/python/pyspark/pandas/sql_processor.py
+++ b/python/pyspark/pandas/sql_processor.py
@@ -29,7 +29,6 @@ from pyspark.pandas.series import Series
from pyspark.pandas.internal import InternalFrame
from pyspark.pandas.namespace import _get_index_map
-
__all__ = ["sql"]
from builtins import globals as builtin_globals
@@ -375,7 +374,7 @@ def _test() -> None:
.appName("pyspark.pandas.sql_processor tests")
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.sql_processor,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/strings.py b/python/pyspark/pandas/strings.py
index b5b7d25b1204..b29482d32d78 100644
--- a/python/pyspark/pandas/strings.py
+++ b/python/pyspark/pandas/strings.py
@@ -18,6 +18,7 @@
"""
String functions on pandas-on-Spark Series
"""
+
from functools import wraps
from typing import (
Any,
@@ -44,7 +45,6 @@ from pyspark.sql import functions as F
from pyspark.sql.functions import pandas_udf
import pyspark.pandas as ps
-
FuncT = TypeVar("FuncT", bound=Callable[..., Any])
@@ -2423,7 +2423,7 @@ def _test() -> None:
.appName("pyspark.pandas.strings tests")
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.strings,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/supported_api_gen.py b/python/pyspark/pandas/supported_api_gen.py
index ddb890e04327..c39b8880f14b 100644
--- a/python/pyspark/pandas/supported_api_gen.py
+++ b/python/pyspark/pandas/supported_api_gen.py
@@ -18,6 +18,7 @@
"""
Generate 'Supported pandas APIs' documentation file
"""
+
import warnings
from enum import Enum, unique
from inspect import getmembers, isclass, isfunction, signature
@@ -374,8 +375,10 @@ def _write_table(
else:
lines.append(" * - :func:`%s`\n" % func_str)
lines.append(" - %s\n" % status.implemented)
- lines.append(" - \n") if not status.missing else lines.append(
- " - %s\n" % status.missing
+ (
+ lines.append(" - \n")
+ if not status.missing
+ else lines.append(" - %s\n" % status.missing)
)
w_fd.writelines(lines)
@@ -432,7 +435,7 @@ def _test() -> None:
import pyspark.pandas.supported_api_gen
globs = pyspark.pandas.supported_api_gen.__dict__.copy()
- (failure_count, test_count) = doctest.testmod(pyspark.pandas.supported_api_gen, globs=globs)
+ failure_count, test_count = doctest.testmod(pyspark.pandas.supported_api_gen, globs=globs)
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/pandas/testing.py b/python/pyspark/pandas/testing.py
index dccd119ed1eb..c2c10c09d724 100644
--- a/python/pyspark/pandas/testing.py
+++ b/python/pyspark/pandas/testing.py
@@ -18,6 +18,7 @@
"""
Public testing utility functions.
"""
+
from typing import Literal, Union
import pyspark.pandas as ps
diff --git a/python/pyspark/pandas/tests/data_type_ops/test_as_type.py b/python/pyspark/pandas/tests/data_type_ops/test_as_type.py
index 2c543f6b6715..3808da3a18f4 100644
--- a/python/pyspark/pandas/tests/data_type_ops/test_as_type.py
+++ b/python/pyspark/pandas/tests/data_type_ops/test_as_type.py
@@ -60,16 +60,18 @@ class AsTypeTestsMixin:
def test_astype_eager_check(self):
psser = self.psdf["float_nan"]
- with ps.option_context("compute.eager_check", True), self.assertRaisesRegex(
- ValueError, "Cannot convert"
+ with (
+ ps.option_context("compute.eager_check", True),
+ self.assertRaisesRegex(ValueError, "Cannot convert"),
):
psser.astype(int)
with ps.option_context("compute.eager_check", False):
psser.astype(int)
psser = self.psdf["decimal_nan"]
- with ps.option_context("compute.eager_check", True), self.assertRaisesRegex(
- ValueError, "Cannot convert"
+ with (
+ ps.option_context("compute.eager_check", True),
+ self.assertRaisesRegex(ValueError, "Cannot convert"),
):
psser.astype(int)
with ps.option_context("compute.eager_check", False):
diff --git a/python/pyspark/pandas/tests/diff_frames_ops/test_dot_series.py b/python/pyspark/pandas/tests/diff_frames_ops/test_dot_series.py
index dd78b0c68f1b..40956309e16a 100644
--- a/python/pyspark/pandas/tests/diff_frames_ops/test_dot_series.py
+++ b/python/pyspark/pandas/tests/diff_frames_ops/test_dot_series.py
@@ -110,13 +110,15 @@ class DiffFramesDotSeriesMixin:
with self.assertRaisesRegex(ValueError, "matrices are not aligned"):
psser.dot(psser_other)
- with ps.option_context("compute.eager_check", False), self.assertRaisesRegex(
- ValueError, "matrices are not aligned"
+ with (
+ ps.option_context("compute.eager_check", False),
+ self.assertRaisesRegex(ValueError, "matrices are not aligned"),
):
psser.dot(psser_other2)
- with ps.option_context("compute.eager_check", True), self.assertRaisesRegex(
- ValueError, "matrices are not aligned"
+ with (
+ ps.option_context("compute.eager_check", True),
+ self.assertRaisesRegex(ValueError, "matrices are not aligned"),
):
psser.dot(psser_other)
diff --git a/python/pyspark/pandas/tests/groupby/test_stat.py b/python/pyspark/pandas/tests/groupby/test_stat.py
index c95cde1742fc..02578a71b568 100644
--- a/python/pyspark/pandas/tests/groupby/test_stat.py
+++ b/python/pyspark/pandas/tests/groupby/test_stat.py
@@ -24,7 +24,6 @@ from pyspark.loose_version import LooseVersion
from pyspark.testing.pandasutils import PandasOnSparkTestCase
from pyspark.testing.sqlutils import SQLTestUtils
-
using_pandas3 = LooseVersion(pd.__version__) >= "3.0.0"
diff --git a/python/pyspark/pandas/tests/indexes/test_default.py b/python/pyspark/pandas/tests/indexes/test_default.py
index 14a251a879c1..e45e9904e1ff 100644
--- a/python/pyspark/pandas/tests/indexes/test_default.py
+++ b/python/pyspark/pandas/tests/indexes/test_default.py
@@ -40,9 +40,10 @@ class DefaultIndexTestsMixin:
self.assertEqual(len(set(pdf.index)), len(pdf))
def test_index_distributed_sequence_cleanup(self):
- with ps.option_context(
- "compute.default_index_type", "distributed-sequence"
- ), ps.option_context("compute.ops_on_diff_frames", True):
+ with (
+ ps.option_context("compute.default_index_type", "distributed-sequence"),
+ ps.option_context("compute.ops_on_diff_frames", True),
+ ):
with ps.option_context("compute.default_index_cache", "LOCAL_CHECKPOINT"):
cached_rdd_ids = [rdd_id for rdd_id in self._legacy_sc._jsc.getPersistentRDDs()]
diff --git a/python/pyspark/pandas/tests/io/test_csv.py b/python/pyspark/pandas/tests/io/test_csv.py
index 2b618f3b11c6..aa9dc6ae56f6 100644
--- a/python/pyspark/pandas/tests/io/test_csv.py
+++ b/python/pyspark/pandas/tests/io/test_csv.py
@@ -32,8 +32,7 @@ def normalize_text(s):
class CsvTestsMixin:
@property
def csv_text(self):
- return normalize_text(
- """
+ return normalize_text("""
name,amount
Alice,100
Bob,-200
@@ -48,67 +47,55 @@ class CsvTestsMixin:
Frank,200
Alice,300
Edith,600
- """
- )
+ """)
@property
def csv_text_2(self):
- return normalize_text(
- """
+ return normalize_text("""
A,B
item1,1
item2,1,2
item3,1,2,3,4
item4,1
- """
- )
+ """)
@property
def csv_text_with_comments(self):
- return normalize_text(
- """
+ return normalize_text("""
# header
%s
# comment
Alice,400
Edith,600
# footer
- """
- % self.csv_text
- )
+ """ % self.csv_text)
@property
def tab_delimited_csv_text(self):
- return normalize_text(
- """
+ return normalize_text("""
name\tamount
Alice\t100
Bob\t-200
Charlie\t300
- """
- )
+ """)
@property
def q_quoted_csv_text(self):
- return normalize_text(
- """
+ return normalize_text("""
QnameQ,QamountQ
QA,liceQ,Q100Q
QB,obQ,Q-200Q
QC,harlieQ,Q300Q
- """
- )
+ """)
@property
def e_escapeted_csv_text(self):
- return normalize_text(
- """
+ return normalize_text("""
name,amount
"AE"lice",100
"BE"ob",-200
"CE"harlie",300
- """
- )
+ """)
@contextmanager
def csv_file(self, csv):
diff --git a/python/pyspark/pandas/tests/io/test_dataframe_conversion.py b/python/pyspark/pandas/tests/io/test_dataframe_conversion.py
index 04f19b66e8ca..004dbea0951f 100644
--- a/python/pyspark/pandas/tests/io/test_dataframe_conversion.py
+++ b/python/pyspark/pandas/tests/io/test_dataframe_conversion.py
@@ -51,8 +51,7 @@ class DataFrameConversionMixin:
return str.translate({ord(c): None for c in string.whitespace})
def test_to_html(self):
- expected = self.strip_all_whitespace(
- """
+ expected = self.strip_all_whitespace("""
<table border="1" class="dataframe">
<thead>
<tr style="text-align: right;"><th></th><th>a</th><th>b</th></tr>
@@ -63,14 +62,12 @@ class DataFrameConversionMixin:
<tr><th>3</th><td>3</td><td>6</td></tr>
</tbody>
</table>
- """
- )
+ """)
got = self.strip_all_whitespace(self.psdf.to_html())
self.assert_eq(got, expected)
# with max_rows set
- expected = self.strip_all_whitespace(
- """
+ expected = self.strip_all_whitespace("""
<table border="1" class="dataframe">
<thead>
<tr style="text-align: right;"><th></th><th>a</th><th>b</th></tr>
@@ -80,8 +77,7 @@ class DataFrameConversionMixin:
<tr><th>1</th><td>2</td><td>5</td></tr>
</tbody>
</table>
- """
- )
+ """)
got = self.strip_all_whitespace(self.psdf.to_html(max_rows=2))
self.assert_eq(got, expected)
diff --git a/python/pyspark/pandas/tests/test_typedef.py b/python/pyspark/pandas/tests/test_typedef.py
index e07551ab3a4e..cb07b36a1c99 100644
--- a/python/pyspark/pandas/tests/test_typedef.py
+++ b/python/pyspark/pandas/tests/test_typedef.py
@@ -102,9 +102,11 @@ class TypeHintTestsMixin:
inferred.dtypes,
[
np.float64,
- np.str_
- if LooseVersion(pd.__version__) < LooseVersion("3.0.0")
- else pd.StringDtype(na_value=np.nan),
+ (
+ np.str_
+ if LooseVersion(pd.__version__) < LooseVersion("3.0.0")
+ else pd.StringDtype(na_value=np.nan)
+ ),
],
)
self.assertEqual(inferred.spark_type, expected)
@@ -133,9 +135,11 @@ class TypeHintTestsMixin:
inferred.dtypes,
[
np.float64,
- np.str_
- if LooseVersion(pd.__version__) < LooseVersion("3.0.0")
- else pd.StringDtype(na_value=np.nan),
+ (
+ np.str_
+ if LooseVersion(pd.__version__) < LooseVersion("3.0.0")
+ else pd.StringDtype(na_value=np.nan)
+ ),
],
)
self.assertEqual(inferred.spark_type, expected)
@@ -192,9 +196,11 @@ class TypeHintTestsMixin:
inferred.dtypes,
[
np.float64,
- np.str_
- if LooseVersion(pd.__version__) < LooseVersion("3.0.0")
- else pd.StringDtype(na_value=np.nan),
+ (
+ np.str_
+ if LooseVersion(pd.__version__) < LooseVersion("3.0.0")
+ else pd.StringDtype(na_value=np.nan)
+ ),
],
)
self.assertEqual(inferred.spark_type, expected)
@@ -352,9 +358,11 @@ class TypeHintTestsMixin:
# string
np.str_: (np.str_, StringType()),
str: (
- np.str_
- if LooseVersion(pd.__version__) < LooseVersion("3.0.0")
- else pd.StringDtype(na_value=np.nan),
+ (
+ np.str_
+ if LooseVersion(pd.__version__) < LooseVersion("3.0.0")
+ else pd.StringDtype(na_value=np.nan)
+ ),
StringType(),
),
# bool
@@ -362,9 +370,11 @@ class TypeHintTestsMixin:
# datetime
np.datetime64: (np.datetime64, TimestampType()),
datetime.datetime: (
- np.dtype("datetime64[ns]")
- if LooseVersion(pd.__version__) < LooseVersion("3.0.0")
- else np.dtype("datetime64[us]"),
+ (
+ np.dtype("datetime64[ns]")
+ if LooseVersion(pd.__version__) < LooseVersion("3.0.0")
+ else np.dtype("datetime64[us]")
+ ),
TimestampType(),
),
# DateType
diff --git a/python/pyspark/pandas/typedef/typehints.py b/python/pyspark/pandas/typedef/typehints.py
index 8af93f9dcc13..a1a0344cedab 100644
--- a/python/pyspark/pandas/typedef/typehints.py
+++ b/python/pyspark/pandas/typedef/typehints.py
@@ -18,6 +18,7 @@
"""
Utilities to deal with types. This is mostly focused on python3.
"""
+
import datetime
import decimal
import sys
@@ -33,7 +34,6 @@ from pandas.api.extensions import ExtensionDtype
from pyspark.loose_version import LooseVersion
-
extension_dtypes: Tuple[type, ...]
try:
from pandas import Int8Dtype, Int16Dtype, Int32Dtype, Int64Dtype
@@ -659,9 +659,11 @@ def infer_return_type(f: Callable) -> Union[SeriesType, DataFrameType, ScalarTyp
InternalField(
dtype=index_dtype,
struct_field=types.StructField(
- name=index_name
- if index_name is not None
- else SPARK_INDEX_NAME_FORMAT(level),
+ name=(
+ index_name
+ if index_name is not None
+ else SPARK_INDEX_NAME_FORMAT(level)
+ ),
dataType=index_spark_type,
),
)
@@ -672,9 +674,11 @@ def infer_return_type(f: Callable) -> Union[SeriesType, DataFrameType, ScalarTyp
data_dtypes, data_spark_types = zip(
*(
- pandas_on_spark_type(p.tpe)
- if isclass(p) and issubclass(p, NameTypeHolder)
- else pandas_on_spark_type(p)
+ (
+ pandas_on_spark_type(p.tpe)
+ if isclass(p) and issubclass(p, NameTypeHolder)
+ else pandas_on_spark_type(p)
+ )
for p in data_parameters
)
)
@@ -945,7 +949,7 @@ def _test() -> None:
import pyspark.pandas.typedef.typehints
globs = pyspark.pandas.typedef.typehints.__dict__.copy()
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.typedef.typehints,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/utils.py b/python/pyspark/pandas/utils.py
index 2b55b2721edf..8b3ecc395507 100644
--- a/python/pyspark/pandas/utils.py
+++ b/python/pyspark/pandas/utils.py
@@ -56,7 +56,6 @@ from pyspark.pandas._typing import (
)
from pyspark.pandas.typedef.typehints import as_spark_type
-
if TYPE_CHECKING:
from pyspark.pandas.indexes.base import Index
from pyspark.pandas.base import IndexOpsMixin
@@ -816,13 +815,11 @@ def validate_mode(mode: str) -> str:
@overload
-def verify_temp_column_name(df: PySparkDataFrame, column_name_or_label: str) -> str:
- ...
+def verify_temp_column_name(df: PySparkDataFrame, column_name_or_label: str) -> str: ...
@overload
-def verify_temp_column_name(df: "DataFrame", column_name_or_label: Name) -> Label:
- ...
+def verify_temp_column_name(df: "DataFrame", column_name_or_label: Name) -> Label: ...
def verify_temp_column_name(
@@ -1143,7 +1140,7 @@ def is_ansi_mode_enabled(spark: SparkSession) -> bool:
from pyspark.pandas.config import _key_format, _options_dict
client = cast(ConnectSession, spark).client
- (ansi_mode_support, ansi_enabled) = client.get_config_with_defaults(
+ ansi_mode_support, ansi_enabled = client.get_config_with_defaults(
(
_key_format("compute.ansi_mode_support"),
json.dumps(_options_dict["compute.ansi_mode_support"].default),
@@ -1186,7 +1183,7 @@ def _test() -> None:
spark = (
SparkSession.builder.master("local[4]").appName("pyspark.pandas.utils tests").getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.utils,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pandas/window.py b/python/pyspark/pandas/window.py
index 56692c120d91..294923d5f50e 100644
--- a/python/pyspark/pandas/window.py
+++ b/python/pyspark/pandas/window.py
@@ -2691,7 +2691,7 @@ def _test() -> None:
spark = (
SparkSession.builder.master("local[4]").appName("pyspark.pandas.window tests").getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.pandas.window,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/pipelines/api.py b/python/pyspark/pipelines/api.py
index f109841d657b..f9c642cadc3b 100644
--- a/python/pyspark/pipelines/api.py
+++ b/python/pyspark/pipelines/api.py
@@ -96,8 +96,7 @@ def _validate_stored_dataset_args(
@overload
-def table(query_function: QueryFunction) -> None:
- ...
+def table(query_function: QueryFunction) -> None: ...
@overload
@@ -111,8 +110,7 @@ def table(
partition_cols: Optional[List[str]] = None,
cluster_by: Optional[List[str]] = None,
schema: Optional[Union[StructType, str]] = None,
-) -> Callable[[QueryFunction], None]:
- ...
+) -> Callable[[QueryFunction], None]: ...
def table(
@@ -202,8 +200,7 @@ def table(
@overload
-def materialized_view(query_function: QueryFunction) -> None:
- ...
+def materialized_view(query_function: QueryFunction) -> None: ...
@overload
@@ -217,8 +214,7 @@ def materialized_view(
partition_cols: Optional[List[str]] = None,
cluster_by: Optional[List[str]] = None,
schema: Optional[Union[StructType, str]] = None,
-) -> Callable[[QueryFunction], None]:
- ...
+) -> Callable[[QueryFunction], None]: ...
def materialized_view(
@@ -310,8 +306,7 @@ def materialized_view(
@overload
def temporary_view(
query_function: QueryFunction,
-) -> None:
- ...
+) -> None: ...
@overload
@@ -321,8 +316,7 @@ def temporary_view(
name: Optional[str] = None,
comment: Optional[str] = None,
spark_conf: Optional[Dict[str, str]] = None,
-) -> Callable[[QueryFunction], None]:
- ...
+) -> Callable[[QueryFunction], None]: ...
def temporary_view(
diff --git a/python/pyspark/pipelines/cli.py b/python/pyspark/pipelines/cli.py
index 8c70c8331120..4497c543ea9d 100644
--- a/python/pyspark/pipelines/cli.py
+++ b/python/pyspark/pipelines/cli.py
@@ -21,6 +21,7 @@ Implementation of spark-pipelines CLI.
Example usage:
$ bin/spark-pipelines run --spec /path/to/pipeline.yaml
"""
+
from contextlib import contextmanager
import argparse
import glob
diff --git a/python/pyspark/pipelines/spark_connect_pipeline.py b/python/pyspark/pipelines/spark_connect_pipeline.py
index 2fd11d7cf669..5a9e5802753e 100644
--- a/python/pyspark/pipelines/spark_connect_pipeline.py
+++ b/python/pyspark/pipelines/spark_connect_pipeline.py
@@ -42,7 +42,7 @@ def create_dataflow_graph(
command.pipeline_command.create_dataflow_graph.CopyFrom(inner_command)
# Cast because mypy seems to think `spark`` is a function, not an object. Likely related to
# SPARK-47544.
- (_, properties, _) = cast(Any, spark).client.execute_command(command)
+ _, properties, _ = cast(Any, spark).client.execute_command(command)
return properties["pipeline_command_result"].create_dataflow_graph_result.dataflow_graph_id
diff --git a/python/pyspark/pipelines/tests/test_cli.py b/python/pyspark/pipelines/tests/test_cli.py
index 0124f60c59b1..ba5089bf468d 100644
--- a/python/pyspark/pipelines/tests/test_cli.py
+++ b/python/pyspark/pipelines/tests/test_cli.py
@@ -49,8 +49,7 @@ if should_test_connect and have_yaml:
class CLIUtilityTests(ReusedConnectTestCase):
def test_load_pipeline_spec(self):
with tempfile.NamedTemporaryFile(mode="w") as tmpfile:
- tmpfile.write(
- """
+ tmpfile.write("""
{
"name": "test_pipeline",
"catalog": "test_catalog",
@@ -64,8 +63,7 @@ class CLIUtilityTests(ReusedConnectTestCase):
],
"storage": "storage_path",
}
- """
- )
+ """)
tmpfile.flush()
spec = load_pipeline_spec(Path(tmpfile.name))
assert spec.name == "test_pipeline"
@@ -78,8 +76,7 @@ class CLIUtilityTests(ReusedConnectTestCase):
def test_load_pipeline_spec_name_is_required(self):
with tempfile.NamedTemporaryFile(mode="w") as tmpfile:
- tmpfile.write(
- """
+ tmpfile.write("""
{
"catalog": "test_catalog",
"database": "test_database",
@@ -92,8 +89,7 @@ class CLIUtilityTests(ReusedConnectTestCase):
],
"storage": "storage_path",
}
- """
- )
+ """)
tmpfile.flush()
with self.assertRaises(PySparkException) as context:
load_pipeline_spec(Path(tmpfile.name))
@@ -104,8 +100,7 @@ class CLIUtilityTests(ReusedConnectTestCase):
def test_load_pipeline_spec_schema_fallback(self):
with tempfile.NamedTemporaryFile(mode="w") as tmpfile:
- tmpfile.write(
- """
+ tmpfile.write("""
{
"name": "test_pipeline",
"catalog": "test_catalog",
@@ -119,8 +114,7 @@ class CLIUtilityTests(ReusedConnectTestCase):
],
"storage": "storage_path",
}
- """
- )
+ """)
tmpfile.flush()
spec = load_pipeline_spec(Path(tmpfile.name))
assert spec.catalog == "test_catalog"
@@ -131,8 +125,7 @@ class CLIUtilityTests(ReusedConnectTestCase):
def test_load_pipeline_spec_invalid(self):
with tempfile.NamedTemporaryFile(mode="w") as tmpfile:
- tmpfile.write(
- """
+ tmpfile.write("""
{
"catalogtypo": "test_catalog",
"configuration": {
@@ -143,8 +136,7 @@ class CLIUtilityTests(ReusedConnectTestCase):
{"glob": {"include": "test_include"}}
]
}
- """
- )
+ """)
tmpfile.flush()
with self.assertRaises(PySparkException) as context:
load_pipeline_spec(Path(tmpfile.name))
@@ -193,15 +185,13 @@ class CLIUtilityTests(ReusedConnectTestCase):
with tempfile.TemporaryDirectory() as temp_dir:
spec_path = Path(temp_dir) / "spark-pipeline.yaml"
with spec_path.open("w") as f:
- f.write(
- """
+ f.write("""
{
"catalog": "test_catalog",
"configuration": {},
"libraries": []
}
- """
- )
+ """)
found_spec = find_pipeline_spec(Path(temp_dir))
self.assertEqual(found_spec, spec_path)
@@ -210,15 +200,13 @@ class CLIUtilityTests(ReusedConnectTestCase):
with tempfile.TemporaryDirectory() as temp_dir:
spec_path = Path(temp_dir) / "spark-pipeline.yml"
with spec_path.open("w") as f:
- f.write(
- """
+ f.write("""
{
"catalog": "test_catalog",
"configuration": {},
"libraries": []
}
- """
- )
+ """)
found_spec = find_pipeline_spec(Path(temp_dir))
self.assertEqual(found_spec, spec_path)
@@ -243,15 +231,13 @@ class CLIUtilityTests(ReusedConnectTestCase):
child_dir.mkdir()
spec_path = parent_dir / "spark-pipeline.yaml"
with spec_path.open("w") as f:
- f.write(
- """
+ f.write("""
{
"catalog": "test_catalog",
"configuration": {},
"libraries": []
}
- """
- )
+ """)
found_spec = find_pipeline_spec(Path(child_dir))
self.assertEqual(found_spec, spec_path)
@@ -272,27 +258,19 @@ class CLIUtilityTests(ReusedConnectTestCase):
subdir2 = outer_dir / "subdir2"
subdir2.mkdir()
with (subdir1 / "libraries.py").open("w") as f:
- f.write(
- textwrap.dedent(
- """
+ f.write(textwrap.dedent("""
from pyspark import pipelines as dp
@dp.materialized_view
def mv1():
raise NotImplementedError()
- """
- )
- )
+ """))
with (subdir2 / "libraries.py").open("w") as f:
- f.write(
- textwrap.dedent(
- """
+ f.write(textwrap.dedent("""
from pyspark import pipelines as dp
def mv2():
raise NotImplementedError()
- """
- )
- )
+ """))
registry = LocalGraphElementRegistry()
register_definitions(
@@ -355,15 +333,11 @@ class CLIUtilityTests(ReusedConnectTestCase):
inner_dir2.mkdir()
with (inner_dir1 / "defs.py").open("w") as f:
- f.write(
- textwrap.dedent(
- """
+ f.write(textwrap.dedent("""
import sys
sys.path.append(".")
import mypackage.my_module
- """
- )
- )
+ """))
inner_dir1_mypackage = inner_dir1 / "mypackage"
inner_dir1_mypackage.mkdir()
@@ -528,8 +502,7 @@ class CLIUtilityTests(ReusedConnectTestCase):
def test_pipeline_spec_with_invalid_glob_pattern(self):
"""Test that pipeline spec with invalid glob pattern is rejected."""
with tempfile.NamedTemporaryFile(mode="w") as tmpfile:
- tmpfile.write(
- """
+ tmpfile.write("""
{
"name": "test_pipeline",
"storage": "storage_path",
@@ -537,8 +510,7 @@ class CLIUtilityTests(ReusedConnectTestCase):
{"glob": {"include": "transformations/**/*.py"}}
]
}
- """
- )
+ """)
tmpfile.flush()
with self.assertRaises(PySparkException) as context:
load_pipeline_spec(Path(tmpfile.name))
diff --git a/python/pyspark/pipelines/tests/test_init_cli.py b/python/pyspark/pipelines/tests/test_init_cli.py
index dab1f08a21ae..f4f48ade35aa 100644
--- a/python/pyspark/pipelines/tests/test_init_cli.py
+++ b/python/pyspark/pipelines/tests/test_init_cli.py
@@ -25,7 +25,6 @@ from pyspark.testing.connectutils import (
)
from pyspark.testing.utils import have_yaml, yaml_requirement_message
-
if should_test_connect and have_yaml:
from pyspark.pipelines.cli import (
change_dir,
diff --git a/python/pyspark/profiler.py b/python/pyspark/profiler.py
index e798eab9bb28..28669d5343c3 100644
--- a/python/pyspark/profiler.py
+++ b/python/pyspark/profiler.py
@@ -190,7 +190,7 @@ if has_memory_profiler:
toplevel_code = code
filename = code.co_filename
if sub_lines is None or start_line is None:
- (sub_lines, start_line) = inspect.getsourcelines(code)
+ sub_lines, start_line = inspect.getsourcelines(code)
linenos = range(start_line, start_line + len(sub_lines))
self._toplevel.append((filename, code, linenos))
self[code] = {}
@@ -541,6 +541,6 @@ class MemoryProfiler(Profiler):
if __name__ == "__main__":
import doctest
- (failure_count, test_count) = doctest.testmod()
+ failure_count, test_count = doctest.testmod()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/resource/__init__.py b/python/pyspark/resource/__init__.py
index b4c463451c27..b3b33942827d 100644
--- a/python/pyspark/resource/__init__.py
+++ b/python/pyspark/resource/__init__.py
@@ -18,6 +18,7 @@
"""
APIs to let users manipulate resource requirements.
"""
+
from pyspark.resource.information import ResourceInformation
from pyspark.resource.requests import (
TaskResourceRequest,
diff --git a/python/pyspark/resource/information.py b/python/pyspark/resource/information.py
index 92cfc5a6e8ba..21f91ba06d15 100644
--- a/python/pyspark/resource/information.py
+++ b/python/pyspark/resource/information.py
@@ -19,7 +19,6 @@ from typing import List
class ResourceInformation:
-
"""
Class to hold information about a type of Resource. A resource could be a GPU, FPGA, etc.
The array of addresses are resource specific and its up to the user to interpret the address.
diff --git a/python/pyspark/resource/profile.py b/python/pyspark/resource/profile.py
index f0fb8f0b32d5..9fb7b42a8a73 100644
--- a/python/pyspark/resource/profile.py
+++ b/python/pyspark/resource/profile.py
@@ -29,7 +29,6 @@ if TYPE_CHECKING:
class ResourceProfile:
-
"""
Resource profile to associate with an RDD. A :class:`pyspark.resource.ResourceProfile`
allows the user to specify executor and task requirements for an RDD that will get
@@ -85,8 +84,7 @@ class ResourceProfile:
"""
@overload
- def __init__(self, _java_resource_profile: "JavaObject"):
- ...
+ def __init__(self, _java_resource_profile: "JavaObject"): ...
@overload
def __init__(
@@ -94,8 +92,7 @@ class ResourceProfile:
_java_resource_profile: None = ...,
_exec_req: Optional[Dict[str, ExecutorResourceRequest]] = ...,
_task_req: Optional[Dict[str, TaskResourceRequest]] = ...,
- ):
- ...
+ ): ...
def __init__(
self,
@@ -182,7 +179,6 @@ class ResourceProfile:
class ResourceProfileBuilder:
-
"""
Resource profile Builder to build a resource profile to associate with an RDD.
A ResourceProfile allows the user to specify executor and task requirements for
@@ -330,7 +326,7 @@ def _test() -> None:
globs = globals().copy()
globs["sc"] = SparkContext("local[4]", "profile tests")
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
globs=globs, optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE
)
globs["sc"].stop()
diff --git a/python/pyspark/resource/requests.py b/python/pyspark/resource/requests.py
index 805cecd5dbbe..324a5af361fc 100644
--- a/python/pyspark/resource/requests.py
+++ b/python/pyspark/resource/requests.py
@@ -124,7 +124,6 @@ class ExecutorResourceRequest:
class ExecutorResourceRequests:
-
"""
A set of Executor resource requests. This is used in conjunction with the
:class:`pyspark.resource.ResourceProfileBuilder` to programmatically specify the
@@ -148,16 +147,14 @@ class ExecutorResourceRequests:
_OFFHEAP_MEM = "offHeap"
@overload
- def __init__(self, _jvm: "JVMView"):
- ...
+ def __init__(self, _jvm: "JVMView"): ...
@overload
def __init__(
self,
_jvm: None = ...,
_requests: Optional[Dict[str, ExecutorResourceRequest]] = ...,
- ):
- ...
+ ): ...
def __init__(
self,
@@ -429,7 +426,6 @@ class TaskResourceRequest:
class TaskResourceRequests:
-
"""
A set of task resource requests. This is used in conjunction with the
:class:`pyspark.resource.ResourceProfileBuilder` to programmatically specify the resources
@@ -449,16 +445,14 @@ class TaskResourceRequests:
_CPUS = "cpus"
@overload
- def __init__(self, _jvm: "JVMView"):
- ...
+ def __init__(self, _jvm: "JVMView"): ...
@overload
def __init__(
self,
_jvm: None = ...,
_requests: Optional[Dict[str, TaskResourceRequest]] = ...,
- ):
- ...
+ ): ...
def __init__(
self,
diff --git a/python/pyspark/resultiterable.py b/python/pyspark/resultiterable.py
index 7f35cb629b36..d866ce0f092c 100644
--- a/python/pyspark/resultiterable.py
+++ b/python/pyspark/resultiterable.py
@@ -27,7 +27,6 @@ T = TypeVar("T")
class ResultIterable(Iterable[T]):
-
"""
A special result iterable. This is used because the standard
iterator can not be pickled
diff --git a/python/pyspark/serializers.py b/python/pyspark/serializers.py
index 9fd01ebf5d68..6de64a1062f0 100644
--- a/python/pyspark/serializers.py
+++ b/python/pyspark/serializers.py
@@ -68,7 +68,6 @@ pickle_protocol = pickle.HIGHEST_PROTOCOL
from pyspark import cloudpickle
-
__all__ = [
"PickleSerializer",
"CPickleSerializer",
@@ -135,7 +134,6 @@ class Serializer:
class FramedSerializer(Serializer):
-
"""
Serializer that writes objects as a stream of (length, data) pairs,
where `length` is a 32-bit integer and data is `length` bytes.
@@ -187,7 +185,6 @@ class FramedSerializer(Serializer):
class BatchedSerializer(Serializer):
-
"""
Serializes a stream of objects in batches by calling its wrapped
Serializer with streams of objects.
@@ -222,7 +219,6 @@ class BatchedSerializer(Serializer):
class FlattenedValuesSerializer(BatchedSerializer):
-
"""
Serializes a stream of list of pairs, split the list of values
which contain more than a certain number of objects to make them
@@ -277,7 +273,6 @@ class AutoBatchedSerializer(BatchedSerializer):
class CartesianDeserializer(Serializer):
-
"""
Deserializes the JavaRDD cartesian() of two PythonRDDs.
Due to pyspark batching we cannot simply use the result of the Java RDD cartesian,
@@ -303,7 +298,6 @@ class CartesianDeserializer(Serializer):
class PairDeserializer(Serializer):
-
"""
Deserializes the JavaRDD zip() of two PythonRDDs.
Due to pyspark batching we cannot simply use the result of the Java RDD zip,
@@ -399,9 +393,9 @@ if os.environ.get("PYSPARK_ENABLE_NAMEDTUPLE_PATCH") == "1":
return _hack_namedtuple(cls)
# replace namedtuple with the new one
- collections.namedtuple.__globals__[
- "_old_namedtuple_kwdefaults"
- ] = _old_namedtuple_kwdefaults
+ collections.namedtuple.__globals__["_old_namedtuple_kwdefaults"] = (
+ _old_namedtuple_kwdefaults
+ )
collections.namedtuple.__globals__["_old_namedtuple"] = _old_namedtuple
collections.namedtuple.__globals__["_hack_namedtuple"] = _hack_namedtuple
collections.namedtuple.__code__ = namedtuple.__code__
@@ -423,7 +417,6 @@ if os.environ.get("PYSPARK_ENABLE_NAMEDTUPLE_PATCH") == "1":
class PickleSerializer(FramedSerializer):
-
"""
Serializes objects using Python's pickle serializer:
@@ -468,7 +461,6 @@ else:
class MarshalSerializer(FramedSerializer):
-
"""
Serializes objects using Python's Marshal serializer:
@@ -485,7 +477,6 @@ class MarshalSerializer(FramedSerializer):
class AutoSerializer(FramedSerializer):
-
"""
Choose marshal or pickle as serialization protocol automatically
"""
@@ -534,7 +525,6 @@ class CompressedSerializer(FramedSerializer):
class UTF8Deserializer(Serializer):
-
"""
Deserializes streams written by String.getBytes.
"""
@@ -603,7 +593,6 @@ def write_with_length(obj, stream):
class ChunkedStream:
-
"""
This is a file-like object takes a stream of data, of unknown length, and breaks it into fixed
length frames. The intended use case is serializing large data and sending it immediately over
@@ -665,6 +654,6 @@ class ChunkedStream:
if __name__ == "__main__":
import doctest
- (failure_count, test_count) = doctest.testmod()
+ failure_count, test_count = doctest.testmod()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/shell.py b/python/pyspark/shell.py
index dae4854a237e..788c5143ca19 100644
--- a/python/pyspark/shell.py
+++ b/python/pyspark/shell.py
@@ -105,16 +105,13 @@ else:
sql = spark.sql
-print(
- r"""Welcome to
+print(r"""Welcome to
____ __
/ __/__ ___ _____/ /__
_\ \/ _ \/ _ `/ __/ '_/
/__ / .__/\_,_/_/ /_/\_\ version %s
/_/
-"""
- % version
-)
+""" % version)
print(
"Using Python version %s (%s, %s)"
% (platform.python_version(), platform.python_build()[0], platform.python_build()[1])
diff --git a/python/pyspark/shuffle.py b/python/pyspark/shuffle.py
index 579646549631..f2d8cf949d17 100644
--- a/python/pyspark/shuffle.py
+++ b/python/pyspark/shuffle.py
@@ -35,7 +35,6 @@ from pyspark.serializers import (
)
from pyspark.util import fail_on_stopiteration
-
try:
import psutil
@@ -93,7 +92,6 @@ DiskBytesSpilled = 0
class Aggregator:
-
"""
Aggregator has tree functions to merge values into combiner.
@@ -109,7 +107,6 @@ class Aggregator:
class SimpleAggregator(Aggregator):
-
"""
SimpleAggregator is useful for the cases that combiners have
same type with values
@@ -120,7 +117,6 @@ class SimpleAggregator(Aggregator):
class Merger:
-
"""
Merge shuffled data together by aggregator
"""
@@ -148,7 +144,6 @@ def _compressed_serializer(self, serializer=None):
class ExternalMerger(Merger):
-
"""
External merger will dump the aggregated data into disks when
memory usage goes above the limit, then merge them together.
@@ -696,7 +691,6 @@ class GroupByKey:
class ExternalGroupBy(ExternalMerger):
-
"""
Group by the items by key. If any partition of them can not been
hold in memory, it will do sort based group by.
@@ -849,6 +843,6 @@ class ExternalGroupBy(ExternalMerger):
if __name__ == "__main__":
import doctest
- (failure_count, test_count) = doctest.testmod()
+ failure_count, test_count = doctest.testmod()
if failure_count:
sys.exit(-1)
diff --git a/python/pyspark/sql/__init__.py b/python/pyspark/sql/__init__.py
index eeeeddd00e3a..117f57f29d3f 100644
--- a/python/pyspark/sql/__init__.py
+++ b/python/pyspark/sql/__init__.py
@@ -39,6 +39,7 @@ Important classes of Spark SQL and DataFrames:
- :class:`pyspark.sql.Window`
For working with window functions.
"""
+
from pyspark.sql.types import Geography, Geometry, Row, VariantVal
from pyspark.sql.context import SQLContext, HiveContext, UDFRegistration, UDTFRegistration
from pyspark.sql.session import SparkSession
@@ -53,7 +54,6 @@ from pyspark.sql.window import Window, WindowSpec
from pyspark.sql.pandas.group_ops import PandasCogroupedOps
from pyspark.sql.utils import is_remote
-
__all__ = [
"SparkSession",
"SQLContext",
diff --git a/python/pyspark/sql/avro/functions.py b/python/pyspark/sql/avro/functions.py
index e1b70c5fd359..79e301d08d2a 100644
--- a/python/pyspark/sql/avro/functions.py
+++ b/python/pyspark/sql/avro/functions.py
@@ -19,7 +19,6 @@
A collections of builtin avro functions
"""
-
from typing import Dict, Optional, TYPE_CHECKING, cast
from pyspark.errors import PySparkTypeError
@@ -210,7 +209,7 @@ def _test() -> None:
SparkSession.builder.master("local[4]").appName("sql.avro.functions tests").getOrCreate()
)
globs["spark"] = spark
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.sql.avro.functions,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE,
diff --git a/python/pyspark/sql/catalog.py b/python/pyspark/sql/catalog.py
index a74acc145647..268f4e0a0522 100644
--- a/python/pyspark/sql/catalog.py
+++ b/python/pyspark/sql/catalog.py
@@ -1265,7 +1265,7 @@ def _test() -> None:
globs["spark"] = (
SparkSession.builder.master("local[4]").appName("sql.catalog tests").getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.sql.catalog,
globs=globs,
optionflags=doctest.ELLIPSIS
diff --git a/python/pyspark/sql/classic/column.py b/python/pyspark/sql/classic/column.py
index 75bebe905344..c7acf504098a 100644
--- a/python/pyspark/sql/classic/column.py
+++ b/python/pyspark/sql/classic/column.py
@@ -47,7 +47,7 @@ __all__ = ["Column"]
def _create_column_from_literal(
- literal: Union["LiteralType", "DecimalLiteral", "DateTimeLiteral", "ParentColumn"]
+ literal: Union["LiteralType", "DecimalLiteral", "DateTimeLiteral", "ParentColumn"],
) -> "JavaObject":
from py4j.java_gateway import JVMView
@@ -76,8 +76,7 @@ def _to_java_column(col: "ColumnOrName") -> "JavaObject":
@overload
-def _to_seq(sc: "SparkContext", cols: Iterable["JavaObject"]) -> "JavaObject":
- ...
+def _to_seq(sc: "SparkContext", cols: Iterable["JavaObject"]) -> "JavaObject": ...
@overload
@@ -85,8 +84,7 @@ def _to_seq(
sc: "SparkContext",
cols: Iterable["ColumnOrName"],
converter: Optional[Callable[["ColumnOrName"], "JavaObject"]],
-) -> "JavaObject":
- ...
+) -> "JavaObject": ...
def _to_seq(
@@ -644,7 +642,7 @@ def _test() -> None:
)
globs["spark"] = spark
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.sql.column,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF,
diff --git a/python/pyspark/sql/classic/dataframe.py b/python/pyspark/sql/classic/dataframe.py
index e300d0d39293..fe70d9f0811b 100644
--- a/python/pyspark/sql/classic/dataframe.py
+++ b/python/pyspark/sql/classic/dataframe.py
@@ -76,7 +76,6 @@ from pyspark.sql.pandas.conversion import PandasConversionMixin
from pyspark.sql.pandas.map_ops import PandasMapOpsMixin
from pyspark.sql.table_arg import TableArg
-
if TYPE_CHECKING:
from py4j.java_gateway import JavaObject
import pyarrow as pa
@@ -621,8 +620,7 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
return DataFrame(self._jdf.distinct(), self.sparkSession)
@overload
- def sample(self, fraction: float, seed: Optional[int] = ...) -> ParentDataFrame:
- ...
+ def sample(self, fraction: float, seed: Optional[int] = ...) -> ParentDataFrame: ...
@overload
def sample(
@@ -630,8 +628,7 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
withReplacement: Optional[bool],
fraction: float,
seed: Optional[int] = ...,
- ) -> ParentDataFrame:
- ...
+ ) -> ParentDataFrame: ...
def sample( # type: ignore[misc]
self,
@@ -981,12 +978,10 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
return DataFrame(jdf, self.sparkSession)
@overload
- def head(self) -> Optional[Row]:
- ...
+ def head(self) -> Optional[Row]: ...
@overload
- def head(self, n: int) -> List[Row]:
- ...
+ def head(self, n: int) -> List[Row]: ...
def head(self, n: Optional[int] = None) -> Union[Optional[Row], List[Row]]:
if n is None:
@@ -998,12 +993,10 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
return self.head()
@overload
- def __getitem__(self, item: Union[int, str]) -> Column:
- ...
+ def __getitem__(self, item: Union[int, str]) -> Column: ...
@overload
- def __getitem__(self, item: Union[Column, List, Tuple]) -> ParentDataFrame:
- ...
+ def __getitem__(self, item: Union[Column, List, Tuple]) -> ParentDataFrame: ...
def __getitem__(
self, item: Union[int, str, Column, List, Tuple]
@@ -1038,24 +1031,20 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
return sorted(attrs)
@overload
- def select(self, *cols: "ColumnOrName") -> ParentDataFrame:
- ...
+ def select(self, *cols: "ColumnOrName") -> ParentDataFrame: ...
@overload
- def select(self, __cols: Sequence["ColumnOrName"]) -> ParentDataFrame:
- ...
+ def select(self, __cols: Sequence["ColumnOrName"]) -> ParentDataFrame: ...
def select(self, *cols: Union[Sequence["ColumnOrName"], "ColumnOrName"]) -> ParentDataFrame:
jdf = self._jdf.select(self._jcols(*cols))
return DataFrame(jdf, self.sparkSession)
@overload
- def selectExpr(self, *expr: str) -> ParentDataFrame:
- ...
+ def selectExpr(self, *expr: str) -> ParentDataFrame: ...
@overload
- def selectExpr(self, *expr: List[str]) -> ParentDataFrame:
- ...
+ def selectExpr(self, *expr: List[str]) -> ParentDataFrame: ...
def selectExpr(self, *expr: Union[str, List[str]]) -> ParentDataFrame:
if len(expr) == 1 and isinstance(expr[0], list):
@@ -1076,12 +1065,10 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
return DataFrame(jdf, self.sparkSession)
@overload
- def groupBy(self, *cols: "ColumnOrNameOrOrdinal") -> "GroupedData":
- ...
+ def groupBy(self, *cols: "ColumnOrNameOrOrdinal") -> "GroupedData": ...
@overload
- def groupBy(self, __cols: Sequence["ColumnOrNameOrOrdinal"]) -> "GroupedData":
- ...
+ def groupBy(self, __cols: Sequence["ColumnOrNameOrOrdinal"]) -> "GroupedData": ...
def groupBy(
self, *cols: Union[Sequence["ColumnOrNameOrOrdinal"], "ColumnOrNameOrOrdinal"]
@@ -1092,12 +1079,10 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
return GroupedData(jgd, self)
@overload
- def rollup(self, *cols: "ColumnOrNameOrOrdinal") -> "GroupedData":
- ...
+ def rollup(self, *cols: "ColumnOrNameOrOrdinal") -> "GroupedData": ...
@overload
- def rollup(self, __cols: Sequence["ColumnOrNameOrOrdinal"]) -> "GroupedData":
- ...
+ def rollup(self, __cols: Sequence["ColumnOrNameOrOrdinal"]) -> "GroupedData": ...
def rollup(
self, *cols: Union[Sequence["ColumnOrNameOrOrdinal"], "ColumnOrNameOrOrdinal"]
@@ -1108,12 +1093,10 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
return GroupedData(jgd, self)
@overload
- def cube(self, *cols: "ColumnOrNameOrOrdinal") -> "GroupedData":
- ...
+ def cube(self, *cols: "ColumnOrNameOrOrdinal") -> "GroupedData": ...
@overload
- def cube(self, __cols: Sequence["ColumnOrNameOrOrdinal"]) -> "GroupedData":
- ...
+ def cube(self, __cols: Sequence["ColumnOrNameOrOrdinal"]) -> "GroupedData": ...
def cube(
self, *cols: Union[Sequence["ColumnOrNameOrOrdinal"], "ColumnOrNameOrOrdinal"]
@@ -1145,7 +1128,7 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
assert ids is not None, "ids must not be None"
def to_jcols(
- cols: Union["ColumnOrName", List["ColumnOrName"], Tuple["ColumnOrName", ...]]
+ cols: Union["ColumnOrName", List["ColumnOrName"], Tuple["ColumnOrName", ...]],
) -> "JavaObject":
if isinstance(cols, list):
return self._jcols(*cols)
@@ -1333,8 +1316,7 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
to_replace: "LiteralType",
value: "OptionalPrimitiveType",
subset: Optional[List[str]] = ...,
- ) -> ParentDataFrame:
- ...
+ ) -> ParentDataFrame: ...
@overload
def replace(
@@ -1342,16 +1324,14 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
to_replace: List["LiteralType"],
value: List["OptionalPrimitiveType"],
subset: Optional[List[str]] = ...,
- ) -> ParentDataFrame:
- ...
+ ) -> ParentDataFrame: ...
@overload
def replace(
self,
to_replace: Dict["LiteralType", "OptionalPrimitiveType"],
subset: Optional[List[str]] = ...,
- ) -> ParentDataFrame:
- ...
+ ) -> ParentDataFrame: ...
@overload
def replace(
@@ -1359,8 +1339,7 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
to_replace: List["LiteralType"],
value: "OptionalPrimitiveType",
subset: Optional[List[str]] = ...,
- ) -> ParentDataFrame:
- ...
+ ) -> ParentDataFrame: ...
def replace( # type: ignore[misc]
self,
@@ -1484,8 +1463,7 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
col: str,
probabilities: Union[List[float], Tuple[float]],
relativeError: float,
- ) -> List[float]:
- ...
+ ) -> List[float]: ...
@overload
def approxQuantile(
@@ -1493,8 +1471,7 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
col: Union[List[str], Tuple[str]],
probabilities: Union[List[float], Tuple[float]],
relativeError: float,
- ) -> List[List[float]]:
- ...
+ ) -> List[List[float]]: ...
def approxQuantile(
self,
@@ -1873,12 +1850,10 @@ class DataFrameNaFunctions(ParentDataFrameNaFunctions):
return self.df.dropna(how=how, thresh=thresh, subset=subset)
@overload
- def fill(self, value: "LiteralType", subset: Optional[List[str]] = ...) -> ParentDataFrame:
- ...
+ def fill(self, value: "LiteralType", subset: Optional[List[str]] = ...) -> ParentDataFrame: ...
@overload
- def fill(self, value: Dict[str, "LiteralType"]) -> ParentDataFrame:
- ...
+ def fill(self, value: Dict[str, "LiteralType"]) -> ParentDataFrame: ...
def fill(
self,
@@ -1893,16 +1868,14 @@ class DataFrameNaFunctions(ParentDataFrameNaFunctions):
to_replace: List["LiteralType"],
value: List["OptionalPrimitiveType"],
subset: Optional[List[str]] = ...,
- ) -> ParentDataFrame:
- ...
+ ) -> ParentDataFrame: ...
@overload
def replace(
self,
to_replace: Dict["LiteralType", "OptionalPrimitiveType"],
subset: Optional[List[str]] = ...,
- ) -> ParentDataFrame:
- ...
+ ) -> ParentDataFrame: ...
@overload
def replace(
@@ -1910,8 +1883,7 @@ class DataFrameNaFunctions(ParentDataFrameNaFunctions):
to_replace: List["LiteralType"],
value: "OptionalPrimitiveType",
subset: Optional[List[str]] = ...,
- ) -> ParentDataFrame:
- ...
+ ) -> ParentDataFrame: ...
def replace( # type: ignore[misc]
self,
@@ -1934,8 +1906,7 @@ class DataFrameStatFunctions(ParentDataFrameStatFunctions):
col: str,
probabilities: Union[List[float], Tuple[float]],
relativeError: float,
- ) -> List[float]:
- ...
+ ) -> List[float]: ...
@overload
def approxQuantile(
@@ -1943,8 +1914,7 @@ class DataFrameStatFunctions(ParentDataFrameStatFunctions):
col: Union[List[str], Tuple[str]],
probabilities: Union[List[float], Tuple[float]],
relativeError: float,
- ) -> List[List[float]]:
- ...
+ ) -> List[List[float]]: ...
def approxQuantile(
self,
@@ -2001,7 +1971,7 @@ def _test() -> None:
SparkSession.builder.master("local[4]").appName("sql.classic.dataframe tests").getOrCreate()
)
globs["spark"] = spark
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.sql.dataframe,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF,
diff --git a/python/pyspark/sql/classic/table_arg.py b/python/pyspark/sql/classic/table_arg.py
index 3f3115ed8d3f..97f6fa79e189 100644
--- a/python/pyspark/sql/classic/table_arg.py
+++ b/python/pyspark/sql/classic/table_arg.py
@@ -21,7 +21,6 @@ from pyspark.sql.classic.column import _to_java_column, _to_seq
from pyspark.sql.table_arg import TableArg as ParentTableArg
from pyspark.sql.utils import get_active_spark_context
-
if TYPE_CHECKING:
from py4j.java_gateway import JavaObject
from pyspark.sql._typing import ColumnOrName
diff --git a/python/pyspark/sql/classic/window.py b/python/pyspark/sql/classic/window.py
index c7bc92739b24..3b3c8785cb4c 100644
--- a/python/pyspark/sql/classic/window.py
+++ b/python/pyspark/sql/classic/window.py
@@ -32,7 +32,7 @@ __all__ = ["Window", "WindowSpec"]
def _to_java_cols(
- cols: Tuple[Union["ColumnOrName", Sequence["ColumnOrName"]], ...]
+ cols: Tuple[Union["ColumnOrName", Sequence["ColumnOrName"]], ...],
) -> "JavaObject":
from pyspark.sql.classic.column import _to_seq, _to_java_column
@@ -136,7 +136,7 @@ def _test() -> None:
SparkSession.builder.master("local[4]").appName("sql.classic.window tests").getOrCreate()
)
globs["spark"] = spark
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.sql.window,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF,
diff --git a/python/pyspark/sql/column.py b/python/pyspark/sql/column.py
index 628fd9923e7f..a38ed64eb700 100644
--- a/python/pyspark/sql/column.py
+++ b/python/pyspark/sql/column.py
@@ -39,7 +39,6 @@ __all__ = ["Column"]
class Column(TableValuedFunctionArgument):
-
"""
A column in a DataFrame.
@@ -79,92 +78,77 @@ class Column(TableValuedFunctionArgument):
# arithmetic operators
@dispatch_col_method
- def __neg__(self) -> "Column":
- ...
+ def __neg__(self) -> "Column": ...
@dispatch_col_method
def __add__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def __sub__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def __mul__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def __div__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def __truediv__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def __mod__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def __radd__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def __rsub__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def __rmul__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def __rdiv__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def __rtruediv__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def __rmod__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def __pow__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def __rpow__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
# logistic operators
@dispatch_col_method
@@ -186,26 +170,22 @@ class Column(TableValuedFunctionArgument):
@dispatch_col_method
def __lt__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def __le__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def __ge__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def __gt__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def eqNullSafe(
@@ -281,30 +261,25 @@ class Column(TableValuedFunctionArgument):
@dispatch_col_method
def __and__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def __or__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
- def __invert__(self) -> "Column":
- ...
+ def __invert__(self) -> "Column": ...
@dispatch_col_method
def __rand__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
@dispatch_col_method
def __ror__(
self, other: Union["Column", "LiteralType", "DecimalLiteral", "DateTimeLiteral"]
- ) -> "Column":
- ...
+ ) -> "Column": ...
# container operators
@dispatch_col_method
@@ -648,8 +623,7 @@ class Column(TableValuedFunctionArgument):
...
@dispatch_col_method
- def __iter__(self) -> None:
- ...
+ def __iter__(self) -> None: ...
# string methods
@dispatch_col_method
@@ -825,12 +799,10 @@ class Column(TableValuedFunctionArgument):
...
@overload
- def substr(self, startPos: int, length: int) -> "Column":
- ...
+ def substr(self, startPos: int, length: int) -> "Column": ...
@overload
- def substr(self, startPos: "Column", length: "Column") -> "Column":
- ...
+ def substr(self, startPos: "Column", length: "Column") -> "Column": ...
@dispatch_col_method
def substr(self, startPos: Union[int, "Column"], length: Union[int, "Column"]) -> "Column":
@@ -1610,16 +1582,13 @@ class Column(TableValuedFunctionArgument):
...
@dispatch_col_method
- def __nonzero__(self) -> None:
- ...
+ def __nonzero__(self) -> None: ...
@dispatch_col_method
- def __bool__(self) -> None:
- ...
+ def __bool__(self) -> None: ...
@dispatch_col_method
- def __repr__(self) -> str:
- ...
+ def __repr__(self) -> str: ...
def _test() -> None:
@@ -1631,7 +1600,7 @@ def _test() -> None:
spark = SparkSession.builder.master("local[4]").appName("sql.column tests").getOrCreate()
globs["spark"] = spark
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.sql.column,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF,
diff --git a/python/pyspark/sql/conf.py b/python/pyspark/sql/conf.py
index 9a4cc2e7e162..cdd7665b8437 100644
--- a/python/pyspark/sql/conf.py
+++ b/python/pyspark/sql/conf.py
@@ -163,7 +163,7 @@ def _test() -> None:
globs = pyspark.sql.conf.__dict__.copy()
spark = SparkSession.builder.master("local[4]").appName("sql.conf tests").getOrCreate()
globs["spark"] = spark
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.sql.conf, globs=globs, optionflags=doctest.ELLIPSIS
)
spark.stop()
diff --git a/python/pyspark/sql/connect/_typing.py b/python/pyspark/sql/connect/_typing.py
index 7c8351185d6e..50bb02be0948 100644
--- a/python/pyspark/sql/connect/_typing.py
+++ b/python/pyspark/sql/connect/_typing.py
@@ -26,7 +26,6 @@ from pyspark.sql.column import Column
from pyspark.sql.connect.types import DataType
from pyspark.sql.streaming.state import GroupState
-
ColumnOrName = Union[Column, str]
ColumnOrNameOrOrdinal = Union[Column, str, int]
@@ -77,16 +76,12 @@ class UserDefinedFunctionLike(Protocol):
deterministic: bool
@property
- def returnType(self) -> DataType:
- ...
+ def returnType(self) -> DataType: ...
- def __call__(self, *args: ColumnOrName) -> Column:
- ...
+ def __call__(self, *args: ColumnOrName) -> Column: ...
- def asNondeterministic(self) -> "UserDefinedFunctionLike":
- ...
+ def asNondeterministic(self) -> "UserDefinedFunctionLike": ...
class UserDefinedFunctionCallable(Protocol):
- def __call__(self, *_: ColumnOrName) -> Column:
- ...
+ def __call__(self, *_: ColumnOrName) -> Column: ...
diff --git a/python/pyspark/sql/connect/avro/functions.py b/python/pyspark/sql/connect/avro/functions.py
index 611c1ac1d1f3..d1edfc813a80 100644
--- a/python/pyspark/sql/connect/avro/functions.py
+++ b/python/pyspark/sql/connect/avro/functions.py
@@ -118,7 +118,7 @@ def _test() -> None:
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.sql.connect.avro.functions,
globs=globs,
optionflags=doctest.ELLIPSIS
diff --git a/python/pyspark/sql/connect/catalog.py b/python/pyspark/sql/connect/catalog.py
index 3575f9a187c8..6bb418230e9e 100644
--- a/python/pyspark/sql/connect/catalog.py
+++ b/python/pyspark/sql/connect/catalog.py
@@ -345,7 +345,7 @@ def _test() -> None:
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.sql.connect.catalog,
globs=globs,
optionflags=doctest.ELLIPSIS
diff --git a/python/pyspark/sql/connect/client/artifact.py b/python/pyspark/sql/connect/client/artifact.py
index 3e70cb790548..277c313eef62 100644
--- a/python/pyspark/sql/connect/client/artifact.py
+++ b/python/pyspark/sql/connect/client/artifact.py
@@ -36,7 +36,6 @@ import grpc
import pyspark.sql.connect.proto as proto
import pyspark.sql.connect.proto.base_pb2_grpc as grpc_lib
-
JAR_PREFIX: str = "jars"
PYFILE_PREFIX: str = "pyfiles"
ARCHIVE_PREFIX: str = "archives"
diff --git a/python/pyspark/sql/connect/client/core.py b/python/pyspark/sql/connect/client/core.py
index e86d07aed6d3..1a36699b7881 100644
--- a/python/pyspark/sql/connect/client/core.py
+++ b/python/pyspark/sql/connect/client/core.py
@@ -999,7 +999,7 @@ class SparkConnectClient(object):
logger.debug("Fetching the resources")
cmd = pb2.Command()
cmd.get_resources_command.SetInParent()
- (_, properties, _) = self.execute_command(cmd)
+ _, properties, _ = self.execute_command(cmd)
resources = properties["get_resources_command_result"]
return resources
@@ -1559,7 +1559,7 @@ class SparkConnectClient(object):
else:
if observed_metrics.name == "__python_accumulator__":
for metric in observed_metrics.metrics:
- (aid, update) = pickleSer.loads(LiteralExpression._to_value(metric))
+ aid, update = pickleSer.loads(LiteralExpression._to_value(metric))
if aid == SpecialAccumulatorIds.SQL_UDF_PROFIER:
self._profiler_collector._update(update)
elif observed_metrics.name in observations:
@@ -2256,7 +2256,7 @@ class SparkConnectClient(object):
logger.debug("Creating the ResourceProfile")
cmd = pb2.Command()
cmd.create_resource_profile_command.profile.CopyFrom(profile)
- (_, properties, _) = self.execute_command(cmd)
+ _, properties, _ = self.execute_command(cmd)
profile_id = properties["create_resource_profile_command_result"]
return profile_id
@@ -2269,7 +2269,7 @@ class SparkConnectClient(object):
[pb2.ObjectRef(id=cache_id) for cache_id in cache_ids]
)
command.ml_command.delete.evict_only = evict_only
- (_, properties, _) = self.execute_command(command)
+ _, properties, _ = self.execute_command(command)
assert properties is not None
@@ -2304,7 +2304,7 @@ class SparkConnectClient(object):
def _get_ml_cache_info(self) -> List[str]:
command = pb2.Command()
command.ml_command.get_cache_info.SetInParent()
- (_, properties, _) = self.execute_command(command)
+ _, properties, _ = self.execute_command(command)
assert properties is not None
@@ -2319,7 +2319,7 @@ class SparkConnectClient(object):
command.ml_command.get_model_size.CopyFrom(
pb2.MlCommand.GetModelSize(model_ref=pb2.ObjectRef(id=model_ref_id))
)
- (_, properties, _) = self.execute_command(command)
+ _, properties, _ = self.execute_command(command)
assert properties is not None
diff --git a/python/pyspark/sql/connect/client/reattach.py b/python/pyspark/sql/connect/client/reattach.py
index 7e168ec0fdd6..5750dd045a91 100644
--- a/python/pyspark/sql/connect/client/reattach.py
+++ b/python/pyspark/sql/connect/client/reattach.py
@@ -57,9 +57,9 @@ class ExecutePlanResponseReattachableIterator(Generator):
_lock: ClassVar[RLock] = RLock()
_release_thread_pool_instance: Optional[ThreadPoolExecutor] = None
- _instances: ClassVar[
- weakref.WeakSet["ExecutePlanResponseReattachableIterator"]
- ] = weakref.WeakSet()
+ _instances: ClassVar[weakref.WeakSet["ExecutePlanResponseReattachableIterator"]] = (
+ weakref.WeakSet()
+ )
def __new__(cls, *args: Any, **kwargs: Any) -> "ExecutePlanResponseReattachableIterator":
instance = super().__new__(cls)
diff --git a/python/pyspark/sql/connect/column.py b/python/pyspark/sql/connect/column.py
index ad011d723ed9..2fca83748f1e 100644
--- a/python/pyspark/sql/connect/column.py
+++ b/python/pyspark/sql/connect/column.py
@@ -52,7 +52,6 @@ from pyspark.sql.connect.expressions import (
)
from pyspark.errors.utils import with_origin_to_class
-
if TYPE_CHECKING:
from pyspark.sql.connect._typing import (
LiteralType,
@@ -612,7 +611,7 @@ def _test() -> None:
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.sql.column,
globs=globs,
optionflags=doctest.ELLIPSIS
diff --git a/python/pyspark/sql/connect/conf.py b/python/pyspark/sql/connect/conf.py
index 890bad25877e..fe19f20a9ebc 100644
--- a/python/pyspark/sql/connect/conf.py
+++ b/python/pyspark/sql/connect/conf.py
@@ -146,7 +146,7 @@ def _test() -> None:
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.sql.connect.conf,
globs=globs,
optionflags=doctest.ELLIPSIS
diff --git a/python/pyspark/sql/connect/dataframe.py b/python/pyspark/sql/connect/dataframe.py
index 06f94a17940e..ad97b44cf28e 100644
--- a/python/pyspark/sql/connect/dataframe.py
+++ b/python/pyspark/sql/connect/dataframe.py
@@ -91,7 +91,6 @@ from pyspark.sql.pandas.types import from_arrow_schema, to_arrow_schema
from pyspark.sql.pandas.functions import _validate_vectorized_udf # type: ignore[attr-defined]
from pyspark.sql.table_arg import TableArg
-
if TYPE_CHECKING:
from pyspark.sql.connect._typing import (
ColumnOrName,
@@ -279,12 +278,12 @@ class DataFrame(ParentDataFrame):
return len(self.select().take(1)) == 0
@overload
- def select(self, *cols: "ColumnOrName") -> ParentDataFrame:
- ...
+ def select(self, *cols: "ColumnOrName") -> ParentDataFrame: ...
@overload
- def select(self, __cols: Union[Sequence["ColumnOrName"], "ColumnOrName"]) -> ParentDataFrame:
- ...
+ def select(
+ self, __cols: Union[Sequence["ColumnOrName"], "ColumnOrName"]
+ ) -> ParentDataFrame: ...
def select(self, *cols: Union[Sequence["ColumnOrName"], "ColumnOrName"]) -> ParentDataFrame:
return DataFrame(
@@ -591,12 +590,10 @@ class DataFrame(ParentDataFrame):
return self.head()
@overload
- def groupBy(self, *cols: "ColumnOrNameOrOrdinal") -> "GroupedData":
- ...
+ def groupBy(self, *cols: "ColumnOrNameOrOrdinal") -> "GroupedData": ...
@overload
- def groupBy(self, __cols: Sequence["ColumnOrNameOrOrdinal"]) -> "GroupedData":
- ...
+ def groupBy(self, __cols: Sequence["ColumnOrNameOrOrdinal"]) -> "GroupedData": ...
def groupBy(
self, *cols: Union[Sequence["ColumnOrNameOrOrdinal"], "ColumnOrNameOrOrdinal"]
@@ -608,12 +605,10 @@ class DataFrame(ParentDataFrame):
groupby = groupBy # type: ignore[assignment]
@overload
- def rollup(self, *cols: "ColumnOrNameOrOrdinal") -> "GroupedData":
- ...
+ def rollup(self, *cols: "ColumnOrNameOrOrdinal") -> "GroupedData": ...
@overload
- def rollup(self, __cols: Sequence["ColumnOrNameOrOrdinal"]) -> "GroupedData":
- ...
+ def rollup(self, __cols: Sequence["ColumnOrNameOrOrdinal"]) -> "GroupedData": ...
def rollup(
self, *cols: Union[Sequence["ColumnOrNameOrOrdinal"], "ColumnOrNameOrOrdinal"]
@@ -623,12 +618,10 @@ class DataFrame(ParentDataFrame):
)
@overload
- def cube(self, *cols: "ColumnOrName") -> "GroupedData":
- ...
+ def cube(self, *cols: "ColumnOrName") -> "GroupedData": ...
@overload
- def cube(self, __cols: Sequence["ColumnOrNameOrOrdinal"]) -> "GroupedData":
- ...
+ def cube(self, __cols: Sequence["ColumnOrNameOrOrdinal"]) -> "GroupedData": ...
def cube(
self, *cols: Union[Sequence["ColumnOrNameOrOrdinal"], "ColumnOrNameOrOrdinal"]
@@ -654,12 +647,10 @@ class DataFrame(ParentDataFrame):
)
@overload
- def head(self) -> Optional[Row]:
- ...
+ def head(self) -> Optional[Row]: ...
@overload
- def head(self, n: int) -> List[Row]:
- ...
+ def head(self, n: int) -> List[Row]: ...
def head(self, n: Optional[int] = None) -> Union[Optional[Row], List[Row]]:
if n is None:
@@ -923,7 +914,7 @@ class DataFrame(ParentDataFrame):
assert ids is not None, "ids must not be None"
def _convert_cols(
- cols: Optional[Union["ColumnOrName", List["ColumnOrName"], Tuple["ColumnOrName", ...]]]
+ cols: Optional[Union["ColumnOrName", List["ColumnOrName"], Tuple["ColumnOrName", ...]]],
) -> List[Column]:
if cols is None:
return []
@@ -1699,12 +1690,10 @@ class DataFrame(ParentDataFrame):
return self._col(name)
@overload
- def __getitem__(self, item: Union[int, str]) -> Column:
- ...
+ def __getitem__(self, item: Union[int, str]) -> Column: ...
@overload
- def __getitem__(self, item: Union[Column, List, Tuple]) -> ParentDataFrame:
- ...
+ def __getitem__(self, item: Union[Column, List, Tuple]) -> ParentDataFrame: ...
def __getitem__(
self, item: Union[int, str, Column, List, Tuple]
@@ -2334,7 +2323,7 @@ def _test() -> None:
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.sql.dataframe,
globs=globs,
optionflags=doctest.ELLIPSIS
diff --git a/python/pyspark/sql/connect/datasource.py b/python/pyspark/sql/connect/datasource.py
index c12606aa323b..15f8edafbf50 100644
--- a/python/pyspark/sql/connect/datasource.py
+++ b/python/pyspark/sql/connect/datasource.py
@@ -18,7 +18,6 @@ from typing import Type, TYPE_CHECKING
from pyspark.sql.datasource import DataSourceRegistration as PySparkDataSourceRegistration
-
if TYPE_CHECKING:
from pyspark.sql.datasource import DataSource
from pyspark.sql.connect.session import SparkSession
diff --git a/python/pyspark/sql/connect/expressions.py b/python/pyspark/sql/connect/expressions.py
index 793b562cdd99..cb33aa749b22 100644
--- a/python/pyspark/sql/connect/expressions.py
+++ b/python/pyspark/sql/connect/expressions.py
@@ -104,8 +104,7 @@ class Expression:
def to_plan( # type: ignore[empty-body]
self, session: "SparkConnectClient"
- ) -> "proto.Expression":
- ...
+ ) -> "proto.Expression": ...
def __repr__(self) -> str: # type: ignore[empty-body]
...
diff --git a/python/pyspark/sql/connect/functions/__init__.py b/python/pyspark/sql/connect/functions/__init__.py
index ee7e3bda67b4..7110860521df 100644
--- a/python/pyspark/sql/connect/functions/__init__.py
+++ b/python/pyspark/sql/connect/functions/__init__.py
@@ -16,6 +16,7 @@
#
"""PySpark Functions with Spark Connect"""
+
from pyspark.testing.utils import should_test_connect
if should_test_connect:
diff --git a/python/pyspark/sql/connect/functions/builtin.py b/python/pyspark/sql/connect/functions/builtin.py
index 0510de9361f1..fe0c25ced2ad 100644
--- a/python/pyspark/sql/connect/functions/builtin.py
+++ b/python/pyspark/sql/connect/functions/builtin.py
@@ -71,7 +71,6 @@ from pyspark.sql.utils import enum_to_value as _enum_to_value
# for code reuse.
from pyspark.sql.functions import arrow_udf, pandas_udf # noqa: F401
-
if TYPE_CHECKING:
from pyspark.sql.connect._typing import (
ColumnOrName,
@@ -3142,13 +3141,11 @@ current_date.__doc__ = pysparkfuncs.current_date.__doc__
@overload
-def current_time() -> Column:
- ...
+def current_time() -> Column: ...
@overload
-def current_time(precision: int) -> Column:
- ...
+def current_time(precision: int) -> Column: ...
def current_time(precision: Optional[int] = None) -> Column:
@@ -3435,13 +3432,11 @@ unix_seconds.__doc__ = pysparkfuncs.unix_seconds.__doc__
@overload
-def to_time(str: "ColumnOrName") -> Column:
- ...
+def to_time(str: "ColumnOrName") -> Column: ...
@overload
-def to_time(str: "ColumnOrName", format: "ColumnOrName") -> Column:
- ...
+def to_time(str: "ColumnOrName", format: "ColumnOrName") -> Column: ...
def to_time(str: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> Column:
@@ -3455,13 +3450,11 @@ to_time.__doc__ = pysparkfuncs.to_time.__doc__
@overload
-def to_timestamp(col: "ColumnOrName") -> Column:
- ...
+def to_timestamp(col: "ColumnOrName") -> Column: ...
@overload
-def to_timestamp(col: "ColumnOrName", format: str) -> Column:
- ...
+def to_timestamp(col: "ColumnOrName", format: str) -> Column: ...
def to_timestamp(col: "ColumnOrName", format: Optional[str] = None) -> Column:
@@ -3475,13 +3468,11 @@ to_timestamp.__doc__ = pysparkfuncs.to_timestamp.__doc__
@overload
-def try_to_time(str: "ColumnOrName") -> Column:
- ...
+def try_to_time(str: "ColumnOrName") -> Column: ...
@overload
-def try_to_time(str: "ColumnOrName", format: "ColumnOrName") -> Column:
- ...
+def try_to_time(str: "ColumnOrName", format: "ColumnOrName") -> Column: ...
def try_to_time(str: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> Column:
@@ -3603,13 +3594,11 @@ from_unixtime.__doc__ = pysparkfuncs.from_unixtime.__doc__
@overload
-def unix_timestamp(timestamp: "ColumnOrName", format: str = ...) -> Column:
- ...
+def unix_timestamp(timestamp: "ColumnOrName", format: str = ...) -> Column: ...
@overload
-def unix_timestamp() -> Column:
- ...
+def unix_timestamp() -> Column: ...
def unix_timestamp(
@@ -3998,8 +3987,7 @@ def make_timestamp(
hours: "ColumnOrName",
mins: "ColumnOrName",
secs: "ColumnOrName",
-) -> Column:
- ...
+) -> Column: ...
@overload
@@ -4011,20 +3999,17 @@ def make_timestamp(
mins: "ColumnOrName",
secs: "ColumnOrName",
timezone: "ColumnOrName",
-) -> Column:
- ...
+) -> Column: ...
@overload
-def make_timestamp(*, date: "ColumnOrName", time: "ColumnOrName") -> Column:
- ...
+def make_timestamp(*, date: "ColumnOrName", time: "ColumnOrName") -> Column: ...
@overload
def make_timestamp(
*, date: "ColumnOrName", time: "ColumnOrName", timezone: "ColumnOrName"
-) -> Column:
- ...
+) -> Column: ...
def make_timestamp(
@@ -4095,8 +4080,7 @@ def try_make_timestamp(
hours: "ColumnOrName",
mins: "ColumnOrName",
secs: "ColumnOrName",
-) -> Column:
- ...
+) -> Column: ...
@overload
@@ -4108,20 +4092,17 @@ def try_make_timestamp(
mins: "ColumnOrName",
secs: "ColumnOrName",
timezone: "ColumnOrName",
-) -> Column:
- ...
+) -> Column: ...
@overload
-def try_make_timestamp(*, date: "ColumnOrName", time: "ColumnOrName") -> Column:
- ...
+def try_make_timestamp(*, date: "ColumnOrName", time: "ColumnOrName") -> Column: ...
@overload
def try_make_timestamp(
*, date: "ColumnOrName", time: "ColumnOrName", timezone: "ColumnOrName"
-) -> Column:
- ...
+) -> Column: ...
def try_make_timestamp(
@@ -4237,8 +4218,7 @@ def make_timestamp_ntz(
hours: "ColumnOrName",
mins: "ColumnOrName",
secs: "ColumnOrName",
-) -> Column:
- ...
+) -> Column: ...
@overload
@@ -4246,8 +4226,7 @@ def make_timestamp_ntz(
*,
date: "ColumnOrName",
time: "ColumnOrName",
-) -> Column:
- ...
+) -> Column: ...
def make_timestamp_ntz(
@@ -4297,8 +4276,7 @@ def try_make_timestamp_ntz(
hours: "ColumnOrName",
mins: "ColumnOrName",
secs: "ColumnOrName",
-) -> Column:
- ...
+) -> Column: ...
@overload
@@ -4306,8 +4284,7 @@ def try_make_timestamp_ntz(
*,
date: "ColumnOrName",
time: "ColumnOrName",
-) -> Column:
- ...
+) -> Column: ...
def try_make_timestamp_ntz(
@@ -5480,7 +5457,7 @@ def _test() -> None:
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.sql.connect.functions.builtin,
globs=globs,
optionflags=doctest.ELLIPSIS
diff --git a/python/pyspark/sql/connect/functions/partitioning.py b/python/pyspark/sql/connect/functions/partitioning.py
index 8497345f7507..6c8097621a60 100644
--- a/python/pyspark/sql/connect/functions/partitioning.py
+++ b/python/pyspark/sql/connect/functions/partitioning.py
@@ -22,7 +22,6 @@ from pyspark.sql.column import Column
from pyspark.sql.connect.functions.builtin import _to_col, _invoke_function_over_columns
from pyspark.sql.connect.functions.builtin import lit, _invoke_function
-
if TYPE_CHECKING:
from pyspark.sql.connect._typing import ColumnOrName
@@ -90,7 +89,7 @@ def _test() -> None:
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.sql.connect.functions.partitioning,
globs=globs,
optionflags=doctest.ELLIPSIS
diff --git a/python/pyspark/sql/connect/group.py b/python/pyspark/sql/connect/group.py
index dfd8e756ec6e..9db9e09686b2 100644
--- a/python/pyspark/sql/connect/group.py
+++ b/python/pyspark/sql/connect/group.py
@@ -117,12 +117,10 @@ class GroupedData:
return f"GroupedData[{grouping_str}, value: [{value_str}], type: {type_str}]"
@overload
- def agg(self, *exprs: Column) -> "DataFrame":
- ...
+ def agg(self, *exprs: Column) -> "DataFrame": ...
@overload
- def agg(self, __exprs: Dict[str, str]) -> "DataFrame":
- ...
+ def agg(self, __exprs: Dict[str, str]) -> "DataFrame": ...
def agg(self, *exprs: Union[Column, Dict[str, str]]) -> "DataFrame":
from pyspark.sql.connect.dataframe import DataFrame
@@ -611,7 +609,7 @@ def _test() -> None:
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.sql.connect.group,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF,
diff --git a/python/pyspark/sql/connect/merge.py b/python/pyspark/sql/connect/merge.py
index 7a1c684ca37a..9106e4d09aab 100644
--- a/python/pyspark/sql/connect/merge.py
+++ b/python/pyspark/sql/connect/merge.py
@@ -238,7 +238,7 @@ def _test() -> None:
.remote(os.environ.get("SPARK_CONNECT_TESTING_REMOTE", "local[4]"))
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.sql.connect.merge,
globs=globs,
optionflags=doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF,
diff --git a/python/pyspark/sql/connect/observation.py b/python/pyspark/sql/connect/observation.py
index efb526d5ca8a..1796978a609b 100644
--- a/python/pyspark/sql/connect/observation.py
+++ b/python/pyspark/sql/connect/observation.py
@@ -28,7 +28,6 @@ from pyspark.sql.connect.dataframe import DataFrame
from pyspark.sql.observation import Observation as PySparkObservation
import pyspark.sql.connect.plan as plan
-
__all__ = ["Observation"]
@@ -102,7 +101,7 @@ def _test() -> None:
.getOrCreate()
)
- (failure_count, test_count) = doctest.testmod(
+ failure_count, test_count = doctest.testmod(
pyspark.sql.connect.observation,
globs=globs,
optionflags=doctest.ELLIPSIS
diff --git a/python/pyspark/sql/connect/plan.py b/python/pyspark/sql/connect/plan.py
index 36118359f9e6..4ff7451d1314 100644
--- a/python/pyspark/sql/connect/plan.py
+++ b/python/pyspark/sql/connect/plan.py
@@ -158,7 +158,7 @@ class LogicalPlan:
@staticmethod
def _collect_references(
- cols_or_exprs: Sequence[Union[Column, Expression]]
+ cols_or_exprs: Sequence[Union[Column, Expression]],
) -> Sequence["LogicalPlan"]:
references: List[LogicalPlan] = []
@@ -1017,9 +1017,7 @@ class Join(LogicalPlan):
self._collect_references(
[]
if on is None or isinstance(on, str)
- else [on]
- if isinstance(on, Column)
- else [c for c in on if isinstance(c, Column)]
+ else [on] if isinstance(on, Column) else [c for c in on if isinstance(c, Column)]
),
)
self.left = cast(LogicalPlan, left)
@@ -1140,9 +1138,9 @@ class AsOfJoin(LogicalPlan):
+ (
[]
if on is None or isinstance(on, str)
- else [on]
- if isinstance(on, Column)
- else [c for c in on if isinstance(c, Column)]
+ else (
+ [on] if isinstance(on, Column) else [c for c in on if isinstance(c, Column)]
+ )
)
+ ([tolerance] if tolerance is not None else [])
),
@@ -2717,9 +2715,7 @@ class PythonUDTF:
self._return_type: Optional[DataType] = (
None
if return_type is None
- else UnparsedDataType(return_type)
- if isinstance(return_type, str)
- else return_type
+ else UnparsedDataType(return_type) if isinstance(return_type, str) else return_type
)
self._eval_type = eval_type
self._python_ver = python_ver
diff --git a/python/pyspark/sql/connect/proto/base_pb2.py b/python/pyspark/sql/connect/proto/base_pb2.py
index 24c3a5ab31b0..a17d27544ac4 100644
--- a/python/pyspark/sql/connect/proto/base_pb2.py
+++ b/python/pyspark/sql/connect/proto/base_pb2.py
@@ -20,6 +20,7 @@
# source: spark/connect/base.proto
# Protobuf Python Version: 6.33.5
"""Generated protocol buffer code."""
+
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
@@ -43,7 +44,6 @@ from pyspark.sql.connect.proto import types_pb2 as spark_dot_connect_dot_types__
from pyspark.sql.connect.proto import ml_pb2 as spark_dot_connect_dot_ml__pb2
from pyspark.sql.connect.proto import pipelines_pb2 as spark_dot_connect_dot_pipelines__pb2
-
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto\x1a\x16spark/connect/ml.proto\x1a\x1dspark/connect/pipelines.proto"\xe3\x03\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.conn [...]
)
@@ -53,20 +53,20 @@ _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "pyspark.sql.connect.proto.base_pb2", _globals)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals[
- "DESCRIPTOR"
- ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
- _globals[
- "_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY"
- ]._loaded_options = None
+ _globals["DESCRIPTOR"]._serialized_options = (
+ b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ )
+ _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY"]._loaded_options = (
+ None
+ )
_globals[
"_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY"
]._serialized_options = b"8\001"
_globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._loaded_options = None
_globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_options = b"8\001"
- _globals[
- "_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY"
- ]._loaded_options = None
+ _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY"]._loaded_options = (
+ None
+ )
_globals[
"_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY"
]._serialized_options = b"8\001"
@@ -159,9 +159,9 @@ if not _descriptor._USE_C_DESCRIPTORS:
_globals[
"_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY"
]._serialized_start = 8937
- _globals[
- "_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY"
- ]._serialized_end = 9060
+ _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY"]._serialized_end = (
+ 9060
+ )
_globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_start = 9062
_globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_end = 9150
_globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_start = 9153
@@ -255,9 +255,9 @@ if not _descriptor._USE_C_DESCRIPTORS:
_globals[
"_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY"
]._serialized_start = 17234
- _globals[
- "_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY"
- ]._serialized_end = 17302
+ _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY"]._serialized_end = (
+ 17302
+ )
_globals["_FETCHERRORDETAILSRESPONSE_BREAKINGCHANGEINFO"]._serialized_start = 17360
_globals["_FETCHERRORDETAILSRESPONSE_BREAKINGCHANGEINFO"]._serialized_end = 17610
_globals["_FETCHERRORDETAILSRESPONSE_MITIGATIONCONFIG"]._serialized_start = 17612
diff --git a/python/pyspark/sql/connect/proto/base_pb2.pyi b/python/pyspark/sql/connect/proto/base_pb2.pyi
index 6650704799e3..f464344d44f5 100644
--- a/python/pyspark/sql/connect/proto/base_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/base_pb2.pyi
@@ -33,6 +33,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
+
import builtins
import collections.abc
import google.protobuf.any_pb2
@@ -200,6 +201,7 @@ class UserContext(google.protobuf.message.Message):
messages into this message. Extensions are stored as a `repeated` type to be able to
handle multiple active extensions.
"""
+
def __init__(
self,
*,
@@ -228,6 +230,7 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
@property
def plan(self) -> global___Plan:
"""(Required) The logical plan to be analyzed."""
+
def __init__(
self,
*,
@@ -351,6 +354,7 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
@property
def plan(self) -> global___Plan:
"""(Required) The logical plan to be analyzed."""
+
def __init__(
self,
*,
@@ -368,6 +372,7 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
@property
def plan(self) -> global___Plan:
"""(Required) The logical plan to be analyzed."""
+
def __init__(
self,
*,
@@ -385,6 +390,7 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
@property
def plan(self) -> global___Plan:
"""(Required) The logical plan to be analyzed."""
+
def __init__(
self,
*,
@@ -427,9 +433,11 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
@property
def target_plan(self) -> global___Plan:
"""(Required) The plan to be compared."""
+
@property
def other_plan(self) -> global___Plan:
"""(Required) The other plan to be compared."""
+
def __init__(
self,
*,
@@ -456,6 +464,7 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
@property
def plan(self) -> global___Plan:
"""(Required) The logical plan to get a hashCode."""
+
def __init__(
self,
*,
@@ -474,9 +483,11 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
@property
def relation(self) -> pyspark.sql.connect.proto.relations_pb2.Relation:
"""(Required) The logical plan to persist."""
+
@property
def storage_level(self) -> pyspark.sql.connect.proto.common_pb2.StorageLevel:
"""(Optional) The storage level."""
+
def __init__(
self,
*,
@@ -548,6 +559,7 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
@property
def relation(self) -> pyspark.sql.connect.proto.relations_pb2.Relation:
"""(Required) The logical plan to get the storage level."""
+
def __init__(
self,
*,
@@ -769,9 +781,7 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
self, oneof_group: typing_extensions.Literal["_client_type", b"_client_type"]
) -> typing_extensions.Literal["client_type"] | None: ...
@typing.overload
- def WhichOneof(
- self, oneof_group: typing_extensions.Literal["analyze", b"analyze"]
- ) -> (
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["analyze", b"analyze"]) -> (
typing_extensions.Literal[
"schema",
"explain",
@@ -884,6 +894,7 @@ class AnalyzePlanResponse(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""A best-effort snapshot of the files that compose this Dataset"""
+
def __init__(
self,
*,
@@ -972,6 +983,7 @@ class AnalyzePlanResponse(google.protobuf.message.Message):
@property
def storage_level(self) -> pyspark.sql.connect.proto.common_pb2.StorageLevel:
"""(Required) The StorageLevel as a result of get_storage_level request."""
+
def __init__(
self,
*,
@@ -1141,9 +1153,7 @@ class AnalyzePlanResponse(google.protobuf.message.Message):
b"unpersist",
],
) -> None: ...
- def WhichOneof(
- self, oneof_group: typing_extensions.Literal["result", b"result"]
- ) -> (
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["result", b"result"]) -> (
typing_extensions.Literal[
"schema",
"explain",
@@ -1183,6 +1193,7 @@ class ExecutePlanRequest(google.protobuf.message.Message):
@property
def extension(self) -> google.protobuf.any_pb2.Any:
"""Extension type for request options"""
+
def __init__(
self,
*,
@@ -1275,6 +1286,7 @@ class ExecutePlanRequest(google.protobuf.message.Message):
"""Repeated element for options that can be passed to the request. This element is currently
unused but allows to pass in an extension value used for arbitrary options.
"""
+
@property
def tags(
self,
@@ -1283,6 +1295,7 @@ class ExecutePlanRequest(google.protobuf.message.Message):
Tags cannot contain ',' character and cannot be empty strings.
Used by Interrupt with interrupt.tag.
"""
+
def __init__(
self,
*,
@@ -1292,8 +1305,9 @@ class ExecutePlanRequest(google.protobuf.message.Message):
operation_id: builtins.str | None = ...,
plan: global___Plan | None = ...,
client_type: builtins.str | None = ...,
- request_options: collections.abc.Iterable[global___ExecutePlanRequest.RequestOption]
- | None = ...,
+ request_options: (
+ collections.abc.Iterable[global___ExecutePlanRequest.RequestOption] | None
+ ) = ...,
tags: collections.abc.Iterable[builtins.str] | None = ...,
) -> None: ...
def HasField(
@@ -1522,10 +1536,12 @@ class ExecutePlanResponse(google.protobuf.message.Message):
name: builtins.str = ...,
plan_id: builtins.int = ...,
parent: builtins.int = ...,
- execution_metrics: collections.abc.Mapping[
- builtins.str, global___ExecutePlanResponse.Metrics.MetricValue
- ]
- | None = ...,
+ execution_metrics: (
+ collections.abc.Mapping[
+ builtins.str, global___ExecutePlanResponse.Metrics.MetricValue
+ ]
+ | None
+ ) = ...,
) -> None: ...
def ClearField(
self,
@@ -1574,8 +1590,9 @@ class ExecutePlanResponse(google.protobuf.message.Message):
def __init__(
self,
*,
- metrics: collections.abc.Iterable[global___ExecutePlanResponse.Metrics.MetricObject]
- | None = ...,
+ metrics: (
+ collections.abc.Iterable[global___ExecutePlanResponse.Metrics.MetricObject] | None
+ ) = ...,
) -> None: ...
def ClearField(
self, field_name: typing_extensions.Literal["metrics", b"metrics"]
@@ -1615,14 +1632,17 @@ class ExecutePlanResponse(google.protobuf.message.Message):
"""A list of errors that occurred while collecting the observed metrics.
If the length is 0, it means no errors occurred.
"""
+
def __init__(
self,
*,
name: builtins.str = ...,
- values: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
- ]
- | None = ...,
+ values: (
+ collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+ ]
+ | None
+ ) = ...,
keys: collections.abc.Iterable[builtins.str] | None = ...,
plan_id: builtins.int = ...,
root_error_idx: builtins.int | None = ...,
@@ -1726,10 +1746,10 @@ class ExecutePlanResponse(google.protobuf.message.Message):
def __init__(
self,
*,
- stages: collections.abc.Iterable[
- global___ExecutePlanResponse.ExecutionProgress.StageInfo
- ]
- | None = ...,
+ stages: (
+ collections.abc.Iterable[global___ExecutePlanResponse.ExecutionProgress.StageInfo]
+ | None
+ ) = ...,
num_inflight_tasks: builtins.int = ...,
) -> None: ...
def ClearField(
@@ -1782,56 +1802,69 @@ class ExecutePlanResponse(google.protobuf.message.Message):
@property
def sql_command_result(self) -> global___ExecutePlanResponse.SqlCommandResult:
"""Special case for executing SQL commands."""
+
@property
def write_stream_operation_start_result(
self,
) -> pyspark.sql.connect.proto.commands_pb2.WriteStreamOperationStartResult:
"""Response for a streaming query."""
+
@property
def streaming_query_command_result(
self,
) -> pyspark.sql.connect.proto.commands_pb2.StreamingQueryCommandResult:
"""Response for commands on a streaming query."""
+
@property
def get_resources_command_result(
self,
) -> pyspark.sql.connect.proto.commands_pb2.GetResourcesCommandResult:
"""Response for 'SparkContext.resources'."""
+
@property
def streaming_query_manager_command_result(
self,
) -> pyspark.sql.connect.proto.commands_pb2.StreamingQueryManagerCommandResult:
"""Response for commands on the streaming query manager."""
+
@property
def streaming_query_listener_events_result(
self,
) -> pyspark.sql.connect.proto.commands_pb2.StreamingQueryListenerEventsResult:
"""Response for commands on the client side streaming query listener."""
+
@property
def result_complete(self) -> global___ExecutePlanResponse.ResultComplete:
"""Response type informing if the stream is complete in reattachable execution."""
+
@property
def create_resource_profile_command_result(
self,
) -> pyspark.sql.connect.proto.commands_pb2.CreateResourceProfileCommandResult:
"""Response for command that creates ResourceProfile."""
+
@property
def execution_progress(self) -> global___ExecutePlanResponse.ExecutionProgress:
"""(Optional) Intermediate query progress reports."""
+
@property
def checkpoint_command_result(self) -> global___CheckpointCommandResult:
"""Response for command that checkpoints a DataFrame."""
+
@property
def ml_command_result(self) -> pyspark.sql.connect.proto.ml_pb2.MlCommandResult:
"""ML command response"""
+
@property
def pipeline_event_result(self) -> pyspark.sql.connect.proto.pipelines_pb2.PipelineEventResult:
"""Response containing pipeline event that is streamed back to the client during a pipeline run"""
+
@property
def pipeline_command_result(
self,
) -> pyspark.sql.connect.proto.pipelines_pb2.PipelineCommandResult:
"""Pipeline command response"""
+
@property
def pipeline_query_function_execution_signal(
self,
@@ -1839,14 +1872,17 @@ class ExecutePlanResponse(google.protobuf.message.Message):
"""A signal from the server to the client to execute the query function for a flow, and to
register its result with the server.
"""
+
@property
def extension(self) -> google.protobuf.any_pb2.Any:
"""Support arbitrary result objects."""
+
@property
def metrics(self) -> global___ExecutePlanResponse.Metrics:
"""Metrics for the query execution. Typically, this field is only present in the last
batch of results and then represent the overall state of the query execution.
"""
+
@property
def observed_metrics(
self,
@@ -1854,9 +1890,11 @@ class ExecutePlanResponse(google.protobuf.message.Message):
global___ExecutePlanResponse.ObservedMetrics
]:
"""The metrics observed during the execution of the query plan."""
+
@property
def schema(self) -> pyspark.sql.connect.proto.types_pb2.DataType:
"""(Optional) The Spark schema. This field is available when `collect` is called."""
+
def __init__(
self,
*,
@@ -1866,32 +1904,42 @@ class ExecutePlanResponse(google.protobuf.message.Message):
response_id: builtins.str = ...,
arrow_batch: global___ExecutePlanResponse.ArrowBatch | None = ...,
sql_command_result: global___ExecutePlanResponse.SqlCommandResult | None = ...,
- write_stream_operation_start_result: pyspark.sql.connect.proto.commands_pb2.WriteStreamOperationStartResult
- | None = ...,
- streaming_query_command_result: pyspark.sql.connect.proto.commands_pb2.StreamingQueryCommandResult
- | None = ...,
- get_resources_command_result: pyspark.sql.connect.proto.commands_pb2.GetResourcesCommandResult
- | None = ...,
- streaming_query_manager_command_result: pyspark.sql.connect.proto.commands_pb2.StreamingQueryManagerCommandResult
- | None = ...,
- streaming_query_listener_events_result: pyspark.sql.connect.proto.commands_pb2.StreamingQueryListenerEventsResult
- | None = ...,
+ write_stream_operation_start_result: (
+ pyspark.sql.connect.proto.commands_pb2.WriteStreamOperationStartResult | None
+ ) = ...,
+ streaming_query_command_result: (
+ pyspark.sql.connect.proto.commands_pb2.StreamingQueryCommandResult | None
+ ) = ...,
+ get_resources_command_result: (
+ pyspark.sql.connect.proto.commands_pb2.GetResourcesCommandResult | None
+ ) = ...,
+ streaming_query_manager_command_result: (
+ pyspark.sql.connect.proto.commands_pb2.StreamingQueryManagerCommandResult | None
+ ) = ...,
+ streaming_query_listener_events_result: (
+ pyspark.sql.connect.proto.commands_pb2.StreamingQueryListenerEventsResult | None
+ ) = ...,
result_complete: global___ExecutePlanResponse.ResultComplete | None = ...,
- create_resource_profile_command_result: pyspark.sql.connect.proto.commands_pb2.CreateResourceProfileCommandResult
- | None = ...,
+ create_resource_profile_command_result: (
+ pyspark.sql.connect.proto.commands_pb2.CreateResourceProfileCommandResult | None
+ ) = ...,
execution_progress: global___ExecutePlanResponse.ExecutionProgress | None = ...,
checkpoint_command_result: global___CheckpointCommandResult | None = ...,
ml_command_result: pyspark.sql.connect.proto.ml_pb2.MlCommandResult | None = ...,
- pipeline_event_result: pyspark.sql.connect.proto.pipelines_pb2.PipelineEventResult
- | None = ...,
- pipeline_command_result: pyspark.sql.connect.proto.pipelines_pb2.PipelineCommandResult
- | None = ...,
- pipeline_query_function_execution_signal: pyspark.sql.connect.proto.pipelines_pb2.PipelineQueryFunctionExecutionSignal
- | None = ...,
+ pipeline_event_result: (
+ pyspark.sql.connect.proto.pipelines_pb2.PipelineEventResult | None
+ ) = ...,
+ pipeline_command_result: (
+ pyspark.sql.connect.proto.pipelines_pb2.PipelineCommandResult | None
+ ) = ...,
+ pipeline_query_function_execution_signal: (
+ pyspark.sql.connect.proto.pipelines_pb2.PipelineQueryFunctionExecutionSignal | None
+ ) = ...,
extension: google.protobuf.any_pb2.Any | None = ...,
metrics: global___ExecutePlanResponse.Metrics | None = ...,
- observed_metrics: collections.abc.Iterable[global___ExecutePlanResponse.ObservedMetrics]
- | None = ...,
+ observed_metrics: (
+ collections.abc.Iterable[global___ExecutePlanResponse.ObservedMetrics] | None
+ ) = ...,
schema: pyspark.sql.connect.proto.types_pb2.DataType | None = ...,
) -> None: ...
def HasField(
@@ -2179,6 +2227,7 @@ class ConfigRequest(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Required) The config keys to get."""
+
def __init__(
self,
*,
@@ -2195,6 +2244,7 @@ class ConfigRequest(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___KeyValue]:
"""(Required) The config key-value pairs to get. The value will be used as the default value."""
+
def __init__(
self,
*,
@@ -2211,6 +2261,7 @@ class ConfigRequest(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Required) The config keys to get optionally."""
+
def __init__(
self,
*,
@@ -2248,6 +2299,7 @@ class ConfigRequest(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Required) The config keys to unset."""
+
def __init__(
self,
*,
@@ -2264,6 +2316,7 @@ class ConfigRequest(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Required) The config keys to check the config is modifiable."""
+
def __init__(
self,
*,
@@ -2293,6 +2346,7 @@ class ConfigRequest(google.protobuf.message.Message):
@property
def user_context(self) -> global___UserContext:
"""(Required) User context"""
+
@property
def operation(self) -> global___ConfigRequest.Operation:
"""(Required) The operation for the config."""
@@ -2385,6 +2439,7 @@ class ConfigResponse(google.protobuf.message.Message):
Available when the operation is 'Get', 'GetWithDefault', 'GetOption', 'GetAll'.
Also available for the operation 'IsModifiable' with boolean string "true" and "false".
"""
+
@property
def warnings(
self,
@@ -2393,6 +2448,7 @@ class ConfigResponse(google.protobuf.message.Message):
Warning messages for deprecated or unsupported configurations.
"""
+
def __init__(
self,
*,
@@ -2463,6 +2519,7 @@ class AddArtifactsRequest(google.protobuf.message.Message):
@property
def data(self) -> global___AddArtifactsRequest.ArtifactChunk:
"""A single data chunk."""
+
def __init__(
self,
*,
@@ -2491,8 +2548,9 @@ class AddArtifactsRequest(google.protobuf.message.Message):
def __init__(
self,
*,
- artifacts: collections.abc.Iterable[global___AddArtifactsRequest.SingleChunkArtifact]
- | None = ...,
+ artifacts: (
+ collections.abc.Iterable[global___AddArtifactsRequest.SingleChunkArtifact] | None
+ ) = ...,
) -> None: ...
def ClearField(
self, field_name: typing_extensions.Literal["artifacts", b"artifacts"]
@@ -2523,6 +2581,7 @@ class AddArtifactsRequest(google.protobuf.message.Message):
@property
def initial_chunk(self) -> global___AddArtifactsRequest.ArtifactChunk:
"""The first/initial chunk."""
+
def __init__(
self,
*,
@@ -2585,11 +2644,13 @@ class AddArtifactsRequest(google.protobuf.message.Message):
The server side is notified about the total size of the large artifact as well as the
number of chunks to expect.
"""
+
@property
def chunk(self) -> global___AddArtifactsRequest.ArtifactChunk:
"""A chunk of an artifact excluding metadata. This can be any chunk of a large artifact
excluding the first chunk (which is included in `BeginChunkedArtifact`).
"""
+
def __init__(
self,
*,
@@ -2717,13 +2778,15 @@ class AddArtifactsResponse(google.protobuf.message.Message):
global___AddArtifactsResponse.ArtifactSummary
]:
"""The list of artifact(s) seen by the server."""
+
def __init__(
self,
*,
session_id: builtins.str = ...,
server_side_session_id: builtins.str = ...,
- artifacts: collections.abc.Iterable[global___AddArtifactsResponse.ArtifactSummary]
- | None = ...,
+ artifacts: (
+ collections.abc.Iterable[global___AddArtifactsResponse.ArtifactSummary] | None
+ ) = ...,
) -> None: ...
def ClearField(
self,
@@ -2782,6 +2845,7 @@ class ArtifactStatusesRequest(google.protobuf.message.Message):
The relative path of the file on the server's filesystem will be the same as the name of
the provided artifact)
"""
+
def __init__(
self,
*,
@@ -2898,15 +2962,16 @@ class ArtifactStatusesResponse(google.protobuf.message.Message):
builtins.str, global___ArtifactStatusesResponse.ArtifactStatus
]:
"""A map of artifact names to their statuses."""
+
def __init__(
self,
*,
session_id: builtins.str = ...,
server_side_session_id: builtins.str = ...,
- statuses: collections.abc.Mapping[
- builtins.str, global___ArtifactStatusesResponse.ArtifactStatus
- ]
- | None = ...,
+ statuses: (
+ collections.abc.Mapping[builtins.str, global___ArtifactStatusesResponse.ArtifactStatus]
+ | None
+ ) = ...,
) -> None: ...
def ClearField(
self,
@@ -3082,6 +3147,7 @@ class InterruptResponse(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""Operation ids of the executions which were interrupted."""
+
def __init__(
self,
*,
@@ -3875,6 +3941,7 @@ class FetchErrorDetailsResponse(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""The message parameters for the error framework."""
+
@property
def query_contexts(
self,
@@ -3889,18 +3956,19 @@ class FetchErrorDetailsResponse(google.protobuf.message.Message):
@property
def breaking_change_info(self) -> global___FetchErrorDetailsResponse.BreakingChangeInfo:
"""Additional information if the error was caused by a breaking change."""
+
def __init__(
self,
*,
error_class: builtins.str | None = ...,
message_parameters: collections.abc.Mapping[builtins.str, builtins.str] | None = ...,
- query_contexts: collections.abc.Iterable[
- global___FetchErrorDetailsResponse.QueryContext
- ]
- | None = ...,
+ query_contexts: (
+ collections.abc.Iterable[global___FetchErrorDetailsResponse.QueryContext] | None
+ ) = ...,
sql_state: builtins.str | None = ...,
- breaking_change_info: global___FetchErrorDetailsResponse.BreakingChangeInfo
- | None = ...,
+ breaking_change_info: (
+ global___FetchErrorDetailsResponse.BreakingChangeInfo | None
+ ) = ...,
) -> None: ...
def HasField(
self,
@@ -3971,6 +4039,7 @@ class FetchErrorDetailsResponse(google.protobuf.message.Message):
"""A message explaining how the user can migrate their job to work
with the breaking change.
"""
+
@property
def mitigation_config(self) -> global___FetchErrorDetailsResponse.MitigationConfig:
"""A spark config flag that can be used to mitigate the breaking change."""
@@ -4075,15 +4144,16 @@ class FetchErrorDetailsResponse(google.protobuf.message.Message):
@property
def spark_throwable(self) -> global___FetchErrorDetailsResponse.SparkThrowable:
"""The structured data of a SparkThrowable exception."""
+
def __init__(
self,
*,
error_type_hierarchy: collections.abc.Iterable[builtins.str] | None = ...,
message: builtins.str = ...,
- stack_trace: collections.abc.Iterable[
- global___FetchErrorDetailsResponse.StackTraceElement
- ]
- | None = ...,
+ stack_trace: (
+ collections.abc.Iterable[global___FetchErrorDetailsResponse.StackTraceElement]
+ | None
+ ) = ...,
cause_idx: builtins.int | None = ...,
spark_throwable: global___FetchErrorDetailsResponse.SparkThrowable | None = ...,
) -> None: ...
@@ -4146,6 +4216,7 @@ class FetchErrorDetailsResponse(google.protobuf.message.Message):
global___FetchErrorDetailsResponse.Error
]:
"""A list of errors."""
+
def __init__(
self,
*,
@@ -4188,6 +4259,7 @@ class CheckpointCommandResult(google.protobuf.message.Message):
@property
def relation(self) -> pyspark.sql.connect.proto.relations_pb2.CachedRemoteRelation:
"""(Required) The logical plan checkpointed."""
+
def __init__(
self,
*,
@@ -4368,6 +4440,7 @@ class GetStatusRequest(google.protobuf.message.Message):
"""Get status of operations with these operation_ids.
If unset or empty, returns status of all operations in the session.
"""
+
@property
def extensions(
self,
@@ -4375,6 +4448,7 @@ class GetStatusRequest(google.protobuf.message.Message):
google.protobuf.any_pb2.Any
]:
"""Extension point for custom operation-level status requests."""
+
def __init__(
self,
*,
@@ -4426,6 +4500,7 @@ class GetStatusRequest(google.protobuf.message.Message):
Get status of operations in the session.
"""
+
@property
def extensions(
self,
@@ -4433,6 +4508,7 @@ class GetStatusRequest(google.protobuf.message.Message):
google.protobuf.any_pb2.Any
]:
"""Extension point for custom status request types."""
+
def __init__(
self,
*,
@@ -4524,13 +4600,25 @@ class GetStatusResponse(google.protobuf.message.Message):
builtins.type,
): # noqa: F821
DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
- OPERATION_STATE_UNSPECIFIED: GetStatusResponse.OperationStatus._OperationState.ValueType # 0
- OPERATION_STATE_UNKNOWN: GetStatusResponse.OperationStatus._OperationState.ValueType # 1
- OPERATION_STATE_RUNNING: GetStatusResponse.OperationStatus._OperationState.ValueType # 2
- OPERATION_STATE_TERMINATING: GetStatusResponse.OperationStatus._OperationState.ValueType # 3
- OPERATION_STATE_SUCCEEDED: GetStatusResponse.OperationStatus._OperationState.ValueType # 4
+ OPERATION_STATE_UNSPECIFIED: (
+ GetStatusResponse.OperationStatus._OperationState.ValueType
+ ) # 0
+ OPERATION_STATE_UNKNOWN: (
+ GetStatusResponse.OperationStatus._OperationState.ValueType
+ ) # 1
+ OPERATION_STATE_RUNNING: (
+ GetStatusResponse.OperationStatus._OperationState.ValueType
+ ) # 2
+ OPERATION_STATE_TERMINATING: (
+ GetStatusResponse.OperationStatus._OperationState.ValueType
+ ) # 3
+ OPERATION_STATE_SUCCEEDED: (
+ GetStatusResponse.OperationStatus._OperationState.ValueType
+ ) # 4
OPERATION_STATE_FAILED: GetStatusResponse.OperationStatus._OperationState.ValueType # 5
- OPERATION_STATE_CANCELLED: GetStatusResponse.OperationStatus._OperationState.ValueType # 6
+ OPERATION_STATE_CANCELLED: (
+ GetStatusResponse.OperationStatus._OperationState.ValueType
+ ) # 6
class OperationState(_OperationState, metaclass=_OperationStateEnumTypeWrapper): ...
OPERATION_STATE_UNSPECIFIED: GetStatusResponse.OperationStatus.OperationState.ValueType # 0
@@ -4555,6 +4643,7 @@ class GetStatusResponse(google.protobuf.message.Message):
google.protobuf.any_pb2.Any
]:
"""Extension point for custom operation-level status fields."""
+
def __init__(
self,
*,
@@ -4586,6 +4675,7 @@ class GetStatusResponse(google.protobuf.message.Message):
global___GetStatusResponse.OperationStatus
]:
"""Status information about requested operations."""
+
@property
def extensions(
self,
@@ -4593,13 +4683,15 @@ class GetStatusResponse(google.protobuf.message.Message):
google.protobuf.any_pb2.Any
]:
"""Extension point for custom status response types."""
+
def __init__(
self,
*,
session_id: builtins.str = ...,
server_side_session_id: builtins.str = ...,
- operation_statuses: collections.abc.Iterable[global___GetStatusResponse.OperationStatus]
- | None = ...,
+ operation_statuses: (
+ collections.abc.Iterable[global___GetStatusResponse.OperationStatus] | None
+ ) = ...,
extensions: collections.abc.Iterable[google.protobuf.any_pb2.Any] | None = ...,
) -> None: ...
def ClearField(
diff --git a/python/pyspark/sql/connect/proto/base_pb2_grpc.py b/python/pyspark/sql/connect/proto/base_pb2_grpc.py
index 14fd88bd0f11..c8321d41e66b 100644
--- a/python/pyspark/sql/connect/proto/base_pb2_grpc.py
+++ b/python/pyspark/sql/connect/proto/base_pb2_grpc.py
@@ -16,6 +16,7 @@
#
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
"""Client and server classes corresponding to protobuf-defined services."""
+
import grpc
from pyspark.sql.connect.proto import base_pb2 as spark_dot_connect_dot_base__pb2
diff --git a/python/pyspark/sql/connect/proto/catalog_pb2.py b/python/pyspark/sql/connect/proto/catalog_pb2.py
index f99c6d18cda1..122e55770338 100644
--- a/python/pyspark/sql/connect/proto/catalog_pb2.py
+++ b/python/pyspark/sql/connect/proto/catalog_pb2.py
@@ -20,6 +20,7 @@
# source: spark/connect/catalog.proto
# Protobuf Python Version: 6.33.5
"""Generated protocol buffer code."""
+
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
@@ -37,7 +38,6 @@ _sym_db = _symbol_database.Default()
from pyspark.sql.connect.proto import common_pb2 as spark_dot_connect_dot_common__pb2
from pyspark.sql.connect.proto import types_pb2 as spark_dot_connect_dot_types__pb2
-
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x1bspark/connect/catalog.proto\x12\rspark.connect\x1a\x1aspark/connect/common.proto\x1a\x19spark/connect/types.proto"\xc6\x0e\n\x07\x43\x61talog\x12K\n\x10\x63urrent_database\x18\x01 \x01(\x0b\x32\x1e.spark.connect.CurrentDatabaseH\x00R\x0f\x63urrentDatabase\x12U\n\x14set_current_database\x18\x02 \x01(\x0b\x32!.spark.connect.SetCurrentDatabaseH\x00R\x12setCurrentDatabase\x12\x45\n\x0elist_databases\x18\x03 \x01(\x0b\x32\x1c.spark.connect.ListDatabasesH\x00R\rlistDatabases\x12<\n [...]
)
@@ -49,9 +49,9 @@ _builder.BuildTopDescriptorsAndMessages(
)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals[
- "DESCRIPTOR"
- ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ _globals["DESCRIPTOR"]._serialized_options = (
+ b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ )
_globals["_CREATEEXTERNALTABLE_OPTIONSENTRY"]._loaded_options = None
_globals["_CREATEEXTERNALTABLE_OPTIONSENTRY"]._serialized_options = b"8\001"
_globals["_CREATETABLE_OPTIONSENTRY"]._loaded_options = None
diff --git a/python/pyspark/sql/connect/proto/catalog_pb2.pyi b/python/pyspark/sql/connect/proto/catalog_pb2.pyi
index 2879b44618b5..72f4ddaf8888 100644
--- a/python/pyspark/sql/connect/proto/catalog_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/catalog_pb2.pyi
@@ -33,6 +33,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
+
import builtins
import collections.abc
import google.protobuf.descriptor
@@ -281,9 +282,7 @@ class Catalog(google.protobuf.message.Message):
b"uncache_table",
],
) -> None: ...
- def WhichOneof(
- self, oneof_group: typing_extensions.Literal["cat_type", b"cat_type"]
- ) -> (
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["cat_type", b"cat_type"]) -> (
typing_extensions.Literal[
"current_database",
"set_current_database",
@@ -708,11 +707,13 @@ class CreateExternalTable(google.protobuf.message.Message):
@property
def schema(self) -> pyspark.sql.connect.proto.types_pb2.DataType:
"""(Optional)"""
+
@property
def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""Options could be empty for valid data source format.
The map key is case insensitive.
"""
+
def __init__(
self,
*,
@@ -814,11 +815,13 @@ class CreateTable(google.protobuf.message.Message):
@property
def schema(self) -> pyspark.sql.connect.proto.types_pb2.DataType:
"""(Optional)"""
+
@property
def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""Options could be empty for valid data source format.
The map key is case insensitive.
"""
+
def __init__(
self,
*,
@@ -982,6 +985,7 @@ class CacheTable(google.protobuf.message.Message):
@property
def storage_level(self) -> pyspark.sql.connect.proto.common_pb2.StorageLevel:
"""(Optional)"""
+
def __init__(
self,
*,
diff --git a/python/pyspark/sql/connect/proto/commands_pb2.py b/python/pyspark/sql/connect/proto/commands_pb2.py
index 53a15ae9c30a..60cbc3f1eb83 100644
--- a/python/pyspark/sql/connect/proto/commands_pb2.py
+++ b/python/pyspark/sql/connect/proto/commands_pb2.py
@@ -20,6 +20,7 @@
# source: spark/connect/commands.proto
# Protobuf Python Version: 6.33.5
"""Generated protocol buffer code."""
+
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
@@ -41,7 +42,6 @@ from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_rel
from pyspark.sql.connect.proto import ml_pb2 as spark_dot_connect_dot_ml__pb2
from pyspark.sql.connect.proto import pipelines_pb2 as spark_dot_connect_dot_pipelines__pb2
-
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x16spark/connect/ml.proto\x1a\x1dspark/connect/pipelines.proto"\xfb\x0e\n\x07\x43ommand\x12]\n\x11register_function\x18\x01 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x10registerFunction\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOpe [...]
)
@@ -53,9 +53,9 @@ _builder.BuildTopDescriptorsAndMessages(
)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals[
- "DESCRIPTOR"
- ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ _globals["DESCRIPTOR"]._serialized_options = (
+ b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ )
_globals["_SQLCOMMAND_ARGSENTRY"]._loaded_options = None
_globals["_SQLCOMMAND_ARGSENTRY"]._serialized_options = b"8\001"
_globals["_SQLCOMMAND_NAMEDARGUMENTSENTRY"]._loaded_options = None
@@ -154,9 +154,9 @@ if not _descriptor._USE_C_DESCRIPTORS:
_globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT"]._serialized_end = 9542
_globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE"]._serialized_start = 9544
_globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE"]._serialized_end = 9659
- _globals[
- "_STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT"
- ]._serialized_start = 9661
+ _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT"]._serialized_start = (
+ 9661
+ )
_globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT"]._serialized_end = 9720
_globals[
"_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYLISTENERINSTANCE"
diff --git a/python/pyspark/sql/connect/proto/commands_pb2.pyi b/python/pyspark/sql/connect/proto/commands_pb2.pyi
index 48f746a34acc..2505dad6e6dd 100644
--- a/python/pyspark/sql/connect/proto/commands_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/commands_pb2.pyi
@@ -33,6 +33,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
+
import builtins
import collections.abc
import google.protobuf.any_pb2
@@ -161,11 +162,13 @@ class Command(google.protobuf.message.Message):
"""This field is used to mark extensions to the protocol. When plugins generate arbitrary
Commands they can add them here. During the planning the correct resolution is done.
"""
+
def __init__(
self,
*,
- register_function: pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction
- | None = ...,
+ register_function: (
+ pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction | None
+ ) = ...,
write_operation: global___WriteOperation | None = ...,
create_dataframe_view: global___CreateDataFrameViewCommand | None = ...,
write_operation_v2: global___WriteOperationV2 | None = ...,
@@ -174,16 +177,20 @@ class Command(google.protobuf.message.Message):
streaming_query_command: global___StreamingQueryCommand | None = ...,
get_resources_command: global___GetResourcesCommand | None = ...,
streaming_query_manager_command: global___StreamingQueryManagerCommand | None = ...,
- register_table_function: pyspark.sql.connect.proto.relations_pb2.CommonInlineUserDefinedTableFunction
- | None = ...,
- streaming_query_listener_bus_command: global___StreamingQueryListenerBusCommand
- | None = ...,
- register_data_source: pyspark.sql.connect.proto.relations_pb2.CommonInlineUserDefinedDataSource
- | None = ...,
+ register_table_function: (
+ pyspark.sql.connect.proto.relations_pb2.CommonInlineUserDefinedTableFunction | None
+ ) = ...,
+ streaming_query_listener_bus_command: (
+ global___StreamingQueryListenerBusCommand | None
+ ) = ...,
+ register_data_source: (
+ pyspark.sql.connect.proto.relations_pb2.CommonInlineUserDefinedDataSource | None
+ ) = ...,
create_resource_profile_command: global___CreateResourceProfileCommand | None = ...,
checkpoint_command: global___CheckpointCommand | None = ...,
- remove_cached_remote_relation_command: global___RemoveCachedRemoteRelationCommand
- | None = ...,
+ remove_cached_remote_relation_command: (
+ global___RemoveCachedRemoteRelationCommand | None
+ ) = ...,
merge_into_table_command: global___MergeIntoTableCommand | None = ...,
ml_command: pyspark.sql.connect.proto.ml_pb2.MlCommand | None = ...,
execute_external_command: global___ExecuteExternalCommand | None = ...,
@@ -382,6 +389,7 @@ class SqlCommand(google.protobuf.message.Message):
builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
]:
"""(Optional) A map of parameter names to literal expressions."""
+
@property
def pos_args(
self,
@@ -389,6 +397,7 @@ class SqlCommand(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
]:
"""(Optional) A sequence of literal expressions for positional parameters in the SQL query text."""
+
@property
def named_arguments(
self,
@@ -398,6 +407,7 @@ class SqlCommand(google.protobuf.message.Message):
"""(Optional) A map of parameter names to expressions.
It cannot coexist with `pos_arguments`.
"""
+
@property
def pos_arguments(
self,
@@ -407,29 +417,34 @@ class SqlCommand(google.protobuf.message.Message):
"""(Optional) A sequence of expressions for positional parameters in the SQL query text.
It cannot coexist with `named_arguments`.
"""
+
@property
def input(self) -> pyspark.sql.connect.proto.relations_pb2.Relation:
"""(Optional) The relation that this SQL command will be built on."""
+
def __init__(
self,
*,
sql: builtins.str = ...,
- args: collections.abc.Mapping[
- builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
- ]
- | None = ...,
- pos_args: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
- ]
- | None = ...,
- named_arguments: collections.abc.Mapping[
- builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
- pos_arguments: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
+ args: (
+ collections.abc.Mapping[
+ builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+ ]
+ | None
+ ) = ...,
+ pos_args: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression.Literal]
+ | None
+ ) = ...,
+ named_arguments: (
+ collections.abc.Mapping[
+ builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None
+ ) = ...,
+ pos_arguments: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
input: pyspark.sql.connect.proto.relations_pb2.Relation | None = ...,
) -> None: ...
def HasField(
@@ -558,7 +573,9 @@ class WriteOperation(google.protobuf.message.Message):
): # noqa: F821
DESCRIPTOR: google.protobuf.descriptor.EnumDescriptor
TABLE_SAVE_METHOD_UNSPECIFIED: WriteOperation.SaveTable._TableSaveMethod.ValueType # 0
- TABLE_SAVE_METHOD_SAVE_AS_TABLE: WriteOperation.SaveTable._TableSaveMethod.ValueType # 1
+ TABLE_SAVE_METHOD_SAVE_AS_TABLE: (
+ WriteOperation.SaveTable._TableSaveMethod.ValueType
+ ) # 1
TABLE_SAVE_METHOD_INSERT_INTO: WriteOperation.SaveTable._TableSaveMethod.ValueType # 2
class TableSaveMethod(_TableSaveMethod, metaclass=_TableSaveMethodEnumTypeWrapper): ...
@@ -633,24 +650,29 @@ class WriteOperation(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) List of columns to sort the output by."""
+
@property
def partitioning_columns(
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) List of columns for partitioning."""
+
@property
def bucket_by(self) -> global___WriteOperation.BucketBy:
"""(Optional) Bucketing specification. Bucketing must set the number of buckets and the columns
to bucket by.
"""
+
@property
def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""(Optional) A list of configuration options."""
+
@property
def clustering_columns(
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) Columns used for clustering the table."""
+
def __init__(
self,
*,
@@ -818,9 +840,11 @@ class WriteOperationV2(google.protobuf.message.Message):
"""(Optional) List of columns for partitioning for output table created by `create`,
`createOrReplace`, or `replace`
"""
+
@property
def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""(Optional) A list of configuration options."""
+
@property
def table_properties(
self,
@@ -831,21 +855,22 @@ class WriteOperationV2(google.protobuf.message.Message):
@property
def overwrite_condition(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression:
"""(Optional) A condition for overwrite saving mode"""
+
@property
def clustering_columns(
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) Columns used for clustering the table."""
+
def __init__(
self,
*,
input: pyspark.sql.connect.proto.relations_pb2.Relation | None = ...,
table_name: builtins.str = ...,
provider: builtins.str | None = ...,
- partitioning_columns: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
+ partitioning_columns: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
options: collections.abc.Mapping[builtins.str, builtins.str] | None = ...,
table_properties: collections.abc.Mapping[builtins.str, builtins.str] | None = ...,
mode: global___WriteOperationV2.Mode.ValueType = ...,
@@ -969,6 +994,7 @@ class WriteStreamOperationStart(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) Columns used for clustering the table."""
+
def __init__(
self,
*,
@@ -1064,9 +1090,7 @@ class WriteStreamOperationStart(google.protobuf.message.Message):
self, oneof_group: typing_extensions.Literal["sink_destination", b"sink_destination"]
) -> typing_extensions.Literal["path", "table_name"] | None: ...
@typing.overload
- def WhichOneof(
- self, oneof_group: typing_extensions.Literal["trigger", b"trigger"]
- ) -> (
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["trigger", b"trigger"]) -> (
typing_extensions.Literal[
"processing_time_interval",
"available_now",
@@ -1285,6 +1309,7 @@ class StreamingQueryCommand(google.protobuf.message.Message):
@property
def await_termination(self) -> global___StreamingQueryCommand.AwaitTerminationCommand:
"""awaitTermination() API. Waits for the termination of the query."""
+
def __init__(
self,
*,
@@ -1348,9 +1373,7 @@ class StreamingQueryCommand(google.protobuf.message.Message):
b"stop",
],
) -> None: ...
- def WhichOneof(
- self, oneof_group: typing_extensions.Literal["command", b"command"]
- ) -> (
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["command", b"command"]) -> (
typing_extensions.Literal[
"status",
"last_progress",
@@ -1414,6 +1437,7 @@ class StreamingQueryCommandResult(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""Progress reports as an array of json strings."""
+
def __init__(
self,
*,
@@ -1531,6 +1555,7 @@ class StreamingQueryCommandResult(google.protobuf.message.Message):
@property
def query_id(self) -> global___StreamingQueryInstanceId:
"""(Required) Query instance id. See `StreamingQueryInstanceId`."""
+
@property
def status(self) -> global___StreamingQueryCommandResult.StatusResult: ...
@property
@@ -1648,8 +1673,9 @@ class StreamingQueryManagerCommand(google.protobuf.message.Message):
self,
*,
listener_payload: builtins.bytes = ...,
- python_listener_payload: pyspark.sql.connect.proto.expressions_pb2.PythonUDF
- | None = ...,
+ python_listener_payload: (
+ pyspark.sql.connect.proto.expressions_pb2.PythonUDF | None
+ ) = ...,
id: builtins.str = ...,
) -> None: ...
def HasField(
@@ -1702,6 +1728,7 @@ class StreamingQueryManagerCommand(google.protobuf.message.Message):
@property
def add_listener(self) -> global___StreamingQueryManagerCommand.StreamingQueryListenerCommand:
"""addListener API."""
+
@property
def remove_listener(
self,
@@ -1714,13 +1741,16 @@ class StreamingQueryManagerCommand(google.protobuf.message.Message):
*,
active: builtins.bool = ...,
get_query: builtins.str = ...,
- await_any_termination: global___StreamingQueryManagerCommand.AwaitAnyTerminationCommand
- | None = ...,
+ await_any_termination: (
+ global___StreamingQueryManagerCommand.AwaitAnyTerminationCommand | None
+ ) = ...,
reset_terminated: builtins.bool = ...,
- add_listener: global___StreamingQueryManagerCommand.StreamingQueryListenerCommand
- | None = ...,
- remove_listener: global___StreamingQueryManagerCommand.StreamingQueryListenerCommand
- | None = ...,
+ add_listener: (
+ global___StreamingQueryManagerCommand.StreamingQueryListenerCommand | None
+ ) = ...,
+ remove_listener: (
+ global___StreamingQueryManagerCommand.StreamingQueryListenerCommand | None
+ ) = ...,
list_listeners: builtins.bool = ...,
) -> None: ...
def HasField(
@@ -1765,9 +1795,7 @@ class StreamingQueryManagerCommand(google.protobuf.message.Message):
b"reset_terminated",
],
) -> None: ...
- def WhichOneof(
- self, oneof_group: typing_extensions.Literal["command", b"command"]
- ) -> (
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["command", b"command"]) -> (
typing_extensions.Literal[
"active",
"get_query",
@@ -1800,10 +1828,12 @@ class StreamingQueryManagerCommandResult(google.protobuf.message.Message):
def __init__(
self,
*,
- active_queries: collections.abc.Iterable[
- global___StreamingQueryManagerCommandResult.StreamingQueryInstance
- ]
- | None = ...,
+ active_queries: (
+ collections.abc.Iterable[
+ global___StreamingQueryManagerCommandResult.StreamingQueryInstance
+ ]
+ | None
+ ) = ...,
) -> None: ...
def ClearField(
self, field_name: typing_extensions.Literal["active_queries", b"active_queries"]
@@ -1874,6 +1904,7 @@ class StreamingQueryManagerCommandResult(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Required) Reference IDs of listener instances."""
+
def __init__(
self,
*,
@@ -1910,13 +1941,15 @@ class StreamingQueryManagerCommandResult(google.protobuf.message.Message):
*,
active: global___StreamingQueryManagerCommandResult.ActiveResult | None = ...,
query: global___StreamingQueryManagerCommandResult.StreamingQueryInstance | None = ...,
- await_any_termination: global___StreamingQueryManagerCommandResult.AwaitAnyTerminationResult
- | None = ...,
+ await_any_termination: (
+ global___StreamingQueryManagerCommandResult.AwaitAnyTerminationResult | None
+ ) = ...,
reset_terminated: builtins.bool = ...,
add_listener: builtins.bool = ...,
remove_listener: builtins.bool = ...,
- list_listeners: global___StreamingQueryManagerCommandResult.ListStreamingQueryListenerResult
- | None = ...,
+ list_listeners: (
+ global___StreamingQueryManagerCommandResult.ListStreamingQueryListenerResult | None
+ ) = ...,
) -> None: ...
def HasField(
self,
@@ -1960,9 +1993,7 @@ class StreamingQueryManagerCommandResult(google.protobuf.message.Message):
b"result_type",
],
) -> None: ...
- def WhichOneof(
- self, oneof_group: typing_extensions.Literal["result_type", b"result_type"]
- ) -> (
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["result_type", b"result_type"]) -> (
typing_extensions.Literal[
"active",
"query",
@@ -2148,10 +2179,12 @@ class GetResourcesCommandResult(google.protobuf.message.Message):
def __init__(
self,
*,
- resources: collections.abc.Mapping[
- builtins.str, pyspark.sql.connect.proto.common_pb2.ResourceInformation
- ]
- | None = ...,
+ resources: (
+ collections.abc.Mapping[
+ builtins.str, pyspark.sql.connect.proto.common_pb2.ResourceInformation
+ ]
+ | None
+ ) = ...,
) -> None: ...
def ClearField(
self, field_name: typing_extensions.Literal["resources", b"resources"]
@@ -2168,6 +2201,7 @@ class CreateResourceProfileCommand(google.protobuf.message.Message):
@property
def profile(self) -> pyspark.sql.connect.proto.common_pb2.ResourceProfile:
"""(Required) The ResourceProfile to be built on the server-side."""
+
def __init__(
self,
*,
@@ -2208,6 +2242,7 @@ class RemoveCachedRemoteRelationCommand(google.protobuf.message.Message):
@property
def relation(self) -> pyspark.sql.connect.proto.relations_pb2.CachedRemoteRelation:
"""(Required) The remote to be related"""
+
def __init__(
self,
*,
@@ -2241,6 +2276,7 @@ class CheckpointCommand(google.protobuf.message.Message):
@property
def storage_level(self) -> pyspark.sql.connect.proto.common_pb2.StorageLevel:
"""(Optional) For local checkpoint, the storage level to use."""
+
def __init__(
self,
*,
@@ -2296,9 +2332,11 @@ class MergeIntoTableCommand(google.protobuf.message.Message):
@property
def source_table_plan(self) -> pyspark.sql.connect.proto.relations_pb2.Relation:
"""(Required) The relation of the source table."""
+
@property
def merge_condition(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression:
"""(Required) The condition to match the source and target."""
+
@property
def match_actions(
self,
@@ -2306,6 +2344,7 @@ class MergeIntoTableCommand(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Optional) The actions to be taken when the condition is matched."""
+
@property
def not_matched_actions(
self,
@@ -2313,6 +2352,7 @@ class MergeIntoTableCommand(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Optional) The actions to be taken when the condition is not matched."""
+
@property
def not_matched_by_source_actions(
self,
@@ -2328,18 +2368,15 @@ class MergeIntoTableCommand(google.protobuf.message.Message):
target_table_name: builtins.str = ...,
source_table_plan: pyspark.sql.connect.proto.relations_pb2.Relation | None = ...,
merge_condition: pyspark.sql.connect.proto.expressions_pb2.Expression | None = ...,
- match_actions: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
- not_matched_actions: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
- not_matched_by_source_actions: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
+ match_actions: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
+ not_matched_actions: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
+ not_matched_by_source_actions: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
with_schema_evolution: builtins.bool = ...,
) -> None: ...
def HasField(
@@ -2402,6 +2439,7 @@ class ExecuteExternalCommand(google.protobuf.message.Message):
@property
def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""(Optional) The options for the runner."""
+
def __init__(
self,
*,
diff --git a/python/pyspark/sql/connect/proto/common_pb2.py b/python/pyspark/sql/connect/proto/common_pb2.py
index 9868a198e685..c8108b627f6f 100644
--- a/python/pyspark/sql/connect/proto/common_pb2.py
+++ b/python/pyspark/sql/connect/proto/common_pb2.py
@@ -20,6 +20,7 @@
# source: spark/connect/common.proto
# Protobuf Python Version: 6.33.5
"""Generated protocol buffer code."""
+
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
@@ -45,9 +46,9 @@ _builder.BuildTopDescriptorsAndMessages(
)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals[
- "DESCRIPTOR"
- ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ _globals["DESCRIPTOR"]._serialized_options = (
+ b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ )
_globals["_RESOURCEPROFILE_EXECUTORRESOURCESENTRY"]._loaded_options = None
_globals["_RESOURCEPROFILE_EXECUTORRESOURCESENTRY"]._serialized_options = b"8\001"
_globals["_RESOURCEPROFILE_TASKRESOURCESENTRY"]._loaded_options = None
diff --git a/python/pyspark/sql/connect/proto/common_pb2.pyi b/python/pyspark/sql/connect/proto/common_pb2.pyi
index 95addf6589d1..4854a6e0af77 100644
--- a/python/pyspark/sql/connect/proto/common_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/common_pb2.pyi
@@ -33,6 +33,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
+
import builtins
import collections.abc
import google.protobuf.descriptor
@@ -111,6 +112,7 @@ class ResourceInformation(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Required) An array of strings describing the addresses of the resource."""
+
def __init__(
self,
*,
@@ -273,6 +275,7 @@ class ResourceProfile(google.protobuf.message.Message):
"""(Optional) Resource requests for executors. Mapped from the resource name
(e.g., cores, memory, CPU) to its specific request.
"""
+
@property
def task_resources(
self,
@@ -280,13 +283,16 @@ class ResourceProfile(google.protobuf.message.Message):
"""(Optional) Resource requests for tasks. Mapped from the resource name
(e.g., cores, memory, CPU) to its specific request.
"""
+
def __init__(
self,
*,
- executor_resources: collections.abc.Mapping[builtins.str, global___ExecutorResourceRequest]
- | None = ...,
- task_resources: collections.abc.Mapping[builtins.str, global___TaskResourceRequest]
- | None = ...,
+ executor_resources: (
+ collections.abc.Mapping[builtins.str, global___ExecutorResourceRequest] | None
+ ) = ...,
+ task_resources: (
+ collections.abc.Mapping[builtins.str, global___TaskResourceRequest] | None
+ ) = ...,
) -> None: ...
def ClearField(
self,
@@ -384,6 +390,7 @@ class JvmOrigin(google.protobuf.message.Message):
global___StackTraceElement
]:
"""(Optional) Stack trace."""
+
def __init__(
self,
*,
diff --git a/python/pyspark/sql/connect/proto/example_plugins_pb2.py b/python/pyspark/sql/connect/proto/example_plugins_pb2.py
index 47041d38ae46..a100ddb9bd55 100644
--- a/python/pyspark/sql/connect/proto/example_plugins_pb2.py
+++ b/python/pyspark/sql/connect/proto/example_plugins_pb2.py
@@ -20,6 +20,7 @@
# source: spark/connect/example_plugins.proto
# Protobuf Python Version: 6.33.5
"""Generated protocol buffer code."""
+
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
@@ -37,7 +38,6 @@ _sym_db = _symbol_database.Default()
from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_relations__pb2
from pyspark.sql.connect.proto import expressions_pb2 as spark_dot_connect_dot_expressions__pb2
-
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n#spark/connect/example_plugins.proto\x12\rspark.connect\x1a\x1dspark/connect/relations.proto\x1a\x1fspark/connect/expressions.proto"i\n\x15\x45xamplePluginRelation\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12!\n\x0c\x63ustom_field\x18\x02 \x01(\tR\x0b\x63ustomField"m\n\x17\x45xamplePluginExpression\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild\x12!\n\x0c\x63ustom_field\x18\x02 \x01(\tR\x0b\x63ustomField"9\n\x1 [...]
)
@@ -49,9 +49,9 @@ _builder.BuildTopDescriptorsAndMessages(
)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals[
- "DESCRIPTOR"
- ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ _globals["DESCRIPTOR"]._serialized_options = (
+ b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ )
_globals["_EXAMPLEPLUGINRELATION"]._serialized_start = 118
_globals["_EXAMPLEPLUGINRELATION"]._serialized_end = 223
_globals["_EXAMPLEPLUGINEXPRESSION"]._serialized_start = 225
diff --git a/python/pyspark/sql/connect/proto/example_plugins_pb2.pyi b/python/pyspark/sql/connect/proto/example_plugins_pb2.pyi
index 1be966ff1e40..7cfbe56ae25d 100644
--- a/python/pyspark/sql/connect/proto/example_plugins_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/example_plugins_pb2.pyi
@@ -33,6 +33,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
+
import builtins
import google.protobuf.descriptor
import google.protobuf.message
diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.py b/python/pyspark/sql/connect/proto/expressions_pb2.py
index 2aed56b108de..8e2087da18a0 100644
--- a/python/pyspark/sql/connect/proto/expressions_pb2.py
+++ b/python/pyspark/sql/connect/proto/expressions_pb2.py
@@ -20,6 +20,7 @@
# source: spark/connect/expressions.proto
# Protobuf Python Version: 6.33.5
"""Generated protocol buffer code."""
+
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
@@ -38,7 +39,6 @@ from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2
from pyspark.sql.connect.proto import types_pb2 as spark_dot_connect_dot_types__pb2
from pyspark.sql.connect.proto import common_pb2 as spark_dot_connect_dot_common__pb2
-
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x1fspark/connect/expressions.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x19spark/connect/types.proto\x1a\x1aspark/connect/common.proto"\x92\x38\n\nExpression\x12\x37\n\x06\x63ommon\x18\x12 \x01(\x0b\x32\x1f.spark.connect.ExpressionCommonR\x06\x63ommon\x12=\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralH\x00R\x07literal\x12\x62\n\x14unresolved_attribute\x18\x02 \x01(\x0b\x32-.spark.connect.Expression.UnresolvedAttributeH\x00R\x13unresolved [...]
)
@@ -50,9 +50,9 @@ _builder.BuildTopDescriptorsAndMessages(
)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals[
- "DESCRIPTOR"
- ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ _globals["DESCRIPTOR"]._serialized_options = (
+ b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ )
_globals["_EXPRESSION_LITERAL_ARRAY"].fields_by_name["element_type"]._loaded_options = None
_globals["_EXPRESSION_LITERAL_ARRAY"].fields_by_name[
"element_type"
diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.pyi b/python/pyspark/sql/connect/proto/expressions_pb2.pyi
index e2e23dd8c553..87b71cd5e7ca 100644
--- a/python/pyspark/sql/connect/proto/expressions_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/expressions_pb2.pyi
@@ -33,6 +33,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
+
import builtins
import collections.abc
import google.protobuf.any_pb2
@@ -113,6 +114,7 @@ class Expression(google.protobuf.message.Message):
@property
def value(self) -> global___Expression:
"""This is an expression for future proofing. We are expecting literals on the server side."""
+
def __init__(
self,
*,
@@ -158,9 +160,11 @@ class Expression(google.protobuf.message.Message):
@property
def lower(self) -> global___Expression.Window.WindowFrame.FrameBoundary:
"""(Required) The lower bound of the frame."""
+
@property
def upper(self) -> global___Expression.Window.WindowFrame.FrameBoundary:
"""(Required) The upper bound of the frame."""
+
def __init__(
self,
*,
@@ -185,6 +189,7 @@ class Expression(google.protobuf.message.Message):
@property
def window_function(self) -> global___Expression:
"""(Required) The window function."""
+
@property
def partition_spec(
self,
@@ -192,6 +197,7 @@ class Expression(google.protobuf.message.Message):
global___Expression
]:
"""(Optional) The way that input rows are partitioned."""
+
@property
def order_spec(
self,
@@ -199,12 +205,14 @@ class Expression(google.protobuf.message.Message):
global___Expression.SortOrder
]:
"""(Optional) Ordering of rows in a partition."""
+
@property
def frame_spec(self) -> global___Expression.Window.WindowFrame:
"""(Optional) Window frame in a partition.
If not set, it will be treated as 'UnspecifiedFrame'.
"""
+
def __init__(
self,
*,
@@ -319,6 +327,7 @@ class Expression(google.protobuf.message.Message):
@property
def child(self) -> global___Expression:
"""(Required) The expression that evaluates to the partition ID."""
+
def __init__(
self,
*,
@@ -361,6 +370,7 @@ class Expression(google.protobuf.message.Message):
@property
def expr(self) -> global___Expression:
"""(Required) the expression to be casted."""
+
@property
def type(self) -> pyspark.sql.connect.proto.types_pb2.DataType: ...
type_str: builtins.str
@@ -502,6 +512,7 @@ class Expression(google.protobuf.message.Message):
This field is deprecated since Spark 4.1+. Use data_type field instead.
"""
+
@property
def elements(
self,
@@ -509,6 +520,7 @@ class Expression(google.protobuf.message.Message):
global___Expression.Literal
]:
"""The literal values that make up the array elements."""
+
def __init__(
self,
*,
@@ -538,6 +550,7 @@ class Expression(google.protobuf.message.Message):
This field is deprecated since Spark 4.1+. Use data_type field instead.
"""
+
@property
def value_type(self) -> pyspark.sql.connect.proto.types_pb2.DataType:
"""(Deprecated) The value type of the map.
@@ -545,6 +558,7 @@ class Expression(google.protobuf.message.Message):
This field is deprecated since Spark 4.1+ and should only be set
if the data_type field is not set. Use data_type field instead.
"""
+
@property
def keys(
self,
@@ -552,6 +566,7 @@ class Expression(google.protobuf.message.Message):
global___Expression.Literal
]:
"""The literal keys that make up the map."""
+
@property
def values(
self,
@@ -559,6 +574,7 @@ class Expression(google.protobuf.message.Message):
global___Expression.Literal
]:
"""The literal values that make up the map."""
+
def __init__(
self,
*,
@@ -599,6 +615,7 @@ class Expression(google.protobuf.message.Message):
This field is deprecated since Spark 4.1+ because using DataType as the type of a struct
is ambiguous. Use data_type field instead.
"""
+
@property
def elements(
self,
@@ -606,6 +623,7 @@ class Expression(google.protobuf.message.Message):
global___Expression.Literal
]:
"""The literal values that make up the struct elements."""
+
def __init__(
self,
*,
@@ -792,6 +810,7 @@ class Expression(google.protobuf.message.Message):
If the data_type field is not set at the root level, the data type will be
inferred or retrieved from the deprecated data type fields using best efforts.
"""
+
def __init__(
self,
*,
@@ -1206,11 +1225,13 @@ class Expression(google.protobuf.message.Message):
"""(Required) The expression to extract value from, can be
Map, Array, Struct or array of Structs.
"""
+
@property
def extraction(self) -> global___Expression:
"""(Required) The expression to describe the extraction, can be
key of Map, index of Array, field name of Struct.
"""
+
def __init__(
self,
*,
@@ -1245,6 +1266,7 @@ class Expression(google.protobuf.message.Message):
When not set, it means this field will be dropped.
"""
+
def __init__(
self,
*,
@@ -1279,6 +1301,7 @@ class Expression(google.protobuf.message.Message):
@property
def expr(self) -> global___Expression:
"""(Required) The expression that alias will be added on."""
+
@property
def name(
self,
@@ -1324,6 +1347,7 @@ class Expression(google.protobuf.message.Message):
The function body should use 'UnresolvedAttribute' as arguments, the sever side will
replace 'UnresolvedAttribute' with 'UnresolvedNamedLambdaVariable'.
"""
+
@property
def arguments(
self,
@@ -1331,12 +1355,14 @@ class Expression(google.protobuf.message.Message):
global___Expression.UnresolvedNamedLambdaVariable
]:
"""(Required) Function variables. Must contains 1 ~ 3 variables."""
+
def __init__(
self,
*,
function: global___Expression | None = ...,
- arguments: collections.abc.Iterable[global___Expression.UnresolvedNamedLambdaVariable]
- | None = ...,
+ arguments: (
+ collections.abc.Iterable[global___Expression.UnresolvedNamedLambdaVariable] | None
+ ) = ...,
) -> None: ...
def HasField(
self, field_name: typing_extensions.Literal["function", b"function"]
@@ -1357,6 +1383,7 @@ class Expression(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Required) a list of name parts for the variable. Must not be empty."""
+
def __init__(
self,
*,
@@ -1440,6 +1467,7 @@ class Expression(google.protobuf.message.Message):
"""This field is used to mark extensions to the protocol. When plugins generate arbitrary
relations they can add them here. During the planning the correct resolution is done.
"""
+
def __init__(
self,
*,
@@ -1457,8 +1485,9 @@ class Expression(google.protobuf.message.Message):
window: global___Expression.Window | None = ...,
unresolved_extract_value: global___Expression.UnresolvedExtractValue | None = ...,
update_fields: global___Expression.UpdateFields | None = ...,
- unresolved_named_lambda_variable: global___Expression.UnresolvedNamedLambdaVariable
- | None = ...,
+ unresolved_named_lambda_variable: (
+ global___Expression.UnresolvedNamedLambdaVariable | None
+ ) = ...,
common_inline_user_defined_function: global___CommonInlineUserDefinedFunction | None = ...,
call_function: global___CallFunction | None = ...,
named_argument_expression: global___NamedArgumentExpression | None = ...,
@@ -1574,9 +1603,7 @@ class Expression(google.protobuf.message.Message):
b"window",
],
) -> None: ...
- def WhichOneof(
- self, oneof_group: typing_extensions.Literal["expr_type", b"expr_type"]
- ) -> (
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["expr_type", b"expr_type"]) -> (
typing_extensions.Literal[
"literal",
"unresolved_attribute",
@@ -1613,6 +1640,7 @@ class ExpressionCommon(google.protobuf.message.Message):
@property
def origin(self) -> pyspark.sql.connect.proto.common_pb2.Origin:
"""(Required) Keep the information of the origin for this expression such as stacktrace."""
+
def __init__(
self,
*,
@@ -1644,6 +1672,7 @@ class CommonInlineUserDefinedFunction(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Expression]:
"""(Optional) Function arguments. Empty arguments are allowed."""
+
@property
def python_udf(self) -> global___PythonUDF: ...
@property
@@ -1725,6 +1754,7 @@ class PythonUDF(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) Additional includes for the Python UDF."""
+
def __init__(
self,
*,
@@ -1772,6 +1802,7 @@ class ScalarScalaUDF(google.protobuf.message.Message):
pyspark.sql.connect.proto.types_pb2.DataType
]:
"""(Optional) Input type(s) of the UDF"""
+
@property
def outputType(self) -> pyspark.sql.connect.proto.types_pb2.DataType:
"""(Required) Output type of the UDF"""
@@ -1783,8 +1814,9 @@ class ScalarScalaUDF(google.protobuf.message.Message):
self,
*,
payload: builtins.bytes = ...,
- inputTypes: collections.abc.Iterable[pyspark.sql.connect.proto.types_pb2.DataType]
- | None = ...,
+ inputTypes: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.types_pb2.DataType] | None
+ ) = ...,
outputType: pyspark.sql.connect.proto.types_pb2.DataType | None = ...,
nullable: builtins.bool = ...,
aggregate: builtins.bool = ...,
@@ -1862,6 +1894,7 @@ class TypedAggregateExpression(google.protobuf.message.Message):
@property
def scalar_scala_udf(self) -> global___ScalarScalaUDF:
"""(Required) The aggregate function object packed into bytes."""
+
def __init__(
self,
*,
@@ -1888,6 +1921,7 @@ class CallFunction(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Expression]:
"""(Optional) Function arguments. Empty arguments are allowed."""
+
def __init__(
self,
*,
@@ -1913,6 +1947,7 @@ class NamedArgumentExpression(google.protobuf.message.Message):
@property
def value(self) -> global___Expression:
"""(Required) The value expression of the named argument."""
+
def __init__(
self,
*,
@@ -1965,9 +2000,11 @@ class MergeAction(google.protobuf.message.Message):
@property
def key(self) -> global___Expression:
"""(Required) The key of the assignment."""
+
@property
def value(self) -> global___Expression:
"""(Required) The value of the assignment."""
+
def __init__(
self,
*,
@@ -1989,6 +2026,7 @@ class MergeAction(google.protobuf.message.Message):
@property
def condition(self) -> global___Expression:
"""(Optional) The condition expression of the merge action."""
+
@property
def assignments(
self,
@@ -1996,6 +2034,7 @@ class MergeAction(google.protobuf.message.Message):
global___MergeAction.Assignment
]:
"""(Optional) The assignments of the merge action. Required for ActionTypes INSERT and UPDATE."""
+
def __init__(
self,
*,
@@ -2070,6 +2109,7 @@ class SubqueryExpression(google.protobuf.message.Message):
global___Expression
]:
"""(Optional) The way that input rows are partitioned."""
+
@property
def order_spec(
self,
@@ -2126,11 +2166,13 @@ class SubqueryExpression(google.protobuf.message.Message):
@property
def table_arg_options(self) -> global___SubqueryExpression.TableArgOptions:
"""(Optional) Options specific to table arguments."""
+
@property
def in_subquery_values(
self,
) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Expression]:
"""(Optional) IN subquery values."""
+
def __init__(
self,
*,
diff --git a/python/pyspark/sql/connect/proto/ml_common_pb2.py b/python/pyspark/sql/connect/proto/ml_common_pb2.py
index 136f7c21d595..dda976b333d9 100644
--- a/python/pyspark/sql/connect/proto/ml_common_pb2.py
+++ b/python/pyspark/sql/connect/proto/ml_common_pb2.py
@@ -20,6 +20,7 @@
# source: spark/connect/ml_common.proto
# Protobuf Python Version: 6.33.5
"""Generated protocol buffer code."""
+
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
@@ -36,7 +37,6 @@ _sym_db = _symbol_database.Default()
from pyspark.sql.connect.proto import expressions_pb2 as spark_dot_connect_dot_expressions__pb2
-
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x1dspark/connect/ml_common.proto\x12\rspark.connect\x1a\x1fspark/connect/expressions.proto"\xa5\x01\n\x08MlParams\x12;\n\x06params\x18\x01 \x03(\x0b\x32#.spark.connect.MlParams.ParamsEntryR\x06params\x1a\\\n\x0bParamsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x05value:\x02\x38\x01"\x90\x02\n\nMlOperator\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x10\n\x03uid\x18\x02 \x01(\tR\x03uid\x12:\n\x04type [...]
)
@@ -48,9 +48,9 @@ _builder.BuildTopDescriptorsAndMessages(
)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals[
- "DESCRIPTOR"
- ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ _globals["DESCRIPTOR"]._serialized_options = (
+ b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ )
_globals["_MLPARAMS_PARAMSENTRY"]._loaded_options = None
_globals["_MLPARAMS_PARAMSENTRY"]._serialized_options = b"8\001"
_globals["_MLPARAMS"]._serialized_start = 82
diff --git a/python/pyspark/sql/connect/proto/ml_common_pb2.pyi b/python/pyspark/sql/connect/proto/ml_common_pb2.pyi
index bc540028eb08..23be14b7634e 100644
--- a/python/pyspark/sql/connect/proto/ml_common_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/ml_common_pb2.pyi
@@ -33,6 +33,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
+
import builtins
import collections.abc
import google.protobuf.descriptor
@@ -84,13 +85,16 @@ class MlParams(google.protobuf.message.Message):
builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
]:
"""User-supplied params"""
+
def __init__(
self,
*,
- params: collections.abc.Mapping[
- builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
- ]
- | None = ...,
+ params: (
+ collections.abc.Mapping[
+ builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+ ]
+ | None
+ ) = ...,
) -> None: ...
def ClearField(self, field_name: typing_extensions.Literal["params", b"params"]) -> None: ...
diff --git a/python/pyspark/sql/connect/proto/ml_pb2.py b/python/pyspark/sql/connect/proto/ml_pb2.py
index 48b6fde046fd..9cbc85c65b6d 100644
--- a/python/pyspark/sql/connect/proto/ml_pb2.py
+++ b/python/pyspark/sql/connect/proto/ml_pb2.py
@@ -20,6 +20,7 @@
# source: spark/connect/ml.proto
# Protobuf Python Version: 6.33.5
"""Generated protocol buffer code."""
+
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
@@ -38,7 +39,6 @@ from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_rel
from pyspark.sql.connect.proto import expressions_pb2 as spark_dot_connect_dot_expressions__pb2
from pyspark.sql.connect.proto import ml_common_pb2 as spark_dot_connect_dot_ml__common__pb2
-
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x16spark/connect/ml.proto\x12\rspark.connect\x1a\x1dspark/connect/relations.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/ml_common.proto"\xc7\x0e\n\tMlCommand\x12\x30\n\x03\x66it\x18\x01 \x01(\x0b\x32\x1c.spark.connect.MlCommand.FitH\x00R\x03\x66it\x12,\n\x05\x66\x65tch\x18\x02 \x01(\x0b\x32\x14.spark.connect.FetchH\x00R\x05\x66\x65tch\x12\x39\n\x06\x64\x65lete\x18\x03 \x01(\x0b\x32\x1f.spark.connect.MlCommand.DeleteH\x00R\x06\x64\x65lete\x12\x36\n\x05write\ [...]
)
@@ -48,9 +48,9 @@ _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "pyspark.sql.connect.proto.ml_pb2", _globals)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals[
- "DESCRIPTOR"
- ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ _globals["DESCRIPTOR"]._serialized_options = (
+ b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ )
_globals["_MLCOMMAND_WRITE_OPTIONSENTRY"]._loaded_options = None
_globals["_MLCOMMAND_WRITE_OPTIONSENTRY"]._serialized_options = b"8\001"
_globals["_MLCOMMAND"]._serialized_start = 137
diff --git a/python/pyspark/sql/connect/proto/ml_pb2.pyi b/python/pyspark/sql/connect/proto/ml_pb2.pyi
index 156ef846a8d1..759bde72134a 100644
--- a/python/pyspark/sql/connect/proto/ml_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/ml_pb2.pyi
@@ -33,6 +33,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
+
import builtins
import collections.abc
import google.protobuf.descriptor
@@ -67,12 +68,15 @@ class MlCommand(google.protobuf.message.Message):
@property
def estimator(self) -> pyspark.sql.connect.proto.ml_common_pb2.MlOperator:
"""(Required) Estimator information (its type should be OPERATOR_TYPE_ESTIMATOR)"""
+
@property
def params(self) -> pyspark.sql.connect.proto.ml_common_pb2.MlParams:
"""(Optional) parameters of the Estimator"""
+
@property
def dataset(self) -> pyspark.sql.connect.proto.relations_pb2.Relation:
"""(Required) the training dataset"""
+
def __init__(
self,
*,
@@ -132,8 +136,9 @@ class MlCommand(google.protobuf.message.Message):
def __init__(
self,
*,
- obj_refs: collections.abc.Iterable[pyspark.sql.connect.proto.ml_common_pb2.ObjectRef]
- | None = ...,
+ obj_refs: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.ml_common_pb2.ObjectRef] | None
+ ) = ...,
evict_only: builtins.bool | None = ...,
) -> None: ...
def HasField(
@@ -201,9 +206,11 @@ class MlCommand(google.protobuf.message.Message):
@property
def operator(self) -> pyspark.sql.connect.proto.ml_common_pb2.MlOperator:
"""Estimator or evaluator"""
+
@property
def obj_ref(self) -> pyspark.sql.connect.proto.ml_common_pb2.ObjectRef:
"""The cached model"""
+
@property
def params(self) -> pyspark.sql.connect.proto.ml_common_pb2.MlParams:
"""(Optional) The parameters of operator which could be estimator/evaluator or a cached model"""
@@ -216,6 +223,7 @@ class MlCommand(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""(Optional) The options of the writer"""
+
def __init__(
self,
*,
@@ -317,12 +325,15 @@ class MlCommand(google.protobuf.message.Message):
@property
def evaluator(self) -> pyspark.sql.connect.proto.ml_common_pb2.MlOperator:
"""(Required) Evaluator information (its type should be OPERATOR_TYPE_EVALUATOR)"""
+
@property
def params(self) -> pyspark.sql.connect.proto.ml_common_pb2.MlParams:
"""(Optional) parameters of the Evaluator"""
+
@property
def dataset(self) -> pyspark.sql.connect.proto.relations_pb2.Relation:
"""(Required) the evaluating dataset"""
+
def __init__(
self,
*,
@@ -506,9 +517,7 @@ class MlCommand(google.protobuf.message.Message):
b"write",
],
) -> None: ...
- def WhichOneof(
- self, oneof_group: typing_extensions.Literal["command", b"command"]
- ) -> (
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["command", b"command"]) -> (
typing_extensions.Literal[
"fit",
"fetch",
@@ -638,6 +647,7 @@ class MlCommandResult(google.protobuf.message.Message):
@property
def operator_info(self) -> global___MlCommandResult.MlOperatorInfo:
"""Operator information"""
+
def __init__(
self,
*,
diff --git a/python/pyspark/sql/connect/proto/pipelines_pb2.py b/python/pyspark/sql/connect/proto/pipelines_pb2.py
index 31d3996891b9..712c00d3d11f 100644
--- a/python/pyspark/sql/connect/proto/pipelines_pb2.py
+++ b/python/pyspark/sql/connect/proto/pipelines_pb2.py
@@ -20,6 +20,7 @@
# source: spark/connect/pipelines.proto
# Protobuf Python Version: 6.33.5
"""Generated protocol buffer code."""
+
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
@@ -40,7 +41,6 @@ from pyspark.sql.connect.proto import common_pb2 as spark_dot_connect_dot_common
from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_relations__pb2
from pyspark.sql.connect.proto import types_pb2 as spark_dot_connect_dot_types__pb2
-
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x1dspark/connect/pipelines.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1aspark/connect/common.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"\xa4\'\n\x0fPipelineCommand\x12h\n\x15\x63reate_dataflow_graph\x18\x01 \x01(\x0b\x32\x32.spark.connect.PipelineCommand.CreateDataflowGraphH\x00R\x13\x63reateDataflowGraph\x12R\n\rdefine_output\x18\x02 \x01(\x0b\x32+.spark.connect.PipelineCommand.DefineOut [...]
)
@@ -52,21 +52,21 @@ _builder.BuildTopDescriptorsAndMessages(
)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals[
- "DESCRIPTOR"
- ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ _globals["DESCRIPTOR"]._serialized_options = (
+ b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ )
_globals["_PIPELINECOMMAND_CREATEDATAFLOWGRAPH_SQLCONFENTRY"]._loaded_options = None
_globals["_PIPELINECOMMAND_CREATEDATAFLOWGRAPH_SQLCONFENTRY"]._serialized_options = b"8\001"
- _globals[
- "_PIPELINECOMMAND_DEFINEOUTPUT_TABLEDETAILS_TABLEPROPERTIESENTRY"
- ]._loaded_options = None
+ _globals["_PIPELINECOMMAND_DEFINEOUTPUT_TABLEDETAILS_TABLEPROPERTIESENTRY"]._loaded_options = (
+ None
+ )
_globals[
"_PIPELINECOMMAND_DEFINEOUTPUT_TABLEDETAILS_TABLEPROPERTIESENTRY"
]._serialized_options = b"8\001"
_globals["_PIPELINECOMMAND_DEFINEOUTPUT_SINKDETAILS_OPTIONSENTRY"]._loaded_options = None
- _globals[
- "_PIPELINECOMMAND_DEFINEOUTPUT_SINKDETAILS_OPTIONSENTRY"
- ]._serialized_options = b"8\001"
+ _globals["_PIPELINECOMMAND_DEFINEOUTPUT_SINKDETAILS_OPTIONSENTRY"]._serialized_options = (
+ b"8\001"
+ )
_globals["_PIPELINECOMMAND_DEFINEFLOW_SQLCONFENTRY"]._loaded_options = None
_globals["_PIPELINECOMMAND_DEFINEFLOW_SQLCONFENTRY"]._serialized_options = b"8\001"
_globals["_PIPELINECOMMAND_DEFINEFLOWQUERYFUNCTIONRESULT"].fields_by_name[
@@ -102,9 +102,9 @@ if not _descriptor._USE_C_DESCRIPTORS:
_globals[
"_PIPELINECOMMAND_DEFINEOUTPUT_TABLEDETAILS_TABLEPROPERTIESENTRY"
]._serialized_start = 2532
- _globals[
- "_PIPELINECOMMAND_DEFINEOUTPUT_TABLEDETAILS_TABLEPROPERTIESENTRY"
- ]._serialized_end = 2598
+ _globals["_PIPELINECOMMAND_DEFINEOUTPUT_TABLEDETAILS_TABLEPROPERTIESENTRY"]._serialized_end = (
+ 2598
+ )
_globals["_PIPELINECOMMAND_DEFINEOUTPUT_SINKDETAILS"]._serialized_start = 2622
_globals["_PIPELINECOMMAND_DEFINEOUTPUT_SINKDETAILS"]._serialized_end = 2831
_globals["_PIPELINECOMMAND_DEFINEOUTPUT_SINKDETAILS_OPTIONSENTRY"]._serialized_start = 2762
diff --git a/python/pyspark/sql/connect/proto/pipelines_pb2.pyi b/python/pyspark/sql/connect/proto/pipelines_pb2.pyi
index 19ca1914e11e..e986e801acc5 100644
--- a/python/pyspark/sql/connect/proto/pipelines_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/pipelines_pb2.pyi
@@ -33,6 +33,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
+
import builtins
import collections.abc
import google.protobuf.any_pb2
@@ -128,6 +129,7 @@ class PipelineCommand(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""SQL configurations for all flows in this graph."""
+
def __init__(
self,
*,
@@ -246,6 +248,7 @@ class PipelineCommand(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""Optional table properties."""
+
@property
def partition_cols(
self,
@@ -261,6 +264,7 @@ class PipelineCommand(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""Optional cluster columns for the table."""
+
def __init__(
self,
*,
@@ -386,6 +390,7 @@ class PipelineCommand(google.protobuf.message.Message):
@property
def source_code_location(self) -> global___SourceCodeLocation:
"""The location in source code that this output was defined."""
+
@property
def table_details(self) -> global___PipelineCommand.DefineOutput.TableDetails: ...
@property
@@ -532,6 +537,7 @@ class PipelineCommand(google.protobuf.message.Message):
"""An unresolved relation that defines the dataset's flow. Empty if the query function
that defines the flow cannot be analyzed at the time of flow definition.
"""
+
def __init__(
self,
*,
@@ -607,6 +613,7 @@ class PipelineCommand(google.protobuf.message.Message):
@property
def source_code_location(self) -> global___SourceCodeLocation:
"""The location in source code that this flow was defined."""
+
@property
def relation_flow_details(
self,
@@ -629,8 +636,9 @@ class PipelineCommand(google.protobuf.message.Message):
sql_conf: collections.abc.Mapping[builtins.str, builtins.str] | None = ...,
client_id: builtins.str | None = ...,
source_code_location: global___SourceCodeLocation | None = ...,
- relation_flow_details: global___PipelineCommand.DefineFlow.WriteRelationFlowDetails
- | None = ...,
+ relation_flow_details: (
+ global___PipelineCommand.DefineFlow.WriteRelationFlowDetails | None
+ ) = ...,
extension: google.protobuf.any_pb2.Any | None = ...,
once: builtins.bool | None = ...,
) -> None: ...
@@ -753,6 +761,7 @@ class PipelineCommand(google.protobuf.message.Message):
@property
def define_output(self) -> global___PipelineCommand.DefineOutput:
"""The output (table or materialized view or sink) definition."""
+
@property
def define_flows(
self,
@@ -771,12 +780,14 @@ class PipelineCommand(google.protobuf.message.Message):
google.protobuf.any_pb2.Any
]:
"""Reserved field for protocol extensions."""
+
def __init__(
self,
*,
define_output: global___PipelineCommand.DefineOutput | None = ...,
- define_flows: collections.abc.Iterable[global___PipelineCommand.DefineFlow]
- | None = ...,
+ define_flows: (
+ collections.abc.Iterable[global___PipelineCommand.DefineFlow] | None
+ ) = ...,
full_refresh: builtins.bool | None = ...,
storage: builtins.str | None = ...,
extension: collections.abc.Iterable[google.protobuf.any_pb2.Any] | None = ...,
@@ -1087,6 +1098,7 @@ class PipelineCommand(google.protobuf.message.Message):
@property
def relation(self) -> pyspark.sql.connect.proto.relations_pb2.Relation:
"""An unresolved relation that defines the dataset's flow."""
+
def __init__(
self,
*,
@@ -1194,6 +1206,7 @@ class PipelineCommand(google.protobuf.message.Message):
that are not yet defined in this version of the proto. During planning, the
engine will resolve and dispatch the concrete command contained in this field.
"""
+
def __init__(
self,
*,
@@ -1203,10 +1216,12 @@ class PipelineCommand(google.protobuf.message.Message):
drop_dataflow_graph: global___PipelineCommand.DropDataflowGraph | None = ...,
start_run: global___PipelineCommand.StartRun | None = ...,
define_sql_graph_elements: global___PipelineCommand.DefineSqlGraphElements | None = ...,
- get_query_function_execution_signal_stream: global___PipelineCommand.GetQueryFunctionExecutionSignalStream
- | None = ...,
- define_flow_query_function_result: global___PipelineCommand.DefineFlowQueryFunctionResult
- | None = ...,
+ get_query_function_execution_signal_stream: (
+ global___PipelineCommand.GetQueryFunctionExecutionSignalStream | None
+ ) = ...,
+ define_flow_query_function_result: (
+ global___PipelineCommand.DefineFlowQueryFunctionResult | None
+ ) = ...,
execute_output_flows: global___PipelineCommand.ExecuteOutputFlows | None = ...,
extension: google.protobuf.any_pb2.Any | None = ...,
) -> None: ...
@@ -1330,11 +1345,13 @@ class PipelineCommandResult(google.protobuf.message.Message):
@property
def resolved_identifier(self) -> pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier:
"""Resolved identifier of the output"""
+
def __init__(
self,
*,
- resolved_identifier: pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier
- | None = ...,
+ resolved_identifier: (
+ pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier | None
+ ) = ...,
) -> None: ...
def HasField(
self,
@@ -1366,11 +1383,13 @@ class PipelineCommandResult(google.protobuf.message.Message):
@property
def resolved_identifier(self) -> pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier:
"""Resolved identifier of the flow"""
+
def __init__(
self,
*,
- resolved_identifier: pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier
- | None = ...,
+ resolved_identifier: (
+ pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier | None
+ ) = ...,
) -> None: ...
def HasField(
self,
@@ -1409,8 +1428,9 @@ class PipelineCommandResult(google.protobuf.message.Message):
def __init__(
self,
*,
- create_dataflow_graph_result: global___PipelineCommandResult.CreateDataflowGraphResult
- | None = ...,
+ create_dataflow_graph_result: (
+ global___PipelineCommandResult.CreateDataflowGraphResult | None
+ ) = ...,
define_output_result: global___PipelineCommandResult.DefineOutputResult | None = ...,
define_flow_result: global___PipelineCommandResult.DefineFlowResult | None = ...,
) -> None: ...
@@ -1531,6 +1551,7 @@ class SourceCodeLocation(google.protobuf.message.Message):
that are not yet defined in this version of the proto. During planning, the
engine will resolve and dispatch the concrete command contained in this field.
"""
+
def __init__(
self,
*,
@@ -1607,6 +1628,7 @@ class PipelineQueryFunctionExecutionSignal(google.protobuf.message.Message):
This field is deprecated since Spark 4.2+. Use flow_identifiers field instead.
"""
+
@property
def flow_identifiers(
self,
@@ -1614,14 +1636,14 @@ class PipelineQueryFunctionExecutionSignal(google.protobuf.message.Message):
pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier
]:
"""The identifier of flows that are ready to be re-evaluated"""
+
def __init__(
self,
*,
flow_names: collections.abc.Iterable[builtins.str] | None = ...,
- flow_identifiers: collections.abc.Iterable[
- pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier
- ]
- | None = ...,
+ flow_identifiers: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier] | None
+ ) = ...,
) -> None: ...
def ClearField(
self,
@@ -1654,6 +1676,7 @@ class PipelineAnalysisContext(google.protobuf.message.Message):
@property
def flow_identifier(self) -> pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier:
"""The identifier of the Flow involved in this analysis"""
+
@property
def extension(
self,
@@ -1661,6 +1684,7 @@ class PipelineAnalysisContext(google.protobuf.message.Message):
google.protobuf.any_pb2.Any
]:
"""Reserved field for protocol extensions."""
+
def __init__(
self,
*,
diff --git a/python/pyspark/sql/connect/proto/relations_pb2.py b/python/pyspark/sql/connect/proto/relations_pb2.py
index 835587c3333d..36ad7def3696 100644
--- a/python/pyspark/sql/connect/proto/relations_pb2.py
+++ b/python/pyspark/sql/connect/proto/relations_pb2.py
@@ -20,6 +20,7 @@
# source: spark/connect/relations.proto
# Protobuf Python Version: 6.33.5
"""Generated protocol buffer code."""
+
from google.protobuf import descriptor as _descriptor
from google.protobuf import descriptor_pool as _descriptor_pool
from google.protobuf import runtime_version as _runtime_version
@@ -41,7 +42,6 @@ from pyspark.sql.connect.proto import catalog_pb2 as spark_dot_connect_dot_catal
from pyspark.sql.connect.proto import common_pb2 as spark_dot_connect_dot_common__pb2
from pyspark.sql.connect.proto import ml_common_pb2 as spark_dot_connect_dot_ml__common__pb2
-
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x19spark/connect/types.proto\x1a\x1bspark/connect/catalog.proto\x1a\x1aspark/connect/common.proto\x1a\x1dspark/connect/ml_common.proto"\x8c\x1e\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x [...]
)
@@ -53,9 +53,9 @@ _builder.BuildTopDescriptorsAndMessages(
)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals[
- "DESCRIPTOR"
- ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ _globals["DESCRIPTOR"]._serialized_options = (
+ b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ )
_globals["_RELATIONCOMMON"].fields_by_name["source_info"]._loaded_options = None
_globals["_RELATIONCOMMON"].fields_by_name["source_info"]._serialized_options = b"\030\001"
_globals["_SQL_ARGSENTRY"]._loaded_options = None
diff --git a/python/pyspark/sql/connect/proto/relations_pb2.pyi b/python/pyspark/sql/connect/proto/relations_pb2.pyi
index 974ab9bc422d..ff6e02850731 100644
--- a/python/pyspark/sql/connect/proto/relations_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/relations_pb2.pyi
@@ -33,6 +33,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
+
import builtins
import collections.abc
import google.protobuf.any_pb2
@@ -221,6 +222,7 @@ class Relation(google.protobuf.message.Message):
@property
def fill_na(self) -> global___NAFill:
"""NA functions"""
+
@property
def drop_na(self) -> global___NADrop: ...
@property
@@ -228,6 +230,7 @@ class Relation(google.protobuf.message.Message):
@property
def summary(self) -> global___StatSummary:
"""stat functions"""
+
@property
def crosstab(self) -> global___StatCrosstab: ...
@property
@@ -245,14 +248,17 @@ class Relation(google.protobuf.message.Message):
@property
def catalog(self) -> pyspark.sql.connect.proto.catalog_pb2.Catalog:
"""Catalog API (experimental / unstable)"""
+
@property
def ml_relation(self) -> global___MlRelation:
"""ML relation"""
+
@property
def extension(self) -> google.protobuf.any_pb2.Any:
"""This field is used to mark extensions to the protocol. When plugins generate arbitrary
relations they can add them here. During the planning the correct resolution is done.
"""
+
@property
def unknown(self) -> global___Unknown: ...
def __init__(
@@ -295,11 +301,13 @@ class Relation(google.protobuf.message.Message):
html_string: global___HtmlString | None = ...,
cached_local_relation: global___CachedLocalRelation | None = ...,
cached_remote_relation: global___CachedRemoteRelation | None = ...,
- common_inline_user_defined_table_function: global___CommonInlineUserDefinedTableFunction
- | None = ...,
+ common_inline_user_defined_table_function: (
+ global___CommonInlineUserDefinedTableFunction | None
+ ) = ...,
as_of_join: global___AsOfJoin | None = ...,
- common_inline_user_defined_data_source: global___CommonInlineUserDefinedDataSource
- | None = ...,
+ common_inline_user_defined_data_source: (
+ global___CommonInlineUserDefinedDataSource | None
+ ) = ...,
with_relations: global___WithRelations | None = ...,
transpose: global___Transpose | None = ...,
unresolved_table_valued_function: global___UnresolvedTableValuedFunction | None = ...,
@@ -575,9 +583,7 @@ class Relation(google.protobuf.message.Message):
b"with_watermark",
],
) -> None: ...
- def WhichOneof(
- self, oneof_group: typing_extensions.Literal["rel_type", b"rel_type"]
- ) -> (
+ def WhichOneof(self, oneof_group: typing_extensions.Literal["rel_type", b"rel_type"]) -> (
typing_extensions.Literal[
"read",
"project",
@@ -663,15 +669,19 @@ class MlRelation(google.protobuf.message.Message):
@property
def obj_ref(self) -> pyspark.sql.connect.proto.ml_common_pb2.ObjectRef:
"""Object reference"""
+
@property
def transformer(self) -> pyspark.sql.connect.proto.ml_common_pb2.MlOperator:
"""Could be an ML transformer like VectorAssembler"""
+
@property
def input(self) -> global___Relation:
"""the input dataframe"""
+
@property
def params(self) -> pyspark.sql.connect.proto.ml_common_pb2.MlParams:
"""the operator specific parameters"""
+
def __init__(
self,
*,
@@ -724,6 +734,7 @@ class MlRelation(google.protobuf.message.Message):
@property
def model_summary_dataset(self) -> global___Relation:
"""(Optional) the dataset for restoring the model summary"""
+
def __init__(
self,
*,
@@ -830,6 +841,7 @@ class Fetch(google.protobuf.message.Message):
global___Fetch.Method.Args
]:
"""(Optional) the arguments of the method"""
+
def __init__(
self,
*,
@@ -845,11 +857,13 @@ class Fetch(google.protobuf.message.Message):
@property
def obj_ref(self) -> pyspark.sql.connect.proto.ml_common_pb2.ObjectRef:
"""(Required) reference to the object on the server side"""
+
@property
def methods(
self,
) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Fetch.Method]:
"""(Required) the calling method chains"""
+
def __init__(
self,
*,
@@ -891,6 +905,7 @@ class RelationCommon(google.protobuf.message.Message):
@property
def origin(self) -> pyspark.sql.connect.proto.common_pb2.Origin:
"""(Optional) Keep the information of the origin for this expression such as stacktrace."""
+
def __init__(
self,
*,
@@ -984,6 +999,7 @@ class SQL(google.protobuf.message.Message):
builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
]:
"""(Optional) A map of parameter names to literal expressions."""
+
@property
def pos_args(
self,
@@ -991,6 +1007,7 @@ class SQL(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
]:
"""(Optional) A sequence of literal expressions for positional parameters in the SQL query text."""
+
@property
def named_arguments(
self,
@@ -1000,6 +1017,7 @@ class SQL(google.protobuf.message.Message):
"""(Optional) A map of parameter names to expressions.
It cannot coexist with `pos_arguments`.
"""
+
@property
def pos_arguments(
self,
@@ -1009,26 +1027,30 @@ class SQL(google.protobuf.message.Message):
"""(Optional) A sequence of expressions for positional parameters in the SQL query text.
It cannot coexist with `named_arguments`.
"""
+
def __init__(
self,
*,
query: builtins.str = ...,
- args: collections.abc.Mapping[
- builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
- ]
- | None = ...,
- pos_args: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
- ]
- | None = ...,
- named_arguments: collections.abc.Mapping[
- builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
- pos_arguments: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
+ args: (
+ collections.abc.Mapping[
+ builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+ ]
+ | None
+ ) = ...,
+ pos_args: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression.Literal]
+ | None
+ ) = ...,
+ named_arguments: (
+ collections.abc.Mapping[
+ builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None
+ ) = ...,
+ pos_arguments: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
) -> None: ...
def ClearField(
self,
@@ -1067,6 +1089,7 @@ class WithRelations(google.protobuf.message.Message):
"""(Required) Plan at the root of the query tree. This plan is expected to contain one or more
references. Those references get expanded later on by the engine.
"""
+
@property
def references(
self,
@@ -1074,6 +1097,7 @@ class WithRelations(google.protobuf.message.Message):
"""(Required) Plans referenced by the root plan. Relations in this list are also allowed to
contain references to other relations in this list, as long they do not form cycles.
"""
+
def __init__(
self,
*,
@@ -1123,6 +1147,7 @@ class Read(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""Options for the named table. The map key is case insensitive."""
+
def __init__(
self,
*,
@@ -1180,11 +1205,13 @@ class Read(google.protobuf.message.Message):
data source format. This options could be empty for valid data source format.
The map key is case insensitive.
"""
+
@property
def paths(
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) A list of path for file-system backed data sources."""
+
@property
def predicates(
self,
@@ -1319,6 +1346,7 @@ class Project(google.protobuf.message.Message):
For example, `SELECT ABS(-1)` is valid plan without an input plan.
"""
+
@property
def expressions(
self,
@@ -1326,12 +1354,14 @@ class Project(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Required) A Project requires at least one expression."""
+
def __init__(
self,
*,
input: global___Relation | None = ...,
- expressions: collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
- | None = ...,
+ expressions: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
) -> None: ...
def HasField(
self, field_name: typing_extensions.Literal["input", b"input"]
@@ -1355,9 +1385,11 @@ class Filter(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) Input relation for a Filter."""
+
@property
def condition(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression:
"""(Required) A Filter must have a condition expression."""
+
def __init__(
self,
*,
@@ -1440,9 +1472,11 @@ class Join(google.protobuf.message.Message):
@property
def left(self) -> global___Relation:
"""(Required) Left input relation for a Join."""
+
@property
def right(self) -> global___Relation:
"""(Required) Right input relation for a Join."""
+
@property
def join_condition(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression:
"""(Optional) The join condition. Could be unset when `using_columns` is utilized.
@@ -1461,9 +1495,11 @@ class Join(google.protobuf.message.Message):
This field does not co-exist with join_condition.
"""
+
@property
def join_data_type(self) -> global___Join.JoinDataType:
"""(Optional) Only used by joinWith. Set the left and right join data types."""
+
def __init__(
self,
*,
@@ -1550,6 +1586,7 @@ class SetOperation(google.protobuf.message.Message):
@property
def left_input(self) -> global___Relation:
"""(Required) Left input relation for a Set operation."""
+
@property
def right_input(self) -> global___Relation:
"""(Required) Right input relation for a Set operation."""
@@ -1761,6 +1798,7 @@ class Aggregate(google.protobuf.message.Message):
@property
def col(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression:
"""(Required) The column to pivot"""
+
@property
def values(
self,
@@ -1772,14 +1810,17 @@ class Aggregate(google.protobuf.message.Message):
Note that if it is empty, the server side will immediately trigger a job to collect
the distinct values of the column.
"""
+
def __init__(
self,
*,
col: pyspark.sql.connect.proto.expressions_pb2.Expression | None = ...,
- values: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
- ]
- | None = ...,
+ values: (
+ collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+ ]
+ | None
+ ) = ...,
) -> None: ...
def HasField(
self, field_name: typing_extensions.Literal["col", b"col"]
@@ -1799,13 +1840,14 @@ class Aggregate(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Required) Individual grouping set"""
+
def __init__(
self,
*,
- grouping_set: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
+ grouping_set: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
+ | None
+ ) = ...,
) -> None: ...
def ClearField(
self, field_name: typing_extensions.Literal["grouping_set", b"grouping_set"]
@@ -1829,6 +1871,7 @@ class Aggregate(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Required) Expressions for grouping keys"""
+
@property
def aggregate_expressions(
self,
@@ -1836,9 +1879,11 @@ class Aggregate(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Required) List of values that will be translated to columns in the output DataFrame."""
+
@property
def pivot(self) -> global___Aggregate.Pivot:
"""(Optional) Pivots a column of the current `DataFrame` and performs the specified aggregation."""
+
@property
def grouping_sets(
self,
@@ -1846,19 +1891,18 @@ class Aggregate(google.protobuf.message.Message):
global___Aggregate.GroupingSets
]:
"""(Optional) List of values that will be translated to columns in the output DataFrame."""
+
def __init__(
self,
*,
input: global___Relation | None = ...,
group_type: global___Aggregate.GroupType.ValueType = ...,
- grouping_expressions: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
- aggregate_expressions: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
+ grouping_expressions: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
+ aggregate_expressions: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
pivot: global___Aggregate.Pivot | None = ...,
grouping_sets: collections.abc.Iterable[global___Aggregate.GroupingSets] | None = ...,
) -> None: ...
@@ -1896,6 +1940,7 @@ class Sort(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) Input relation for a Sort."""
+
@property
def order(
self,
@@ -1909,10 +1954,10 @@ class Sort(google.protobuf.message.Message):
self,
*,
input: global___Relation | None = ...,
- order: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression.SortOrder
- ]
- | None = ...,
+ order: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression.SortOrder]
+ | None
+ ) = ...,
is_global: builtins.bool | None = ...,
) -> None: ...
def HasField(
@@ -1951,6 +1996,7 @@ class Drop(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
+
@property
def columns(
self,
@@ -1958,17 +2004,20 @@ class Drop(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Optional) columns to drop."""
+
@property
def column_names(
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) names of columns to drop."""
+
def __init__(
self,
*,
input: global___Relation | None = ...,
- columns: collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
- | None = ...,
+ columns: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
column_names: collections.abc.Iterable[builtins.str] | None = ...,
) -> None: ...
def HasField(
@@ -1997,6 +2046,7 @@ class Deduplicate(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) Input relation for a Deduplicate."""
+
@property
def column_names(
self,
@@ -2367,6 +2417,7 @@ class SubqueryAlias(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) Qualifier of the alias."""
+
def __init__(
self,
*,
@@ -2533,6 +2584,7 @@ class StatSummary(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
+
@property
def statistics(
self,
@@ -2552,6 +2604,7 @@ class StatSummary(google.protobuf.message.Message):
If no statistics are given, this function computes 'count', 'mean', 'stddev', 'min',
'approximate quartiles' (percentiles at 25%, 50%, and 75%), and 'max'.
"""
+
def __init__(
self,
*,
@@ -2580,11 +2633,13 @@ class StatDescribe(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
+
@property
def cols(
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) Columns to compute statistics on."""
+
def __init__(
self,
*,
@@ -2749,11 +2804,13 @@ class StatApproxQuantile(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
+
@property
def cols(
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Required) The names of the numerical columns."""
+
@property
def probabilities(
self,
@@ -2810,6 +2867,7 @@ class StatFreqItems(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
+
@property
def cols(
self,
@@ -2884,9 +2942,11 @@ class StatSampleBy(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
+
@property
def col(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression:
"""(Required) The column that defines strata."""
+
@property
def fractions(
self,
@@ -2957,11 +3017,13 @@ class NAFill(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
+
@property
def cols(
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) Optional list of column names to consider."""
+
@property
def values(
self,
@@ -2973,15 +3035,16 @@ class NAFill(google.protobuf.message.Message):
Should contain at least 1 item.
Only 4 data types are supported now: bool, long, double, string
"""
+
def __init__(
self,
*,
input: global___Relation | None = ...,
cols: collections.abc.Iterable[builtins.str] | None = ...,
- values: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
- ]
- | None = ...,
+ values: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression.Literal]
+ | None
+ ) = ...,
) -> None: ...
def HasField(
self, field_name: typing_extensions.Literal["input", b"input"]
@@ -3008,6 +3071,7 @@ class NADrop(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
+
@property
def cols(
self,
@@ -3082,12 +3146,14 @@ class NAReplace(google.protobuf.message.Message):
Only 4 data types are supported now: null, bool, double, string.
"""
+
@property
def new_value(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression.Literal:
"""(Required) The new value.
Should be of the same data type with the old value.
"""
+
def __init__(
self,
*,
@@ -3113,6 +3179,7 @@ class NAReplace(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
+
@property
def cols(
self,
@@ -3121,6 +3188,7 @@ class NAReplace(google.protobuf.message.Message):
When it is empty, all the type-compatible columns in the input relation will be considered.
"""
+
@property
def replacements(
self,
@@ -3128,6 +3196,7 @@ class NAReplace(google.protobuf.message.Message):
global___NAReplace.Replacement
]:
"""(Optional) The value replacement mapping."""
+
def __init__(
self,
*,
@@ -3157,6 +3226,7 @@ class ToDF(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation of RenameColumnsBySameLengthNames."""
+
@property
def column_names(
self,
@@ -3166,6 +3236,7 @@ class ToDF(google.protobuf.message.Message):
The number of columns of the input relation must be equal to the length
of this field. If this is not true, an exception will be returned.
"""
+
def __init__(
self,
*,
@@ -3232,6 +3303,7 @@ class WithColumnsRenamed(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
+
@property
def rename_columns_map(
self,
@@ -3243,6 +3315,7 @@ class WithColumnsRenamed(google.protobuf.message.Message):
does not require that all input relation column names to present as keys.
duplicated B are not allowed.
"""
+
@property
def renames(
self,
@@ -3278,6 +3351,7 @@ class WithColumns(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
+
@property
def aliases(
self,
@@ -3294,14 +3368,15 @@ class WithColumns(google.protobuf.message.Message):
An exception is thrown when duplicated names are present in the mapping.
"""
+
def __init__(
self,
*,
input: global___Relation | None = ...,
- aliases: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression.Alias
- ]
- | None = ...,
+ aliases: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression.Alias]
+ | None
+ ) = ...,
) -> None: ...
def HasField(
self, field_name: typing_extensions.Literal["input", b"input"]
@@ -3369,13 +3444,15 @@ class Hint(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Optional) Hint parameters."""
+
def __init__(
self,
*,
input: global___Relation | None = ...,
name: builtins.str = ...,
- parameters: collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
- | None = ...,
+ parameters: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
) -> None: ...
def HasField(
self, field_name: typing_extensions.Literal["input", b"input"]
@@ -3407,8 +3484,10 @@ class Unpivot(google.protobuf.message.Message):
def __init__(
self,
*,
- values: collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
- | None = ...,
+ values: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
+ | None
+ ) = ...,
) -> None: ...
def ClearField(
self, field_name: typing_extensions.Literal["values", b"values"]
@@ -3422,6 +3501,7 @@ class Unpivot(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
+
@property
def ids(
self,
@@ -3429,6 +3509,7 @@ class Unpivot(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Required) Id columns."""
+
@property
def values(self) -> global___Unpivot.Values:
"""(Optional) Value columns to unpivot."""
@@ -3440,8 +3521,9 @@ class Unpivot(google.protobuf.message.Message):
self,
*,
input: global___Relation | None = ...,
- ids: collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
- | None = ...,
+ ids: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
values: global___Unpivot.Values | None = ...,
variable_column_name: builtins.str = ...,
value_column_name: builtins.str = ...,
@@ -3488,6 +3570,7 @@ class Transpose(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
+
@property
def index_columns(
self,
@@ -3497,14 +3580,14 @@ class Transpose(google.protobuf.message.Message):
"""(Optional) A list of columns that will be treated as the indices.
Only single column is supported now.
"""
+
def __init__(
self,
*,
input: global___Relation | None = ...,
- index_columns: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
+ index_columns: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
) -> None: ...
def HasField(
self, field_name: typing_extensions.Literal["input", b"input"]
@@ -3530,12 +3613,14 @@ class UnresolvedTableValuedFunction(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Optional) Function arguments. Empty arguments are allowed."""
+
def __init__(
self,
*,
function_name: builtins.str = ...,
- arguments: collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
- | None = ...,
+ arguments: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
) -> None: ...
def ClearField(
self,
@@ -3554,12 +3639,14 @@ class ToSchema(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
+
@property
def schema(self) -> pyspark.sql.connect.proto.types_pb2.DataType:
"""(Required) The user provided schema.
The Sever side will update the dataframe with this schema.
"""
+
def __init__(
self,
*,
@@ -3584,6 +3671,7 @@ class RepartitionByExpression(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
+
@property
def partition_exprs(
self,
@@ -3597,10 +3685,9 @@ class RepartitionByExpression(google.protobuf.message.Message):
self,
*,
input: global___Relation | None = ...,
- partition_exprs: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
+ partition_exprs: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
num_partitions: builtins.int | None = ...,
) -> None: ...
def HasField(
@@ -3643,6 +3730,7 @@ class MapPartitions(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) Input relation for a mapPartitions-equivalent API: mapInPandas, mapInArrow."""
+
@property
def func(self) -> pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction:
"""(Required) Input user-defined function."""
@@ -3654,8 +3742,9 @@ class MapPartitions(google.protobuf.message.Message):
self,
*,
input: global___Relation | None = ...,
- func: pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction
- | None = ...,
+ func: (
+ pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction | None
+ ) = ...,
is_barrier: builtins.bool | None = ...,
profile_id: builtins.int | None = ...,
) -> None: ...
@@ -3721,6 +3810,7 @@ class GroupMap(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) Input relation for Group Map API: apply, applyInPandas."""
+
@property
def grouping_expressions(
self,
@@ -3728,9 +3818,11 @@ class GroupMap(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Required) Expressions for grouping keys."""
+
@property
def func(self) -> pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction:
"""(Required) Input user-defined function."""
+
@property
def sorting_expressions(
self,
@@ -3738,11 +3830,13 @@ class GroupMap(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Optional) Expressions for sorting. Only used by Scala Sorted Group Map API."""
+
@property
def initial_input(self) -> global___Relation:
"""Below fields are only used by (Flat)MapGroupsWithState
(Optional) Input relation for initial State.
"""
+
@property
def initial_grouping_expressions(
self,
@@ -3759,30 +3853,30 @@ class GroupMap(google.protobuf.message.Message):
@property
def state_schema(self) -> pyspark.sql.connect.proto.types_pb2.DataType:
"""(Optional) The schema for the grouped state."""
+
@property
def transform_with_state_info(self) -> global___TransformWithStateInfo:
"""Below fields are used by TransformWithState and TransformWithStateInPandas
(Optional) TransformWithState related parameters.
"""
+
def __init__(
self,
*,
input: global___Relation | None = ...,
- grouping_expressions: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
- func: pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction
- | None = ...,
- sorting_expressions: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
+ grouping_expressions: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
+ func: (
+ pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction | None
+ ) = ...,
+ sorting_expressions: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
initial_input: global___Relation | None = ...,
- initial_grouping_expressions: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
+ initial_grouping_expressions: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
is_map_groups_with_state: builtins.bool | None = ...,
output_mode: builtins.str | None = ...,
timeout_conf: builtins.str | None = ...,
@@ -3903,6 +3997,7 @@ class TransformWithStateInfo(google.protobuf.message.Message):
"""(Optional) Schema for the output DataFrame.
Only required used for TransformWithStateInPandas.
"""
+
def __init__(
self,
*,
@@ -3965,6 +4060,7 @@ class CoGroupMap(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) One input relation for CoGroup Map API - applyInPandas."""
+
@property
def input_grouping_expressions(
self,
@@ -3972,9 +4068,11 @@ class CoGroupMap(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""Expressions for grouping keys of the first input relation."""
+
@property
def other(self) -> global___Relation:
"""(Required) The other input relation."""
+
@property
def other_grouping_expressions(
self,
@@ -3982,9 +4080,11 @@ class CoGroupMap(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""Expressions for grouping keys of the other input relation."""
+
@property
def func(self) -> pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction:
"""(Required) Input user-defined function."""
+
@property
def input_sorting_expressions(
self,
@@ -3992,6 +4092,7 @@ class CoGroupMap(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Optional) Expressions for sorting. Only used by Scala Sorted CoGroup Map API."""
+
@property
def other_sorting_expressions(
self,
@@ -3999,29 +4100,27 @@ class CoGroupMap(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Optional) Expressions for sorting. Only used by Scala Sorted CoGroup Map API."""
+
def __init__(
self,
*,
input: global___Relation | None = ...,
- input_grouping_expressions: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
+ input_grouping_expressions: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
other: global___Relation | None = ...,
- other_grouping_expressions: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
- func: pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction
- | None = ...,
- input_sorting_expressions: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
- other_sorting_expressions: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
+ other_grouping_expressions: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
+ func: (
+ pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction | None
+ ) = ...,
+ input_sorting_expressions: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
+ other_sorting_expressions: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
) -> None: ...
def HasField(
self,
@@ -4064,6 +4163,7 @@ class ApplyInPandasWithState(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) Input relation for applyInPandasWithState."""
+
@property
def grouping_expressions(
self,
@@ -4071,6 +4171,7 @@ class ApplyInPandasWithState(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Required) Expressions for grouping keys."""
+
@property
def func(self) -> pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction:
"""(Required) Input user-defined function."""
@@ -4086,12 +4187,12 @@ class ApplyInPandasWithState(google.protobuf.message.Message):
self,
*,
input: global___Relation | None = ...,
- grouping_expressions: collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None = ...,
- func: pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction
- | None = ...,
+ grouping_expressions: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
+ func: (
+ pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction | None
+ ) = ...,
output_schema: builtins.str = ...,
state_schema: builtins.str = ...,
output_mode: builtins.str = ...,
@@ -4140,6 +4241,7 @@ class CommonInlineUserDefinedTableFunction(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Optional) Function input arguments. Empty arguments are allowed."""
+
@property
def python_udtf(self) -> global___PythonUDTF: ...
def __init__(
@@ -4147,8 +4249,9 @@ class CommonInlineUserDefinedTableFunction(google.protobuf.message.Message):
*,
function_name: builtins.str = ...,
deterministic: builtins.bool = ...,
- arguments: collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
- | None = ...,
+ arguments: (
+ collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
+ ) = ...,
python_udtf: global___PythonUDTF | None = ...,
... 4837 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org