You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2026/03/19 22:20:50 UTC
(spark) branch master updated: [SPARK-56018][PYTHON] Use ruff as formatter
This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new eac3fe36e846 [SPARK-56018][PYTHON] Use ruff as formatter
eac3fe36e846 is described below
commit eac3fe36e84674bfe82d6d5d3af6758cbd16be3e
Author: Tian Gao <ga...@hotmail.com>
AuthorDate: Fri Mar 20 07:20:37 2026 +0900
[SPARK-56018][PYTHON] Use ruff as formatter
### What changes were proposed in this pull request?
Replace `black` with `ruff format`.
### Why are the changes needed?
There are few reasons we should use `ruff`
1. We already use `ruff` for linter, using it for `format` will reduce a dependency, which makes upgrade easier
2. `ruff` is significantly faster than `black` which is helpful for our pre-commit hooks
3. `ruff` is more customizable if we need
4. Personally I think the taste of `ruff` is slightly better than `black`. For example:
* `ruff` enforces blank spaces for `import`, `class` and `function` better
* `ruff` will put the code back in a single line if it fits
* `ruff` always uses double quote when it can
There are some other details that you'll realize if you take a look at the diff. I think overall `ruff` generates slightly better code than `black` (and `ruff` is probably a bit more strict than `black`).
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
CI needs to pass because we removed the black dependency.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #54840 from gaogaotiantian/use-ruff-as-formatter.
Authored-by: Tian Gao <ga...@hotmail.com>
Signed-off-by: Hyukjin Kwon <gu...@apache.org>
---
.github/workflows/build_and_test.yml | 2 +-
.github/workflows/pages.yml | 2 +-
dev/create-release/spark-rm/Dockerfile | 2 +-
dev/gen-protos.sh | 2 +-
dev/is-changed.py | 2 +-
dev/lint-python | 15 +-
dev/reformat-python | 10 +-
dev/spark-test-image/connect-gen-protos/Dockerfile | 2 +-
dev/spark-test-image/docs/Dockerfile | 2 +-
dev/spark-test-image/lint/Dockerfile | 1 -
pyproject.toml | 1 +
python/conf_viztracer/daemon_viztracer.py | 5 +-
python/conf_vscode/sitecustomize.py | 5 +-
python/pyspark/accumulators.py | 4 +-
python/pyspark/core/context.py | 6 +-
python/pyspark/core/rdd.py | 3 +-
python/pyspark/errors/error_classes.py | 5 +-
python/pyspark/install.py | 2 +-
python/pyspark/ml/classification.py | 2 +-
python/pyspark/ml/clustering.py | 3 +-
python/pyspark/ml/connect/pipeline.py | 4 +-
python/pyspark/ml/connect/tuning.py | 8 +-
python/pyspark/ml/feature.py | 5 +-
python/pyspark/ml/functions.py | 4 +-
python/pyspark/ml/linalg/__init__.py | 23 +-
python/pyspark/ml/torch/distributor.py | 6 +-
python/pyspark/ml/tree.py | 2 +-
python/pyspark/mllib/classification.py | 5 +-
python/pyspark/mllib/clustering.py | 6 +-
python/pyspark/mllib/linalg/__init__.py | 12 +-
python/pyspark/mllib/linalg/distributed.py | 3 +-
python/pyspark/pandas/correlation.py | 16 +-
python/pyspark/pandas/exceptions.py | 5 +-
python/pyspark/pandas/frame.py | 71 +++--
python/pyspark/pandas/generic.py | 10 +-
python/pyspark/pandas/indexes/base.py | 9 +-
python/pyspark/pandas/indexes/multi.py | 3 +-
python/pyspark/pandas/indexing.py | 41 ++-
python/pyspark/pandas/internal.py | 18 +-
python/pyspark/pandas/namespace.py | 6 +-
python/pyspark/pandas/plot/core.py | 11 +-
python/pyspark/pandas/plot/matplotlib.py | 2 +-
python/pyspark/pandas/series.py | 30 +-
python/pyspark/pandas/sql_formatter.py | 3 +-
.../pyspark/pandas/tests/frame/test_reshaping.py | 10 +-
python/pyspark/pandas/tests/indexes/test_drop.py | 6 +-
.../pandas/tests/indexes/test_indexing_loc_2d.py | 21 +-
python/pyspark/pandas/tests/io/test_csv.py | 7 +-
python/pyspark/pandas/tests/series/test_compute.py | 6 +-
python/pyspark/pandas/tests/series/test_index.py | 6 +-
python/pyspark/pandas/tests/test_scalars.py | 5 +-
python/pyspark/pandas/typedef/typehints.py | 3 +-
python/pyspark/pandas/utils.py | 43 +--
python/pyspark/pipelines/cli.py | 6 +-
python/pyspark/pipelines/tests/test_cli.py | 18 +-
python/pyspark/shell.py | 7 +-
python/pyspark/shuffle.py | 2 +-
python/pyspark/sql/classic/dataframe.py | 6 +-
python/pyspark/sql/connect/dataframe.py | 16 +-
python/pyspark/sql/connect/functions/builtin.py | 8 +-
python/pyspark/sql/connect/merge.py | 3 +-
python/pyspark/sql/connect/plan.py | 19 +-
python/pyspark/sql/connect/proto/base_pb2.py | 31 +-
python/pyspark/sql/connect/proto/base_pb2.pyi | 203 ++++--------
python/pyspark/sql/connect/proto/catalog_pb2.py | 7 +-
python/pyspark/sql/connect/proto/catalog_pb2.pyi | 9 +-
python/pyspark/sql/connect/proto/commands_pb2.py | 13 +-
python/pyspark/sql/connect/proto/commands_pb2.pyi | 183 +++++------
python/pyspark/sql/connect/proto/common_pb2.py | 6 +-
python/pyspark/sql/connect/proto/common_pb2.pyi | 14 +-
.../sql/connect/proto/example_plugins_pb2.py | 7 +-
.../pyspark/sql/connect/proto/expressions_pb2.py | 7 +-
.../pyspark/sql/connect/proto/expressions_pb2.pyi | 59 +---
python/pyspark/sql/connect/proto/ml_common_pb2.py | 7 +-
python/pyspark/sql/connect/proto/ml_common_pb2.pyi | 11 +-
python/pyspark/sql/connect/proto/ml_pb2.py | 7 +-
python/pyspark/sql/connect/proto/ml_pb2.pyi | 19 +-
python/pyspark/sql/connect/proto/pipelines_pb2.py | 25 +-
python/pyspark/sql/connect/proto/pipelines_pb2.pyi | 59 ++--
python/pyspark/sql/connect/proto/relations_pb2.py | 7 +-
python/pyspark/sql/connect/proto/relations_pb2.pyi | 343 +++++++--------------
python/pyspark/sql/connect/proto/types_pb2.py | 6 +-
python/pyspark/sql/connect/proto/types_pb2.pyi | 13 +-
python/pyspark/sql/connect/readwriter.py | 6 +-
python/pyspark/sql/connect/session.py | 4 +-
.../streaming/worker/foreach_batch_worker.py | 2 +-
python/pyspark/sql/connect/udtf.py | 4 +-
python/pyspark/sql/context.py | 3 +-
python/pyspark/sql/conversion.py | 6 +-
python/pyspark/sql/datasource_internal.py | 9 +-
python/pyspark/sql/functions/builtin.py | 8 +-
python/pyspark/sql/internal.py | 3 +-
python/pyspark/sql/pandas/group_ops.py | 4 +-
python/pyspark/sql/pandas/serializers.py | 6 +-
python/pyspark/sql/pandas/typehints.py | 4 +-
python/pyspark/sql/plot/core.py | 6 +-
python/pyspark/sql/readwriter.py | 6 +-
python/pyspark/sql/session.py | 8 +-
python/pyspark/sql/streaming/list_state_client.py | 12 +-
python/pyspark/sql/streaming/listener.py | 9 +-
python/pyspark/sql/streaming/map_state_client.py | 2 +-
.../sql/streaming/proto/StateMessage_pb2.pyi | 12 +-
.../sql/streaming/stateful_processor_api_client.py | 28 +-
python/pyspark/sql/streaming/value_state_client.py | 10 +-
python/pyspark/sql/tests/arrow/test_arrow.py | 2 +-
.../sql/tests/arrow/test_arrow_grouped_map.py | 4 +-
python/pyspark/sql/tests/arrow/test_arrow_udtf.py | 87 +++---
.../sql/tests/connect/test_connect_collection.py | 6 +-
.../pyspark/sql/tests/connect/test_connect_stat.py | 2 +-
.../pyspark/sql/tests/connect/test_parity_udtf.py | 6 +-
.../helper/helper_pandas_transform_with_state.py | 36 +--
...t_pandas_transform_with_state_state_variable.py | 8 +-
.../sql/tests/pandas/test_pandas_grouped_map.py | 2 +-
.../tests/pandas/test_pandas_udf_grouped_agg.py | 6 +-
.../sql/tests/pandas/test_pandas_udf_scalar.py | 2 +-
python/pyspark/sql/tests/test_dataframe.py | 4 +-
python/pyspark/sql/tests/test_python_datasource.py | 4 +-
.../sql/tests/test_python_streaming_datasource.py | 12 +-
python/pyspark/sql/tests/test_udtf.py | 207 +++++++------
python/pyspark/sql/types.py | 6 +-
python/pyspark/sql/udf.py | 18 +-
python/pyspark/sql/worker/analyze_udtf.py | 36 ++-
python/pyspark/sql/worker/plan_data_source_read.py | 9 +-
.../pyspark/sql/worker/write_into_data_source.py | 7 +-
python/pyspark/streaming/dstream.py | 3 +-
python/pyspark/testing/utils.py | 20 +-
python/pyspark/tests/test_appsubmit.py | 7 +-
python/pyspark/tests/test_profiler.py | 3 +-
.../upstream/pyarrow/test_pyarrow_array_cast.py | 2 +-
python/pyspark/util.py | 24 +-
python/pyspark/worker.py | 8 +-
python/pyspark/worker_util.py | 6 +-
python/run-tests.py | 192 +++++++-----
python/test_coverage/sitecustomize.py | 12 +-
python/test_support/userlibrary.py | 1 -
135 files changed, 1146 insertions(+), 1355 deletions(-)
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
index 9fec6330ffd3..76f2cafb7819 100644
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -842,7 +842,7 @@ jobs:
python-version: '3.12'
- name: Install dependencies for Python CodeGen check
run: |
- python3.12 -m pip install 'black==26.3.1' 'protobuf==6.33.5' 'mypy==1.8.0' 'mypy-protobuf==3.3.0'
+ python3.12 -m pip install 'ruff==0.14.8' 'protobuf==6.33.5' 'mypy==1.8.0' 'mypy-protobuf==3.3.0'
python3.12 -m pip list
- name: Python CodeGen check for branch-3.5
if: inputs.branch == 'branch-3.5'
diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml
index a17da0ef5aca..6f21d8266a15 100644
--- a/.github/workflows/pages.yml
+++ b/.github/workflows/pages.yml
@@ -62,7 +62,7 @@ jobs:
run: |
pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' \
ipython ipython_genutils sphinx_plotly_directive 'numpy>=1.22' pyarrow 'pandas==2.3.3' 'plotly>=4.8' 'docutils<0.18.0' \
- 'flake8==3.9.0' 'mypy==1.8.0' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'black==26.3.1' \
+ 'flake8==3.9.0' 'mypy==1.8.0' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'ruff==0.14.8' \
'pandas-stubs==1.2.0.53' 'grpcio==1.76.0' 'grpcio-status==1.76.0' 'protobuf==6.33.5' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0' \
'sphinxcontrib-applehelp==1.0.4' 'sphinxcontrib-devhelp==1.0.2' 'sphinxcontrib-htmlhelp==2.0.1' 'sphinxcontrib-qthelp==1.0.3' 'sphinxcontrib-serializinghtml==1.1.5'
- name: Install Ruby for documentation generation
diff --git a/dev/create-release/spark-rm/Dockerfile b/dev/create-release/spark-rm/Dockerfile
index 63b27db3b95d..a4c4c9619333 100644
--- a/dev/create-release/spark-rm/Dockerfile
+++ b/dev/create-release/spark-rm/Dockerfile
@@ -61,7 +61,7 @@ RUN python3.10 -m pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13'
sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe 'pyzmq<24.0.0' \
ipython ipython_genutils sphinx_plotly_directive 'numpy>=1.22' pyarrow pandas \
'plotly>=4.8' 'docutils<0.18.0' 'flake8==3.9.0' 'mypy==1.19.1' 'pytest==7.1.3' \
- 'pytest-mypy-plugins==1.9.3' 'black==26.3.1' 'pandas-stubs==1.2.0.53' \
+ 'pytest-mypy-plugins==1.9.3' 'ruff==0.14.8' 'pandas-stubs==1.2.0.53' \
'grpcio==1.76.0' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0' \
'sphinxcontrib-applehelp==1.0.4' 'sphinxcontrib-devhelp==1.0.2' \
'sphinxcontrib-htmlhelp==2.0.1' 'sphinxcontrib-qthelp==1.0.3' \
diff --git a/dev/gen-protos.sh b/dev/gen-protos.sh
index 7952711c4289..b431ac3c6756 100755
--- a/dev/gen-protos.sh
+++ b/dev/gen-protos.sh
@@ -116,7 +116,7 @@ for f in `find gen/proto/python -name "*.py*"`; do
rm $f.bak
done
-black --config $SPARK_HOME/pyproject.toml gen/proto/python
+ruff format --config $SPARK_HOME/pyproject.toml gen/proto/python
# Last step copy the result files to the destination module.
for f in `find gen/proto/python -name "*.py*"`; do
diff --git a/dev/is-changed.py b/dev/is-changed.py
index 4c3adf691327..df7109c67837 100755
--- a/dev/is-changed.py
+++ b/dev/is-changed.py
@@ -43,7 +43,7 @@ def parse_opts():
"--modules",
type=str,
default=default_value,
- help="A comma-separated list of modules to test " "(default: %s)" % default_value,
+ help="A comma-separated list of modules to test (default: %s)" % default_value,
)
args, unknown = parser.parse_known_args()
diff --git a/dev/lint-python b/dev/lint-python
index a4ee6752d575..bf5a633ba9d0 100755
--- a/dev/lint-python
+++ b/dev/lint-python
@@ -74,7 +74,7 @@ while (( "$#" )); do
shift
done
-if [[ -z "$COMPILE_TEST$BLACK_TEST$PYSPARK_CUSTOM_ERRORS_CHECK_TEST$FLAKE8_TEST$RUFF_TEST$MYPY_TEST$MYPY_EXAMPLES_TEST$MYPY_DATA_TEST" ]]; then
+if [[ -z "$COMPILE_TEST$PYSPARK_CUSTOM_ERRORS_CHECK_TEST$FLAKE8_TEST$RUFF_TEST$MYPY_TEST$MYPY_EXAMPLES_TEST$MYPY_DATA_TEST" ]]; then
COMPILE_TEST=true
BLACK_TEST=true
PYSPARK_CUSTOM_ERRORS_CHECK_TEST=true
@@ -315,6 +315,19 @@ ruff checks failed."
echo
fi
+ RUFF_REPORT=$( ($RUFF_BUILD format --diff python/pyspark dev python/packaging python/benchmarks) 2>&1)
+ RUFF_STATUS=$?
+
+ if [ "$RUFF_STATUS" -ne 0 ]; then
+ echo "ruff format checks failed:"
+ echo "$RUFF_REPORT"
+ echo "$RUFF_STATUS"
+ exit "$RUFF_STATUS"
+ else
+ echo "ruff format checks passed."
+ echo
+ fi
+
}
function black_test {
diff --git a/dev/reformat-python b/dev/reformat-python
index a8e6c25817ac..e7686f0b85ee 100755
--- a/dev/reformat-python
+++ b/dev/reformat-python
@@ -21,12 +21,12 @@ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
FWDIR="$( cd "$DIR"/.. && pwd )"
cd "$FWDIR"
-BLACK_BUILD="${PYTHON_EXECUTABLE} -m black"
-BLACK_VERSION="26.3.1"
-$PYTHON_EXECUTABLE -c 'import black' 2> /dev/null
+RUFF_BUILD="${PYTHON_EXECUTABLE} -m ruff"
+RUFF_VERSION="0.14.8"
+$PYTHON_EXECUTABLE -c 'import ruff' 2> /dev/null
if [ $? -ne 0 ]; then
- echo "The Python library providing the 'black' module was not found. Please install Black, for example, via 'pip install black==$BLACK_VERSION'."
+ echo "The Python library providing the 'ruff' module was not found. Please install Ruff, for example, via 'pip install ruff==$RUFF_VERSION'."
exit 1
fi
-$BLACK_BUILD python/pyspark dev python/packaging python/benchmarks
+$RUFF_BUILD format python/pyspark dev python/packaging python/benchmarks
diff --git a/dev/spark-test-image/connect-gen-protos/Dockerfile b/dev/spark-test-image/connect-gen-protos/Dockerfile
index 7d1ca14c4762..ddd550082769 100644
--- a/dev/spark-test-image/connect-gen-protos/Dockerfile
+++ b/dev/spark-test-image/connect-gen-protos/Dockerfile
@@ -53,7 +53,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
RUN python3.12 -m pip install \
'mypy==1.19.1' \
'mypy-protobuf==3.3.0' \
- 'black==26.3.1'
+ 'ruff==0.14.8'
# Mount the Spark repo at /spark
WORKDIR /spark
diff --git a/dev/spark-test-image/docs/Dockerfile b/dev/spark-test-image/docs/Dockerfile
index f3778a342eb5..5009023fe677 100644
--- a/dev/spark-test-image/docs/Dockerfile
+++ b/dev/spark-test-image/docs/Dockerfile
@@ -89,7 +89,7 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
# See 'docutils<0.18.0' in SPARK-39421
RUN python3.12 -m pip install 'sphinx==4.5.0' mkdocs 'pydata_sphinx_theme>=0.13' sphinx-copybutton nbsphinx numpydoc jinja2 markupsafe \
ipython ipython_genutils sphinx_plotly_directive 'numpy>=1.22' 'pyarrow>=23.0.0' 'pandas==2.3.3' 'plotly>=4.8' 'docutils<0.18.0' \
- 'flake8==3.9.0' 'mypy==1.19.1' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'black==26.3.1' \
+ 'flake8==3.9.0' 'mypy==1.19.1' 'pytest==7.1.3' 'pytest-mypy-plugins==1.9.3' 'ruff==0.14.8' \
'pandas-stubs==1.2.0.53' 'grpcio==1.76.0' 'grpcio-status==1.76.0' 'protobuf==6.33.5' 'grpc-stubs==1.24.11' 'googleapis-common-protos-stubs==2.2.0' \
'sphinxcontrib-applehelp==1.0.4' 'sphinxcontrib-devhelp==1.0.2' 'sphinxcontrib-htmlhelp==2.0.1' 'sphinxcontrib-qthelp==1.0.3' 'sphinxcontrib-serializinghtml==1.1.5' \
&& python3.12 -m pip cache purge
diff --git a/dev/spark-test-image/lint/Dockerfile b/dev/spark-test-image/lint/Dockerfile
index 12f051832ad5..45eaeed0a76b 100644
--- a/dev/spark-test-image/lint/Dockerfile
+++ b/dev/spark-test-image/lint/Dockerfile
@@ -77,7 +77,6 @@ RUN python3.12 -m venv $VIRTUAL_ENV
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
RUN python3.12 -m pip install \
- 'black==26.3.1' \
'flake8==3.9.0' \
'ruff==0.14.8' \
'googleapis-common-protos-stubs==2.2.0' \
diff --git a/pyproject.toml b/pyproject.toml
index ca90a0319b9f..67439bbe4910 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -35,6 +35,7 @@ exclude = [
"*python/pyspark/sql/streaming/proto/*",
"*venv*/*",
]
+line-length = 100
[tool.ruff.lint]
extend-select = [
diff --git a/python/conf_viztracer/daemon_viztracer.py b/python/conf_viztracer/daemon_viztracer.py
index 2897c10bfe70..216f6368080d 100644
--- a/python/conf_viztracer/daemon_viztracer.py
+++ b/python/conf_viztracer/daemon_viztracer.py
@@ -25,13 +25,13 @@ import pyspark.worker
def viztracer_wrapper(func):
-
def wrapper(*args, **kwargs):
result = func(*args, **kwargs)
tracer = viztracer.get_tracer()
if tracer is not None:
tracer.exit_routine()
return result
+
return wrapper
@@ -43,6 +43,5 @@ if __name__ == "__main__":
else:
output_dir = "./"
- sys.argv[:] = ["viztracer", "-m", "pyspark.daemon", "--quiet", "-u",
- "--output_dir", output_dir]
+ sys.argv[:] = ["viztracer", "-m", "pyspark.daemon", "--quiet", "-u", "--output_dir", output_dir]
main()
diff --git a/python/conf_vscode/sitecustomize.py b/python/conf_vscode/sitecustomize.py
index a8bdda6c7db3..486d580efdc1 100644
--- a/python/conf_vscode/sitecustomize.py
+++ b/python/conf_vscode/sitecustomize.py
@@ -19,9 +19,8 @@ try:
import os
import sys
- if (
- "DEBUGPY_ADAPTER_ENDPOINTS" in os.environ and
- not any("debugpy" in arg for arg in sys.orig_argv)
+ if "DEBUGPY_ADAPTER_ENDPOINTS" in os.environ and not any(
+ "debugpy" in arg for arg in sys.orig_argv
):
def install_debugpy():
diff --git a/python/pyspark/accumulators.py b/python/pyspark/accumulators.py
index 7584a840a1c7..66c75e022e61 100644
--- a/python/pyspark/accumulators.py
+++ b/python/pyspark/accumulators.py
@@ -383,9 +383,7 @@ def _start_update_server(
os.remove(socket_path)
server = AccumulatorUnixServer(socket_path, UpdateRequestHandler)
else:
- server = AccumulatorTCPServer(
- ("localhost", 0), UpdateRequestHandler, auth_token
- ) # type: ignore[assignment]
+ server = AccumulatorTCPServer(("localhost", 0), UpdateRequestHandler, auth_token) # type: ignore[assignment]
thread = threading.Thread(target=server.serve_forever)
thread.daemon = True
diff --git a/python/pyspark/core/context.py b/python/pyspark/core/context.py
index 63260f8f3cd9..8bdea00dad4e 100644
--- a/python/pyspark/core/context.py
+++ b/python/pyspark/core/context.py
@@ -411,7 +411,8 @@ class SparkContext:
# see http://stackoverflow.com/questions/23206787/
if isinstance(
- threading.current_thread(), threading._MainThread # type: ignore[attr-defined]
+ threading.current_thread(),
+ threading._MainThread, # type: ignore[attr-defined]
):
signal.signal(signal.SIGINT, signal_handler)
@@ -857,7 +858,8 @@ class SparkContext:
if "__len__" not in dir(c):
c = list(c) # Make it a list so we can compute its length
batchSize = max(
- 1, min(len(c) // numSlices, self._batchSize or 1024) # type: ignore[arg-type]
+ 1,
+ min(len(c) // numSlices, self._batchSize or 1024), # type: ignore[arg-type]
)
serializer = BatchedSerializer(self._unbatched_serializer, batchSize)
diff --git a/python/pyspark/core/rdd.py b/python/pyspark/core/rdd.py
index 18e0d286fc7b..8ec147cd9049 100644
--- a/python/pyspark/core/rdd.py
+++ b/python/pyspark/core/rdd.py
@@ -1731,8 +1731,7 @@ class RDD(Generic[T_co]):
:meth:`SparkContext.setJobGroup`
"""
warnings.warn(
- "Deprecated in 3.1, Use pyspark.InheritableThread with "
- "the pinned thread mode enabled.",
+ "Deprecated in 3.1, Use pyspark.InheritableThread with the pinned thread mode enabled.",
FutureWarning,
)
diff --git a/python/pyspark/errors/error_classes.py b/python/pyspark/errors/error_classes.py
index 30869a3fbb2d..12094d61336e 100644
--- a/python/pyspark/errors/error_classes.py
+++ b/python/pyspark/errors/error_classes.py
@@ -23,9 +23,6 @@ import importlib.resources
# For more information, please see: https://issues.apache.org/jira/browse/SPARK-46810
# This discrepancy will be resolved as part of: https://issues.apache.org/jira/browse/SPARK-47429
ERROR_CLASSES_JSON = (
- importlib.resources
- .files("pyspark.errors")
- .joinpath("error-conditions.json")
- .read_text()
+ importlib.resources.files("pyspark.errors").joinpath("error-conditions.json").read_text()
)
ERROR_CLASSES_MAP = json.loads(ERROR_CLASSES_JSON)
diff --git a/python/pyspark/install.py b/python/pyspark/install.py
index c68745853200..c0e13478b938 100644
--- a/python/pyspark/install.py
+++ b/python/pyspark/install.py
@@ -64,7 +64,7 @@ def checked_versions(spark_version, hadoop_version, hive_version):
spark_version = "spark-%s" % spark_version
if not spark_version.startswith("spark-"):
raise RuntimeError(
- "Spark version should start with 'spark-' prefix; however, " "got %s" % spark_version
+ "Spark version should start with 'spark-' prefix; however, got %s" % spark_version
)
if hadoop_version == "without":
diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py
index de279ab03aa9..f69ecf115f5a 100644
--- a/python/pyspark/ml/classification.py
+++ b/python/pyspark/ml/classification.py
@@ -3565,7 +3565,7 @@ class OneVsRest(
weightCol = self.getWeightCol()
else:
warnings.warn(
- "weightCol is ignored, " "as it is not supported by {} now.".format(classifier)
+ "weightCol is ignored, as it is not supported by {} now.".format(classifier)
)
if weightCol:
diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py
index 740ccd6b546b..c3964724dd3a 100644
--- a/python/pyspark/ml/clustering.py
+++ b/python/pyspark/ml/clustering.py
@@ -1270,8 +1270,7 @@ class _LDAParams(HasMaxIter, HasFeaturesCol, HasSeed, HasCheckpointInterval):
optimizer: Param[str] = Param(
Params._dummy(),
"optimizer",
- "Optimizer or inference algorithm used to estimate the LDA model. "
- "Supported: online, em",
+ "Optimizer or inference algorithm used to estimate the LDA model. Supported: online, em",
typeConverter=TypeConverters.toString,
)
learningOffset: Param[float] = Param(
diff --git a/python/pyspark/ml/connect/pipeline.py b/python/pyspark/ml/connect/pipeline.py
index afc0fa78a9db..55e57b88da04 100644
--- a/python/pyspark/ml/connect/pipeline.py
+++ b/python/pyspark/ml/connect/pipeline.py
@@ -132,9 +132,7 @@ class Pipeline(Estimator["PipelineModel"], _PipelineReadWrite):
>>> loaded_pipeline_model = PipelineModel.loadFromLocal("/tmp/pipeline")
"""
- stages: Param[List[Params]] = Param(
- Params._dummy(), "stages", "a list of pipeline stages"
- ) # type: ignore[assignment]
+ stages: Param[List[Params]] = Param(Params._dummy(), "stages", "a list of pipeline stages") # type: ignore[assignment]
_input_kwargs: Dict[str, Any]
diff --git a/python/pyspark/ml/connect/tuning.py b/python/pyspark/ml/connect/tuning.py
index ac393a3daf1d..9d6a008c99e2 100644
--- a/python/pyspark/ml/connect/tuning.py
+++ b/python/pyspark/ml/connect/tuning.py
@@ -203,14 +203,10 @@ class _CrossValidatorReadWrite(MetaAlgorithmReadWrite):
def _save_meta_algorithm(self, root_path: str, node_path: List[str]) -> Dict[str, Any]:
metadata = self._get_metadata_to_save()
- metadata[
- "estimator"
- ] = self.getEstimator()._save_to_node_path( # type: ignore[attr-defined]
+ metadata["estimator"] = self.getEstimator()._save_to_node_path( # type: ignore[attr-defined]
root_path, node_path + ["crossvalidator_estimator"]
)
- metadata[
- "evaluator"
- ] = self.getEvaluator()._save_to_node_path( # type: ignore[attr-defined]
+ metadata["evaluator"] = self.getEvaluator()._save_to_node_path( # type: ignore[attr-defined]
root_path, node_path + ["crossvalidator_evaluator"]
)
metadata["estimator_param_maps"] = [
diff --git a/python/pyspark/ml/feature.py b/python/pyspark/ml/feature.py
index ce008bc2661a..4024c0c07f4f 100755
--- a/python/pyspark/ml/feature.py
+++ b/python/pyspark/ml/feature.py
@@ -1961,8 +1961,7 @@ class _ImputerParams(HasInputCol, HasInputCols, HasOutputCol, HasOutputCols, Has
missingValue: Param[float] = Param(
Params._dummy(),
"missingValue",
- "The placeholder for the missing values. All occurrences of missingValue "
- "will be imputed.",
+ "The placeholder for the missing values. All occurrences of missingValue will be imputed.",
typeConverter=TypeConverters.toFloat,
)
@@ -3824,7 +3823,7 @@ class _RobustScalerParams(HasInputCol, HasOutputCol, HasRelativeError):
withScaling: Param[bool] = Param(
Params._dummy(),
"withScaling",
- "Whether to scale the data to " "quantile range",
+ "Whether to scale the data to quantile range",
typeConverter=TypeConverters.toBoolean,
)
diff --git a/python/pyspark/ml/functions.py b/python/pyspark/ml/functions.py
index 34edc32ecbcc..b3cd3978df44 100644
--- a/python/pyspark/ml/functions.py
+++ b/python/pyspark/ml/functions.py
@@ -826,9 +826,7 @@ def predict_batch_udf(
raise ValueError(msg.format(num_expected_cols, num_input_cols))
# return transformed predictions to Spark
- yield _validate_and_transform_prediction_result(
- preds, num_input_rows, return_type
- ) # type: ignore
+ yield _validate_and_transform_prediction_result(preds, num_input_rows, return_type) # type: ignore
return pandas_udf(predict, return_type) # type: ignore[call-overload]
diff --git a/python/pyspark/ml/linalg/__init__.py b/python/pyspark/ml/linalg/__init__.py
index 508df369dbde..da6911363b29 100644
--- a/python/pyspark/ml/linalg/__init__.py
+++ b/python/pyspark/ml/linalg/__init__.py
@@ -197,9 +197,9 @@ class VectorUDT(UserDefinedType):
def deserialize(
self, datum: Tuple[int, Optional[int], Optional[List[int]], List[float]]
) -> "Vector":
- assert (
- len(datum) == 4
- ), "VectorUDT.deserialize given row with length %d but requires 4" % len(datum)
+ assert len(datum) == 4, (
+ "VectorUDT.deserialize given row with length %d but requires 4" % len(datum)
+ )
tpe = datum[0]
if tpe == 0:
return SparseVector(cast(int, datum[1]), cast(List[int], datum[2]), datum[3])
@@ -265,9 +265,9 @@ class MatrixUDT(UserDefinedType):
self,
datum: Tuple[int, int, int, Optional[List[int]], Optional[List[int]], List[float], bool],
) -> "Matrix":
- assert (
- len(datum) == 7
- ), "MatrixUDT.deserialize given row with length %d but requires 7" % len(datum)
+ assert len(datum) == 7, (
+ "MatrixUDT.deserialize given row with length %d but requires 7" % len(datum)
+ )
tpe = datum[0]
if tpe == 0:
return SparseMatrix(*datum[1:]) # type: ignore[arg-type]
@@ -625,11 +625,12 @@ class SparseVector(Vector):
)
if self.indices.size > 0:
- assert (
- np.max(self.indices) < self.size
- ), "Index %d is out of the size of vector with size=%d" % (
- np.max(self.indices),
- self.size,
+ assert np.max(self.indices) < self.size, (
+ "Index %d is out of the size of vector with size=%d"
+ % (
+ np.max(self.indices),
+ self.size,
+ )
)
assert np.min(self.indices) >= 0, "Contains negative index %d" % (np.min(self.indices))
diff --git a/python/pyspark/ml/torch/distributor.py b/python/pyspark/ml/torch/distributor.py
index d0eb250d3913..dc0eda962312 100644
--- a/python/pyspark/ml/torch/distributor.py
+++ b/python/pyspark/ml/torch/distributor.py
@@ -276,7 +276,8 @@ class Distributor:
)
)
return
- raise RuntimeError(textwrap.dedent(f"""
+ raise RuntimeError(
+ textwrap.dedent(f"""
This cluster has TLS encryption enabled;
however, {name} does not support
data encryption in transit. To override
@@ -285,7 +286,8 @@ class Distributor:
to 'true' in the Spark configuration. Please note this
will cause model parameters and possibly training
data to be sent between nodes unencrypted.
- """))
+ """)
+ )
class TorchDistributor(Distributor):
diff --git a/python/pyspark/ml/tree.py b/python/pyspark/ml/tree.py
index 86e8422702b1..63f58272aeef 100644
--- a/python/pyspark/ml/tree.py
+++ b/python/pyspark/ml/tree.py
@@ -317,7 +317,7 @@ class _RandomForestParams(_TreeEnsembleParams):
bootstrap: Param[bool] = Param(
Params._dummy(),
"bootstrap",
- "Whether bootstrap samples are used " "when building trees.",
+ "Whether bootstrap samples are used when building trees.",
typeConverter=TypeConverters.toBoolean,
)
diff --git a/python/pyspark/mllib/classification.py b/python/pyspark/mllib/classification.py
index 3d4350d7c3bf..c8b66947e07c 100644
--- a/python/pyspark/mllib/classification.py
+++ b/python/pyspark/mllib/classification.py
@@ -935,7 +935,10 @@ class StreamingLogisticRegressionWithSGD(StreamingLinearAlgorithm):
# LogisticRegressionWithSGD does only binary classification.
self._model = LogisticRegressionModel(
- initialWeights, 0, initialWeights.size, 2 # type: ignore[attr-defined]
+ initialWeights,
+ 0,
+ initialWeights.size, # type: ignore[attr-defined]
+ 2,
)
return self
diff --git a/python/pyspark/mllib/clustering.py b/python/pyspark/mllib/clustering.py
index e3653635f3a6..08d32da2efa7 100644
--- a/python/pyspark/mllib/clustering.py
+++ b/python/pyspark/mllib/clustering.py
@@ -1020,12 +1020,10 @@ class StreamingKMeans:
def _validate(self, dstream: Any) -> None:
if self._model is None:
raise ValueError(
- "Initial centers should be set either by setInitialCenters " "or setRandomCenters."
+ "Initial centers should be set either by setInitialCenters or setRandomCenters."
)
if not isinstance(dstream, DStream):
- raise TypeError(
- "Expected dstream to be of type DStream, " "got type %s" % type(dstream)
- )
+ raise TypeError("Expected dstream to be of type DStream, got type %s" % type(dstream))
@since("1.5.0")
def setK(self, k: int) -> "StreamingKMeans":
diff --git a/python/pyspark/mllib/linalg/__init__.py b/python/pyspark/mllib/linalg/__init__.py
index a32ddc1b84f5..7092fc97dd3f 100644
--- a/python/pyspark/mllib/linalg/__init__.py
+++ b/python/pyspark/mllib/linalg/__init__.py
@@ -207,9 +207,9 @@ class VectorUDT(UserDefinedType):
def deserialize(
self, datum: Tuple[int, Optional[int], Optional[List[int]], List[float]]
) -> "Vector":
- assert (
- len(datum) == 4
- ), "VectorUDT.deserialize given row with length %d but requires 4" % len(datum)
+ assert len(datum) == 4, (
+ "VectorUDT.deserialize given row with length %d but requires 4" % len(datum)
+ )
tpe = datum[0]
if tpe == 0:
return SparseVector(cast(int, datum[1]), cast(List[int], datum[2]), datum[3])
@@ -275,9 +275,9 @@ class MatrixUDT(UserDefinedType):
self,
datum: Tuple[int, int, int, Optional[List[int]], Optional[List[int]], List[float], bool],
) -> "Matrix":
- assert (
- len(datum) == 7
- ), "MatrixUDT.deserialize given row with length %d but requires 7" % len(datum)
+ assert len(datum) == 7, (
+ "MatrixUDT.deserialize given row with length %d but requires 7" % len(datum)
+ )
tpe = datum[0]
if tpe == 0:
return SparseMatrix(
diff --git a/python/pyspark/mllib/linalg/distributed.py b/python/pyspark/mllib/linalg/distributed.py
index f5402bdbe96d..e5d27814602a 100644
--- a/python/pyspark/mllib/linalg/distributed.py
+++ b/python/pyspark/mllib/linalg/distributed.py
@@ -636,8 +636,7 @@ class IndexedRowMatrix(DistributedMatrix):
java_matrix = rows
else:
raise TypeError(
- "rows should be an RDD of IndexedRows or (int, vector) tuples, "
- "got %s" % type(rows)
+ "rows should be an RDD of IndexedRows or (int, vector) tuples, got %s" % type(rows)
)
self._java_matrix_wrapper = JavaModelWrapper(java_matrix)
diff --git a/python/pyspark/pandas/correlation.py b/python/pyspark/pandas/correlation.py
index dbbbc199d432..cf9d38961216 100644
--- a/python/pyspark/pandas/correlation.py
+++ b/python/pyspark/pandas/correlation.py
@@ -244,18 +244,14 @@ def compute(sdf: SparkDataFrame, groupKeys: List[str], method: str) -> SparkData
F.col(CORRELATION_KENDALL_P_COLUMN) - F.col(CORRELATION_KENDALL_Q_COLUMN)
) / F.sqrt(
(
- (
- F.col(CORRELATION_KENDALL_P_COLUMN)
- + F.col(CORRELATION_KENDALL_Q_COLUMN)
- + (F.col(CORRELATION_KENDALL_T_COLUMN))
- )
+ F.col(CORRELATION_KENDALL_P_COLUMN)
+ + F.col(CORRELATION_KENDALL_Q_COLUMN)
+ + (F.col(CORRELATION_KENDALL_T_COLUMN))
)
* (
- (
- F.col(CORRELATION_KENDALL_P_COLUMN)
- + F.col(CORRELATION_KENDALL_Q_COLUMN)
- + (F.col(CORRELATION_KENDALL_U_COLUMN))
- )
+ F.col(CORRELATION_KENDALL_P_COLUMN)
+ + F.col(CORRELATION_KENDALL_Q_COLUMN)
+ + (F.col(CORRELATION_KENDALL_U_COLUMN))
)
)
diff --git a/python/pyspark/pandas/exceptions.py b/python/pyspark/pandas/exceptions.py
index 83283c9a4afe..fce91deec2b2 100644
--- a/python/pyspark/pandas/exceptions.py
+++ b/python/pyspark/pandas/exceptions.py
@@ -91,8 +91,9 @@ class PandasNotImplementedError(NotImplementedError):
)
elif scalar_name is not None:
msg = (
- "The scalar `{0}.{1}` is not reimplemented in pyspark.pandas;"
- " use `pd.{1}`.".format(class_name, scalar_name)
+ "The scalar `{0}.{1}` is not reimplemented in pyspark.pandas; use `pd.{1}`.".format(
+ class_name, scalar_name
+ )
)
else:
if deprecated:
diff --git a/python/pyspark/pandas/frame.py b/python/pyspark/pandas/frame.py
index 9bf735cde981..9baaaac5970f 100644
--- a/python/pyspark/pandas/frame.py
+++ b/python/pyspark/pandas/frame.py
@@ -4093,7 +4093,8 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
]
return DataFrame(
psdf._internal.with_new_columns(
- data_spark_columns, column_labels=self._internal.column_labels # TODO: dtypes?
+ data_spark_columns,
+ column_labels=self._internal.column_labels, # TODO: dtypes?
)
)
@@ -5709,7 +5710,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
) or isinstance(v, MultiIndex)
if is_invalid_assignee:
raise TypeError(
- "Column assignment doesn't support type " "{0}".format(type(v).__name__)
+ "Column assignment doesn't support type {0}".format(type(v).__name__)
)
if callable(v):
kwargs[k] = v(self)
@@ -5718,7 +5719,9 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
(k if is_name_like_tuple(k) else (k,)): (
(v.spark.column, v._internal.data_fields[0])
if isinstance(v, IndexOpsMixin) and not isinstance(v, MultiIndex)
- else (v, None) if isinstance(v, PySparkColumn) else (F.lit(v), None)
+ else (v, None)
+ if isinstance(v, PySparkColumn)
+ else (F.lit(v), None)
)
for k, v in kwargs.items()
}
@@ -6253,7 +6256,11 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
method = None
return self._fillna_with_method(
- value=value, method=method, axis=axis, inplace=inplace, limit=limit # type: ignore[arg-type]
+ value=value,
+ method=method, # type: ignore[arg-type]
+ axis=axis,
+ inplace=inplace,
+ limit=limit,
)
def _fillna_with_method(
@@ -6501,7 +6508,9 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
def op(psser: ps.Series) -> ps.Series:
if psser.name in to_replace_dict:
return psser.replace(
- to_replace=to_replace_dict[psser.name], value=value, regex=regex # type: ignore[arg-type]
+ to_replace=to_replace_dict[psser.name],
+ value=value, # type: ignore[arg-type]
+ regex=regex,
)
else:
return psser
@@ -6517,7 +6526,9 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
def op(psser: ps.Series) -> ps.Series:
return psser.replace(
- to_replace=to_replace, value=value, regex=regex # type: ignore[arg-type]
+ to_replace=to_replace,
+ value=value, # type: ignore[arg-type]
+ regex=regex,
)
psdf = self._apply_series_op(op)
@@ -6897,13 +6908,12 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
)
):
raise TypeError(
- "aggfunc must be a dict mapping from column name "
- "to aggregate functions (string)."
+ "aggfunc must be a dict mapping from column name to aggregate functions (string)."
)
if isinstance(aggfunc, dict) and index is None:
raise NotImplementedError(
- "pivot_table doesn't support aggfunc" " as dict and without index."
+ "pivot_table doesn't support aggfunc as dict and without index."
)
if isinstance(values, list) and index is None:
raise NotImplementedError("values can't be a list without index.")
@@ -7413,7 +7423,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
exclude_list = list(exclude)
if not any((include_list, exclude_list)):
- raise ValueError("at least one of include or exclude must be " "nonempty")
+ raise ValueError("at least one of include or exclude must be nonempty")
# can't both include AND exclude!
if set(include_list).intersection(set(exclude_list)):
@@ -8051,7 +8061,8 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
by = self._internal.index_spark_columns
elif is_list_like(level):
by = [
- self._internal.index_spark_columns[lvl] for lvl in level # type: ignore[union-attr]
+ self._internal.index_spark_columns[lvl]
+ for lvl in level # type: ignore[union-attr]
]
else:
by = [self._internal.index_spark_columns[level]] # type: ignore[index]
@@ -9104,7 +9115,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
common = list(self.columns.intersection(right.columns))
if len(common) > 0 and not lsuffix and not rsuffix:
raise ValueError(
- "columns overlap but no suffix specified: " "{rename}".format(rename=common)
+ "columns overlap but no suffix specified: {rename}".format(rename=common)
)
need_set_index = False
@@ -9757,8 +9768,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
for col_name in dtype_dict.keys():
if col_name not in self.columns:
raise KeyError(
- "Only a column name can be used for the "
- "key in a dtype mappings argument."
+ "Only a column name can be used for the key in a dtype mappings argument."
)
for col_name, col in self.items():
if col_name in dtype_dict:
@@ -10457,8 +10467,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
if index is not None and not is_list_like(index):
raise TypeError(
- "Index must be called with a collection of some kind, "
- "%s was passed" % type(index)
+ "Index must be called with a collection of some kind, %s was passed" % type(index)
)
if columns is not None and not is_list_like(columns):
@@ -10486,9 +10495,9 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
) -> "DataFrame":
# When axis is index, we can mimic pandas by a right outer join.
nlevels = self._internal.index_level
- assert nlevels <= 1 or (
- isinstance(index, ps.MultiIndex) and nlevels == index.nlevels
- ), "MultiIndex DataFrame can only be reindexed with a similar pandas-on-Spark MultiIndex."
+ assert nlevels <= 1 or (isinstance(index, ps.MultiIndex) and nlevels == index.nlevels), (
+ "MultiIndex DataFrame can only be reindexed with a similar pandas-on-Spark MultiIndex."
+ )
index_columns = self._internal.index_spark_column_names
frame = self._internal.resolved_copy.spark_frame.drop(NATURAL_ORDER_COLUMN_NAME)
@@ -10797,7 +10806,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
id_vars = [idv if is_name_like_tuple(idv) else (idv,) for idv in id_vars]
else:
raise ValueError(
- "id_vars must be a list of tuples" " when columns are a MultiIndex"
+ "id_vars must be a list of tuples when columns are a MultiIndex"
)
elif is_name_like_value(id_vars):
id_vars = [(id_vars,)]
@@ -10812,8 +10821,9 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
]
if len(missing) != 0:
raise KeyError(
- "The following 'id_vars' are not present"
- " in the DataFrame: {}".format(missing)
+ "The following 'id_vars' are not present in the DataFrame: {}".format(
+ missing
+ )
)
else:
raise KeyError(
@@ -10830,7 +10840,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
]
else:
raise ValueError(
- "value_vars must be a list of tuples" " when columns are a MultiIndex"
+ "value_vars must be a list of tuples when columns are a MultiIndex"
)
elif is_name_like_value(value_vars):
value_vars = [(value_vars,)]
@@ -10845,8 +10855,9 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
]
if len(missing) != 0:
raise KeyError(
- "The following 'value_vars' are not present"
- " in the DataFrame: {}".format(missing)
+ "The following 'value_vars' are not present in the DataFrame: {}".format(
+ missing
+ )
)
else:
raise KeyError(
@@ -11858,9 +11869,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
Name: one, dtype: int64
"""
if sum(x is not None for x in (items, like, regex)) > 1:
- raise TypeError(
- "Keyword arguments `items`, `like`, or `regex` " "are mutually exclusive"
- )
+ raise TypeError("Keyword arguments `items`, `like`, or `regex` are mutually exclusive")
axis = validate_axis(axis, none_axis=1)
@@ -12086,8 +12095,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
return mapper_fn, dtype, spark_return_type
else:
raise ValueError(
- "`mapper` or `index` or `columns` should be "
- "either dict-like or function type."
+ "`mapper` or `index` or `columns` should be either dict-like or function type."
)
index_mapper_fn = None
@@ -12325,8 +12333,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
newnames = [v_callable(name) for name in curnames]
else:
raise ValueError(
- "`mapper` or `index` or `columns` should be "
- "either dict-like or function type."
+ "`mapper` or `index` or `columns` should be either dict-like or function type."
)
if len(newnames) != len(curnames):
diff --git a/python/pyspark/pandas/generic.py b/python/pyspark/pandas/generic.py
index 92f58b878eda..0488f3dc7971 100644
--- a/python/pyspark/pandas/generic.py
+++ b/python/pyspark/pandas/generic.py
@@ -486,7 +486,7 @@ class Frame(object, metaclass=ABCMeta):
if isinstance(func, tuple):
func, target = func
if target in kwargs:
- raise ValueError("%s is both the pipe target and a keyword " "argument" % target)
+ raise ValueError("%s is both the pipe target and a keyword argument" % target)
kwargs[target] = self
return func(*args, **kwargs)
else:
@@ -1099,9 +1099,7 @@ class Frame(object, metaclass=ABCMeta):
elif isinstance(self, ps.Series):
f = pd.Series.to_excel
else:
- raise TypeError(
- "Constructor expects DataFrame or Series; however, " "got [%s]" % (self,)
- )
+ raise TypeError("Constructor expects DataFrame or Series; however, got [%s]" % (self,))
return validate_arguments_and_invoke_function(
psdf._to_internal_pandas(), self.to_excel, f, args
)
@@ -1219,9 +1217,7 @@ class Frame(object, metaclass=ABCMeta):
elif isinstance(self, ps.Series):
f = pd.Series.to_hdf
else:
- raise TypeError(
- "Constructor expects DataFrame or Series; however, " "got [%s]" % (self,)
- )
+ raise TypeError("Constructor expects DataFrame or Series; however, got [%s]" % (self,))
return validate_arguments_and_invoke_function(
psdf._to_internal_pandas(), self.to_hdf, f, args
)
diff --git a/python/pyspark/pandas/indexes/base.py b/python/pyspark/pandas/indexes/base.py
index 4bed26bce8da..0618e249aa62 100644
--- a/python/pyspark/pandas/indexes/base.py
+++ b/python/pyspark/pandas/indexes/base.py
@@ -550,7 +550,8 @@ class Index(IndexOpsMixin):
"It should only be used if the resulting NumPy ndarray is expected to be small."
)
result = np.asarray(
- self._to_internal_pandas()._values, dtype=dtype # type: ignore[attr-defined, arg-type]
+ self._to_internal_pandas()._values, # type: ignore[attr-defined]
+ dtype=dtype, # type: ignore[arg-type]
)
if copy:
result = result.copy()
@@ -929,9 +930,7 @@ class Index(IndexOpsMixin):
else:
def struct_to_array(scol: Column) -> Column:
- field_names = result._internal.spark_type_for(
- scol
- ).fieldNames() # type: ignore[attr-defined]
+ field_names = result._internal.spark_type_for(scol).fieldNames() # type: ignore[attr-defined]
return F.array([scol[field] for field in field_names])
return result.spark.transform(struct_to_array)
@@ -1262,7 +1261,7 @@ class Index(IndexOpsMixin):
" %d is not a valid level number" % (level,)
)
elif level > 0:
- raise IndexError("Too many levels:" " Index has only 1 level, not %d" % (level + 1))
+ raise IndexError("Too many levels: Index has only 1 level, not %d" % (level + 1))
elif level != self.name:
raise KeyError(
"Requested level ({}) does not match index name ({})".format(level, self.name)
diff --git a/python/pyspark/pandas/indexes/multi.py b/python/pyspark/pandas/indexes/multi.py
index d9eb17baf7d5..0263aa214c92 100644
--- a/python/pyspark/pandas/indexes/multi.py
+++ b/python/pyspark/pandas/indexes/multi.py
@@ -1004,8 +1004,7 @@ class MultiIndex(Index):
if level < 0:
if (level + nlevels) < 0:
raise IndexError(
- "Too many levels: Index has only %d levels, "
- "not %d" % (nlevels, level + 1)
+ "Too many levels: Index has only %d levels, not %d" % (nlevels, level + 1)
)
level = level + nlevels
else:
diff --git a/python/pyspark/pandas/indexing.py b/python/pyspark/pandas/indexing.py
index ee73f7bbcdb8..8d510efb49b6 100644
--- a/python/pyspark/pandas/indexing.py
+++ b/python/pyspark/pandas/indexing.py
@@ -65,9 +65,9 @@ class IndexerLike:
from pyspark.pandas.frame import DataFrame
from pyspark.pandas.series import Series
- assert isinstance(
- psdf_or_psser, (DataFrame, Series)
- ), "unexpected argument type: {}".format(type(psdf_or_psser))
+ assert isinstance(psdf_or_psser, (DataFrame, Series)), (
+ "unexpected argument type: {}".format(type(psdf_or_psser))
+ )
self._psdf_or_psser = psdf_or_psser
@property
@@ -279,7 +279,9 @@ class LocIndexerLike(IndexerLike, metaclass=ABCMeta):
else:
return self._select_rows_else(rows_sel)
- def _select_cols(self, cols_sel: Any, missing_keys: Optional[List[Name]] = None) -> Tuple[
+ def _select_cols(
+ self, cols_sel: Any, missing_keys: Optional[List[Name]] = None
+ ) -> Tuple[
List[Label],
Optional[List[PySparkColumn]],
Optional[List[InternalField]],
@@ -395,7 +397,9 @@ class LocIndexerLike(IndexerLike, metaclass=ABCMeta):
pass
@abstractmethod
- def _select_cols_by_slice(self, cols_sel: slice, missing_keys: Optional[List[Name]]) -> Tuple[
+ def _select_cols_by_slice(
+ self, cols_sel: slice, missing_keys: Optional[List[Name]]
+ ) -> Tuple[
List[Label],
Optional[List[PySparkColumn]],
Optional[List[InternalField]],
@@ -419,7 +423,9 @@ class LocIndexerLike(IndexerLike, metaclass=ABCMeta):
pass
@abstractmethod
- def _select_cols_else(self, cols_sel: Any, missing_keys: Optional[List[Name]]) -> Tuple[
+ def _select_cols_else(
+ self, cols_sel: Any, missing_keys: Optional[List[Name]]
+ ) -> Tuple[
List[Label],
Optional[List[PySparkColumn]],
Optional[List[InternalField]],
@@ -667,7 +673,8 @@ class LocIndexerLike(IndexerLike, metaclass=ABCMeta):
)
internal = self._internal.with_new_spark_column(
- self._psdf_or_psser._column_label, scol # TODO: dtype?
+ self._psdf_or_psser._column_label,
+ scol, # TODO: dtype?
)
self._psdf_or_psser._update_internal_frame(internal, check_same_anchor=False)
else:
@@ -1312,7 +1319,9 @@ class LocIndexer(LocIndexerLike):
data_spark_columns = [cols_sel]
return column_labels, data_spark_columns, None, True, None
- def _select_cols_by_slice(self, cols_sel: slice, missing_keys: Optional[List[Name]]) -> Tuple[
+ def _select_cols_by_slice(
+ self, cols_sel: slice, missing_keys: Optional[List[Name]]
+ ) -> Tuple[
List[Label],
Optional[List[PySparkColumn]],
Optional[List[InternalField]],
@@ -1418,7 +1427,9 @@ class LocIndexer(LocIndexerLike):
return column_labels, data_spark_columns, data_fields, False, None
- def _select_cols_else(self, cols_sel: Any, missing_keys: Optional[List[Name]]) -> Tuple[
+ def _select_cols_else(
+ self, cols_sel: Any, missing_keys: Optional[List[Name]]
+ ) -> Tuple[
List[Label],
Optional[List[PySparkColumn]],
Optional[List[InternalField]],
@@ -1767,7 +1778,9 @@ class iLocIndexer(LocIndexerLike):
"listlike of integers, boolean array] types, got {}".format(cols_sel)
)
- def _select_cols_by_slice(self, cols_sel: slice, missing_keys: Optional[List[Name]]) -> Tuple[
+ def _select_cols_by_slice(
+ self, cols_sel: slice, missing_keys: Optional[List[Name]]
+ ) -> Tuple[
List[Label],
Optional[List[PySparkColumn]],
Optional[List[InternalField]],
@@ -1785,7 +1798,9 @@ class iLocIndexer(LocIndexerLike):
not_none = (
cols_sel.start
if cols_sel.start is not None
- else cols_sel.stop if cols_sel.stop is not None else cols_sel.step
+ else cols_sel.stop
+ if cols_sel.stop is not None
+ else cols_sel.step
)
raise TypeError(
"cannot do slice indexing with these indexers {} of {}".format(
@@ -1812,7 +1827,9 @@ class iLocIndexer(LocIndexerLike):
else:
raise TypeError("cannot perform reduce with flexible type")
- def _select_cols_else(self, cols_sel: Any, missing_keys: Optional[List[Name]]) -> Tuple[
+ def _select_cols_else(
+ self, cols_sel: Any, missing_keys: Optional[List[Name]]
+ ) -> Tuple[
List[Label],
Optional[List[PySparkColumn]],
Optional[List[InternalField]],
diff --git a/python/pyspark/pandas/internal.py b/python/pyspark/pandas/internal.py
index 9ba714dee2d8..d24402c46b68 100644
--- a/python/pyspark/pandas/internal.py
+++ b/python/pyspark/pandas/internal.py
@@ -674,9 +674,9 @@ class InternalFrame:
# index_spark_columns
- assert all(
- isinstance(index_scol, PySparkColumn) for index_scol in index_spark_columns
- ), index_spark_columns
+ assert all(isinstance(index_scol, PySparkColumn) for index_scol in index_spark_columns), (
+ index_spark_columns
+ )
self._index_spark_columns: List[PySparkColumn] = index_spark_columns
@@ -822,9 +822,9 @@ class InternalFrame:
len(index_spark_columns),
len(index_names),
)
- assert all(
- is_name_like_tuple(index_name, check_type=True) for index_name in index_names
- ), index_names
+ assert all(is_name_like_tuple(index_name, check_type=True) for index_name in index_names), (
+ index_names
+ )
self._index_names: List[Optional[Label]] = index_names
@@ -894,9 +894,9 @@ class InternalFrame:
AssertionError: '__index_level_0__' already exists...
"""
index_column = SPARK_DEFAULT_INDEX_NAME
- assert (
- index_column not in sdf.columns
- ), "'%s' already exists in the Spark column names '%s'" % (index_column, sdf.columns)
+ assert index_column not in sdf.columns, (
+ "'%s' already exists in the Spark column names '%s'" % (index_column, sdf.columns)
+ )
if default_index_type is None:
default_index_type = ps.get_option("compute.default_index_type")
diff --git a/python/pyspark/pandas/namespace.py b/python/pyspark/pandas/namespace.py
index 42259ecf00e6..4c22c09219c8 100644
--- a/python/pyspark/pandas/namespace.py
+++ b/python/pyspark/pandas/namespace.py
@@ -437,7 +437,7 @@ def read_csv(
)
if len(missing) > 0:
raise ValueError(
- "Usecols do not match columns, columns expected but not " "found: %s" % missing
+ "Usecols do not match columns, columns expected but not found: %s" % missing
)
if len(column_labels) > 0:
@@ -2284,7 +2284,9 @@ def get_dummies(
(
str(label[len(columns) :])
if len(label) > len(columns) + 1
- else label[len(columns)] if len(label) == len(columns) + 1 else ""
+ else label[len(columns)]
+ if len(label) == len(columns) + 1
+ else ""
)
for label in column_labels
]
diff --git a/python/pyspark/pandas/plot/core.py b/python/pyspark/pandas/plot/core.py
index 1d296a383f19..c9998ffddfd8 100644
--- a/python/pyspark/pandas/plot/core.py
+++ b/python/pyspark/pandas/plot/core.py
@@ -145,7 +145,7 @@ class NumericPlotBase:
# no empty frames or series allowed
if len(numeric_data.columns) == 0:
raise TypeError(
- "Empty {0!r}: no numeric data to " "plot".format(numeric_data.__class__.__name__)
+ "Empty {0!r}: no numeric data to plot".format(numeric_data.__class__.__name__)
)
return data, numeric_data
@@ -420,9 +420,9 @@ class KdePlotBase(NumericPlotBase):
@staticmethod
def compute_kde_col(input_col, bw_method=None, ind=None):
# refers to org.apache.spark.mllib.stat.KernelDensity
- assert bw_method is not None and isinstance(
- bw_method, (int, float)
- ), "'bw_method' must be set as a scalar number."
+ assert bw_method is not None and isinstance(bw_method, (int, float)), (
+ "'bw_method' must be set as a scalar number."
+ )
assert ind is not None, "'ind' must be a scalar array."
@@ -542,8 +542,7 @@ class PandasOnSparkPlotAccessor(PandasObject):
from pyspark.pandas.plot import plotly as module
except ImportError:
raise ImportError(
- "plotly is required for plotting when the "
- "default backend 'plotly' is selected."
+ "plotly is required for plotting when the default backend 'plotly' is selected."
) from None
PandasOnSparkPlotAccessor._backends["plotly"] = module
diff --git a/python/pyspark/pandas/plot/matplotlib.py b/python/pyspark/pandas/plot/matplotlib.py
index f68d1f3efdda..d0fcf65117cc 100644
--- a/python/pyspark/pandas/plot/matplotlib.py
+++ b/python/pyspark/pandas/plot/matplotlib.py
@@ -203,7 +203,7 @@ class PandasOnSparkBoxPlot(PandasBoxPlot, BoxPlotBase):
for stats, ci in zip(bxpstats, conf_intervals):
if ci is not None:
if len(ci) != 2:
- raise ValueError("each confidence interval must " "have two values")
+ raise ValueError("each confidence interval must have two values")
else:
if ci[0] is not None:
stats["cilo"] = ci[0]
diff --git a/python/pyspark/pandas/series.py b/python/pyspark/pandas/series.py
index 8f3d3f821c03..22571219914b 100644
--- a/python/pyspark/pandas/series.py
+++ b/python/pyspark/pandas/series.py
@@ -1050,7 +1050,7 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
rmask = self < right
else:
raise ValueError(
- "Inclusive has to be either string of 'both'," "'left', 'right', or 'neither'."
+ "Inclusive has to be either string of 'both','left', 'right', or 'neither'."
)
return lmask & rmask
@@ -2206,7 +2206,11 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
method = None
return self._fillna_with_method(
- value=value, method=method, axis=axis, inplace=inplace, limit=limit # type: ignore[arg-type]
+ value=value,
+ method=method, # type: ignore[arg-type]
+ axis=axis,
+ inplace=inplace,
+ limit=limit,
)
def _fillna_with_method(
@@ -2290,7 +2294,8 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
return DataFrame(
self._psdf._internal.with_new_spark_column(
- self._column_label, scol.alias(name_like_string(self.name)) # TODO: dtype?
+ self._column_label,
+ scol.alias(name_like_string(self.name)), # TODO: dtype?
)
)._psser_for(self._column_label)
@@ -5182,9 +5187,9 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
elif not isinstance(regex, bool):
raise NotImplementedError("'regex' of %s type is not supported" % type(regex).__name__)
elif regex is True:
- assert isinstance(
- to_replace, str
- ), "If 'regex' is True then 'to_replace' must be a string"
+ assert isinstance(to_replace, str), (
+ "If 'regex' is True then 'to_replace' must be a string"
+ )
if to_replace is None:
if LooseVersion(pd.__version__) < "3.0.0":
@@ -5354,7 +5359,8 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
.alias(self._psdf._internal.spark_column_name_for(self._column_label))
)
internal = self._psdf._internal.with_new_spark_column(
- self._column_label, scol # TODO: dtype?
+ self._column_label,
+ scol, # TODO: dtype?
)
self._update_internal_frame(internal)
else:
@@ -5370,7 +5376,8 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
)
internal = combined["this"]._internal.with_new_spark_column(
- self._column_label, scol # TODO: dtype?
+ self._column_label,
+ scol, # TODO: dtype?
)
self._update_internal_frame(internal.resolved_copy, check_same_anchor=False)
@@ -5750,7 +5757,9 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
*index_scols, cond.alias(self._internal.data_spark_column_names[0])
).distinct()
internal = self._internal.with_new_sdf(
- sdf, index_fields=combined._internal.index_fields, data_fields=[None] # TODO: dtype?
+ sdf,
+ index_fields=combined._internal.index_fields,
+ data_fields=[None], # TODO: dtype?
)
return first_series(DataFrame(internal))
@@ -7443,8 +7452,7 @@ class Series(Frame, IndexOpsMixin, Generic[T]):
)
else:
footer = (
- "\nName: {name}, dtype: {dtype}"
- "\nShowing only the first {length}".format(
+ "\nName: {name}, dtype: {dtype}\nShowing only the first {length}".format(
length=length, name=self.name, dtype=pprint_thing(dtype_name)
)
)
diff --git a/python/pyspark/pandas/sql_formatter.py b/python/pyspark/pandas/sql_formatter.py
index d78bff17f9ae..b6e383fc4cea 100644
--- a/python/pyspark/pandas/sql_formatter.py
+++ b/python/pyspark/pandas/sql_formatter.py
@@ -192,8 +192,7 @@ def sql(
from pyspark.pandas import sql_processor
warnings.warn(
- "Deprecated in 3.3.0, and the legacy behavior "
- "will be removed in the future releases.",
+ "Deprecated in 3.3.0, and the legacy behavior will be removed in the future releases.",
FutureWarning,
)
return sql_processor.sql(query, index_col=index_col, **kwargs)
diff --git a/python/pyspark/pandas/tests/frame/test_reshaping.py b/python/pyspark/pandas/tests/frame/test_reshaping.py
index 89b7a319f587..b5cf7c011f46 100644
--- a/python/pyspark/pandas/tests/frame/test_reshaping.py
+++ b/python/pyspark/pandas/tests/frame/test_reshaping.py
@@ -321,8 +321,9 @@ class FrameReshapingMixin:
expected_result1, result1 = pdf.explode("A"), psdf.explode("A")
expected_result2, result2 = pdf.explode("B"), psdf.explode("B")
- expected_result3, result3 = pdf.explode("A", ignore_index=True), psdf.explode(
- "A", ignore_index=True
+ expected_result3, result3 = (
+ pdf.explode("A", ignore_index=True),
+ psdf.explode("A", ignore_index=True),
)
self.assert_eq(result1, expected_result1, almost=True)
@@ -343,8 +344,9 @@ class FrameReshapingMixin:
expected_result1, result1 = pdf.explode("A"), psdf.explode("A")
expected_result2, result2 = pdf.explode("B"), psdf.explode("B")
- expected_result3, result3 = pdf.explode("A", ignore_index=True), psdf.explode(
- "A", ignore_index=True
+ expected_result3, result3 = (
+ pdf.explode("A", ignore_index=True),
+ psdf.explode("A", ignore_index=True),
)
self.assert_eq(result1, expected_result1, almost=True)
diff --git a/python/pyspark/pandas/tests/indexes/test_drop.py b/python/pyspark/pandas/tests/indexes/test_drop.py
index 03e2b3801c7a..2fbfe5045235 100644
--- a/python/pyspark/pandas/tests/indexes/test_drop.py
+++ b/python/pyspark/pandas/tests/indexes/test_drop.py
@@ -91,15 +91,13 @@ class IndexesDropMixin:
with self.assertRaisesRegex(
ValueError,
- "Cannot remove 4 levels from an index with 3 levels: at least one "
- "level must be left.",
+ "Cannot remove 4 levels from an index with 3 levels: at least one level must be left.",
):
psidx.droplevel([0, 0, 1, 2])
with self.assertRaisesRegex(
ValueError,
- "Cannot remove 3 levels from an index with 3 levels: at least one "
- "level must be left.",
+ "Cannot remove 3 levels from an index with 3 levels: at least one level must be left.",
):
psidx.droplevel([0, 1, 2])
diff --git a/python/pyspark/pandas/tests/indexes/test_indexing_loc_2d.py b/python/pyspark/pandas/tests/indexes/test_indexing_loc_2d.py
index 9cf2c2f27c5f..c41190e431be 100644
--- a/python/pyspark/pandas/tests/indexes/test_indexing_loc_2d.py
+++ b/python/pyspark/pandas/tests/indexes/test_indexing_loc_2d.py
@@ -134,15 +134,16 @@ class IndexingLoc2DMixin:
self.assert_eq(psdf.loc["B":"B", ["bar"]], pdf.loc["B":"B", ["bar"]])
self.assert_eq(psdf.loc[:, "bar":"bar"], pdf.loc[:, "bar":"bar"])
- self.assert_eq(psdf.loc[:, "bar":("baz", "one")], pdf.loc[:, "bar":("baz", "one")])
+ self.assert_eq(psdf.loc[:, "bar" : ("baz", "one")], pdf.loc[:, "bar" : ("baz", "one")])
self.assert_eq(
- psdf.loc[:, ("bar", "two"):("baz", "one")], pdf.loc[:, ("bar", "two"):("baz", "one")]
+ psdf.loc[:, ("bar", "two") : ("baz", "one")],
+ pdf.loc[:, ("bar", "two") : ("baz", "one")],
)
- self.assert_eq(psdf.loc[:, ("bar", "two"):"bar"], pdf.loc[:, ("bar", "two"):"bar"])
+ self.assert_eq(psdf.loc[:, ("bar", "two") : "bar"], pdf.loc[:, ("bar", "two") : "bar"])
self.assert_eq(psdf.loc[:, "a":"bax"], pdf.loc[:, "a":"bax"])
self.assert_eq(
- psdf.loc[:, ("bar", "x"):("baz", "a")],
- pdf.loc[:, ("bar", "x"):("baz", "a")],
+ psdf.loc[:, ("bar", "x") : ("baz", "a")],
+ pdf.loc[:, ("bar", "x") : ("baz", "a")],
almost=True,
)
@@ -157,8 +158,8 @@ class IndexingLoc2DMixin:
self.assert_eq(psdf.loc[:, "bar":"baz"], pdf.loc[:, "bar":"baz"])
- self.assertRaises(KeyError, lambda: psdf.loc[:, "bar":("baz", "one")])
- self.assertRaises(KeyError, lambda: psdf.loc[:, ("bar", "two"):"bar"])
+ self.assertRaises(KeyError, lambda: psdf.loc[:, "bar" : ("baz", "one")])
+ self.assertRaises(KeyError, lambda: psdf.loc[:, ("bar", "two") : "bar"])
# bool list-like column select
bool_list = [True, False, True, False]
@@ -180,9 +181,9 @@ class IndexingLoc2DMixin:
self.assert_eq(psdf.loc["B":"B", 0], pdf.loc["B":"B", 0])
self.assert_eq(psdf.loc["B":"B", [0]], pdf.loc["B":"B", [0]])
self.assert_eq(psdf.loc[:, 0:0], pdf.loc[:, 0:0])
- self.assert_eq(psdf.loc[:, 0:(1, 1)], pdf.loc[:, 0:(1, 1)])
- self.assert_eq(psdf.loc[:, (0, 2):(1, 1)], pdf.loc[:, (0, 2):(1, 1)])
- self.assert_eq(psdf.loc[:, (0, 2):0], pdf.loc[:, (0, 2):0])
+ self.assert_eq(psdf.loc[:, 0 : (1, 1)], pdf.loc[:, 0 : (1, 1)])
+ self.assert_eq(psdf.loc[:, (0, 2) : (1, 1)], pdf.loc[:, (0, 2) : (1, 1)])
+ self.assert_eq(psdf.loc[:, (0, 2) : 0], pdf.loc[:, (0, 2) : 0])
self.assert_eq(psdf.loc[:, -1:2], pdf.loc[:, -1:2])
def test_loc2d_with_known_divisions(self):
diff --git a/python/pyspark/pandas/tests/io/test_csv.py b/python/pyspark/pandas/tests/io/test_csv.py
index aa9dc6ae56f6..dd89b5eb0925 100644
--- a/python/pyspark/pandas/tests/io/test_csv.py
+++ b/python/pyspark/pandas/tests/io/test_csv.py
@@ -61,14 +61,17 @@ class CsvTestsMixin:
@property
def csv_text_with_comments(self):
- return normalize_text("""
+ return normalize_text(
+ """
# header
%s
# comment
Alice,400
Edith,600
# footer
- """ % self.csv_text)
+ """
+ % self.csv_text
+ )
@property
def tab_delimited_csv_text(self):
diff --git a/python/pyspark/pandas/tests/series/test_compute.py b/python/pyspark/pandas/tests/series/test_compute.py
index 76389c34c798..b7af7a593b90 100644
--- a/python/pyspark/pandas/tests/series/test_compute.py
+++ b/python/pyspark/pandas/tests/series/test_compute.py
@@ -161,9 +161,7 @@ class SeriesComputeMixin:
msg = "func must be a string or list of strings"
with self.assertRaisesRegex(TypeError, msg):
psser.aggregate({"x": ["min", "max"]})
- msg = (
- "If the given function is a list, it " "should only contains function names as strings."
- )
+ msg = "If the given function is a list, it should only contains function names as strings."
with self.assertRaisesRegex(ValueError, msg):
psser.aggregate(["min", max])
@@ -517,7 +515,7 @@ class SeriesComputeMixin:
psser.between(1, 4, inclusive="right"), pser.between(1, 4, inclusive="right")
)
expected_err_msg = (
- "Inclusive has to be either string of 'both'," "'left', 'right', or 'neither'"
+ "Inclusive has to be either string of 'both','left', 'right', or 'neither'"
)
with self.assertRaisesRegex(ValueError, expected_err_msg):
psser.between(1, 4, inclusive="middle")
diff --git a/python/pyspark/pandas/tests/series/test_index.py b/python/pyspark/pandas/tests/series/test_index.py
index 9bdc7ae333ec..3045df741eb9 100644
--- a/python/pyspark/pandas/tests/series/test_index.py
+++ b/python/pyspark/pandas/tests/series/test_index.py
@@ -348,14 +348,12 @@ class SeriesIndexMixin:
psser.droplevel(-10)
with self.assertRaisesRegex(
ValueError,
- "Cannot remove 3 levels from an index with 3 levels: "
- "at least one level must be left.",
+ "Cannot remove 3 levels from an index with 3 levels: at least one level must be left.",
):
psser.droplevel([0, 1, 2])
with self.assertRaisesRegex(
ValueError,
- "Cannot remove 5 levels from an index with 3 levels: "
- "at least one level must be left.",
+ "Cannot remove 5 levels from an index with 3 levels: at least one level must be left.",
):
psser.droplevel([1, 1, 1, 1, 1])
diff --git a/python/pyspark/pandas/tests/test_scalars.py b/python/pyspark/pandas/tests/test_scalars.py
index 78e416536614..85e740391ad4 100644
--- a/python/pyspark/pandas/tests/test_scalars.py
+++ b/python/pyspark/pandas/tests/test_scalars.py
@@ -36,8 +36,9 @@ class ScalarTestsMixin:
for scalar_name in missing_scalars:
with self.assertRaisesRegex(
PandasNotImplementedError,
- "The scalar `ps.{0}` is not reimplemented in pyspark.pandas;"
- " use `pd.{0}`.".format(scalar_name),
+ "The scalar `ps.{0}` is not reimplemented in pyspark.pandas; use `pd.{0}`.".format(
+ scalar_name
+ ),
):
getattr(ps, scalar_name)
diff --git a/python/pyspark/pandas/typedef/typehints.py b/python/pyspark/pandas/typedef/typehints.py
index a1a0344cedab..a98691efcb0d 100644
--- a/python/pyspark/pandas/typedef/typehints.py
+++ b/python/pyspark/pandas/typedef/typehints.py
@@ -170,7 +170,8 @@ def as_spark_type(
return types.ArrayType(types.StringType())
elif hasattr(tpe, "__origin__") and issubclass(tpe.__origin__, list):
element_type = as_spark_type(
- tpe.__args__[0], raise_error=raise_error # type: ignore[union-attr]
+ tpe.__args__[0], # type: ignore[union-attr]
+ raise_error=raise_error,
)
if element_type is None:
return None
diff --git a/python/pyspark/pandas/utils.py b/python/pyspark/pandas/utils.py
index 8b3ecc395507..6a07befc24da 100644
--- a/python/pyspark/pandas/utils.py
+++ b/python/pyspark/pandas/utils.py
@@ -142,19 +142,19 @@ def combine_frames(
from pyspark.pandas.series import Series
if all(isinstance(arg, Series) for arg in args):
- assert all(
- same_anchor(arg, args[0]) for arg in args
- ), "Currently only one different DataFrame (from given Series) is supported"
+ assert all(same_anchor(arg, args[0]) for arg in args), (
+ "Currently only one different DataFrame (from given Series) is supported"
+ )
assert not same_anchor(this, args[0]), "We don't need to combine. All series is in this."
that = args[0]._psdf[list(args)]
elif len(args) == 1 and isinstance(args[0], DataFrame):
assert isinstance(args[0], DataFrame)
- assert not same_anchor(
- this, args[0]
- ), "We don't need to combine. `this` and `that` are same."
+ assert not same_anchor(this, args[0]), (
+ "We don't need to combine. `this` and `that` are same."
+ )
that = args[0]
else:
- raise AssertionError("args should be single DataFrame or " "single/multiple Series")
+ raise AssertionError("args should be single DataFrame or single/multiple Series")
if get_option("compute.ops_on_diff_frames"):
@@ -761,8 +761,9 @@ def validate_bool_kwarg(value: Any, arg_name: str) -> Optional[bool]:
"""Ensures that argument passed in arg_name is of type bool."""
if not (isinstance(value, bool) or value is None):
raise TypeError(
- 'For argument "{}" expected type bool, received '
- "type {}.".format(arg_name, type(value).__name__)
+ 'For argument "{}" expected type bool, received type {}.'.format(
+ arg_name, type(value).__name__
+ )
)
return value
@@ -907,26 +908,26 @@ def verify_temp_column_name(
), "The temporary column name should be empty or start and end with `__`: {}".format(
column_name_or_label
)
- assert all(
- column_name_or_label != label for label in df._internal.column_labels
- ), "The given column name `{}` already exists in the pandas-on-Spark DataFrame: {}".format(
- name_like_string(column_name_or_label), df.columns
+ assert all(column_name_or_label != label for label in df._internal.column_labels), (
+ "The given column name `{}` already exists in the pandas-on-Spark DataFrame: {}".format(
+ name_like_string(column_name_or_label), df.columns
+ )
)
df = df._internal.resolved_copy.spark_frame
else:
assert isinstance(column_name_or_label, str), type(column_name_or_label)
- assert column_name_or_label.startswith("__") and column_name_or_label.endswith(
- "__"
- ), "The temporary column name should start and end with `__`: {}".format(
- column_name_or_label
+ assert column_name_or_label.startswith("__") and column_name_or_label.endswith("__"), (
+ "The temporary column name should start and end with `__`: {}".format(
+ column_name_or_label
+ )
)
column_name = column_name_or_label
assert isinstance(df, PySparkDataFrame), type(df)
- assert (
- column_name not in df.columns
- ), "The given column name `{}` already exists in the Spark DataFrame: {}".format(
- column_name, df.columns
+ assert column_name not in df.columns, (
+ "The given column name `{}` already exists in the Spark DataFrame: {}".format(
+ column_name, df.columns
+ )
)
return column_name_or_label
diff --git a/python/pyspark/pipelines/cli.py b/python/pyspark/pipelines/cli.py
index 4497c543ea9d..a096d0f42fc8 100644
--- a/python/pyspark/pipelines/cli.py
+++ b/python/pyspark/pipelines/cli.py
@@ -250,9 +250,9 @@ def register_definitions(
module_spec = importlib.util.spec_from_file_location(file.stem, str(file))
assert module_spec is not None, f"Could not find module spec for {file}"
module = importlib.util.module_from_spec(module_spec)
- assert (
- module_spec.loader is not None
- ), f"Module spec has no loader for {file}"
+ assert module_spec.loader is not None, (
+ f"Module spec has no loader for {file}"
+ )
with add_pipeline_analysis_context(
spark=spark, dataflow_graph_id=dataflow_graph_id, flow_name=None
):
diff --git a/python/pyspark/pipelines/tests/test_cli.py b/python/pyspark/pipelines/tests/test_cli.py
index ba5089bf468d..e183df6c347d 100644
--- a/python/pyspark/pipelines/tests/test_cli.py
+++ b/python/pyspark/pipelines/tests/test_cli.py
@@ -258,19 +258,23 @@ class CLIUtilityTests(ReusedConnectTestCase):
subdir2 = outer_dir / "subdir2"
subdir2.mkdir()
with (subdir1 / "libraries.py").open("w") as f:
- f.write(textwrap.dedent("""
+ f.write(
+ textwrap.dedent("""
from pyspark import pipelines as dp
@dp.materialized_view
def mv1():
raise NotImplementedError()
- """))
+ """)
+ )
with (subdir2 / "libraries.py").open("w") as f:
- f.write(textwrap.dedent("""
+ f.write(
+ textwrap.dedent("""
from pyspark import pipelines as dp
def mv2():
raise NotImplementedError()
- """))
+ """)
+ )
registry = LocalGraphElementRegistry()
register_definitions(
@@ -333,11 +337,13 @@ class CLIUtilityTests(ReusedConnectTestCase):
inner_dir2.mkdir()
with (inner_dir1 / "defs.py").open("w") as f:
- f.write(textwrap.dedent("""
+ f.write(
+ textwrap.dedent("""
import sys
sys.path.append(".")
import mypackage.my_module
- """))
+ """)
+ )
inner_dir1_mypackage = inner_dir1 / "mypackage"
inner_dir1_mypackage.mkdir()
diff --git a/python/pyspark/shell.py b/python/pyspark/shell.py
index 788c5143ca19..dae4854a237e 100644
--- a/python/pyspark/shell.py
+++ b/python/pyspark/shell.py
@@ -105,13 +105,16 @@ else:
sql = spark.sql
-print(r"""Welcome to
+print(
+ r"""Welcome to
____ __
/ __/__ ___ _____/ /__
_\ \/ _ \/ _ `/ __/ '_/
/__ / .__/\_,_/_/ /_/\_\ version %s
/_/
-""" % version)
+"""
+ % version
+)
print(
"Using Python version %s (%s, %s)"
% (platform.python_version(), platform.python_build()[0], platform.python_build()[1])
diff --git a/python/pyspark/shuffle.py b/python/pyspark/shuffle.py
index f2d8cf949d17..30147afe0086 100644
--- a/python/pyspark/shuffle.py
+++ b/python/pyspark/shuffle.py
@@ -61,7 +61,7 @@ except ImportError:
return int(line.split()[1]) >> 10
else:
- warnings.warn("Please install psutil to have better " "support with spilling")
+ warnings.warn("Please install psutil to have better support with spilling")
if platform.system() == "Darwin":
import resource
diff --git a/python/pyspark/sql/classic/dataframe.py b/python/pyspark/sql/classic/dataframe.py
index fe70d9f0811b..1ebc92cbe17c 100644
--- a/python/pyspark/sql/classic/dataframe.py
+++ b/python/pyspark/sql/classic/dataframe.py
@@ -1728,9 +1728,9 @@ class DataFrame(ParentDataFrame, PandasMapOpsMixin, PandasConversionMixin):
self, func: Callable[..., ParentDataFrame], *args: Any, **kwargs: Any
) -> ParentDataFrame:
result = func(self, *args, **kwargs)
- assert isinstance(
- result, DataFrame
- ), "Func returned an instance of type [%s], " "should have been DataFrame." % type(result)
+ assert isinstance(result, DataFrame), (
+ "Func returned an instance of type [%s], should have been DataFrame." % type(result)
+ )
return result
def sameSemantics(self, other: ParentDataFrame) -> bool:
diff --git a/python/pyspark/sql/connect/dataframe.py b/python/pyspark/sql/connect/dataframe.py
index ad97b44cf28e..e33a805b6092 100644
--- a/python/pyspark/sql/connect/dataframe.py
+++ b/python/pyspark/sql/connect/dataframe.py
@@ -353,9 +353,7 @@ class DataFrame(ParentDataFrame):
return self._session
def count(self) -> int:
- table, _ = self.agg(
- F._invoke_function("count", F.lit(1))
- )._to_table() # type: ignore[operator]
+ table, _ = self.agg(F._invoke_function("count", F.lit(1)))._to_table() # type: ignore[operator]
return table[0][0].as_py()
def crossJoin(self, other: ParentDataFrame) -> ParentDataFrame:
@@ -1892,9 +1890,9 @@ class DataFrame(ParentDataFrame):
self, func: Callable[..., ParentDataFrame], *args: Any, **kwargs: Any
) -> ParentDataFrame:
result = func(self, *args, **kwargs)
- assert isinstance(
- result, DataFrame
- ), "Func returned an instance of type [%s], " "should have been DataFrame." % type(result)
+ assert isinstance(result, DataFrame), (
+ "Func returned an instance of type [%s], should have been DataFrame." % type(result)
+ )
return result
def _explain_string(
@@ -2170,7 +2168,11 @@ class DataFrame(ParentDataFrame):
self._execution_info = ei
return MergeIntoWriter(
- self._plan, self._session, table, condition, cb # type: ignore[arg-type]
+ self._plan,
+ self._session,
+ table,
+ condition, # type: ignore[arg-type]
+ cb,
)
def offset(self, num: int) -> ParentDataFrame:
diff --git a/python/pyspark/sql/connect/functions/builtin.py b/python/pyspark/sql/connect/functions/builtin.py
index fe0c25ced2ad..f626228c17fb 100644
--- a/python/pyspark/sql/connect/functions/builtin.py
+++ b/python/pyspark/sql/connect/functions/builtin.py
@@ -1681,7 +1681,7 @@ reduce.__doc__ = pysparkfuncs.reduce.__doc__
def array(
- *cols: Union["ColumnOrName", Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]
+ *cols: Union["ColumnOrName", Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]],
) -> Column:
if len(cols) == 1 and isinstance(cols[0], (list, set, tuple)):
cols = cols[0] # type: ignore[assignment]
@@ -1853,7 +1853,7 @@ concat.__doc__ = pysparkfuncs.concat.__doc__
def create_map(
- *cols: Union["ColumnOrName", Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]
+ *cols: Union["ColumnOrName", Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]],
) -> Column:
if len(cols) == 1 and isinstance(cols[0], (list, set, tuple)):
cols = cols[0] # type: ignore[assignment]
@@ -2052,7 +2052,7 @@ json_tuple.__doc__ = pysparkfuncs.json_tuple.__doc__
def map_concat(
- *cols: Union["ColumnOrName", Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]
+ *cols: Union["ColumnOrName", Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]],
) -> Column:
if len(cols) == 1 and isinstance(cols[0], (list, set, tuple)):
cols = cols[0] # type: ignore[assignment]
@@ -2338,7 +2338,7 @@ sort_array.__doc__ = pysparkfuncs.sort_array.__doc__
def struct(
- *cols: Union["ColumnOrName", Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]
+ *cols: Union["ColumnOrName", Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]],
) -> Column:
if len(cols) == 1 and isinstance(cols[0], (list, set, tuple)):
cols = cols[0] # type: ignore[assignment]
diff --git a/python/pyspark/sql/connect/merge.py b/python/pyspark/sql/connect/merge.py
index 9106e4d09aab..9464e33c014c 100644
--- a/python/pyspark/sql/connect/merge.py
+++ b/python/pyspark/sql/connect/merge.py
@@ -42,7 +42,8 @@ def _build_merge_action(
else:
proto_assignments = [
proto.MergeAction.Assignment(
- key=expr(k).to_plan(client), value=v.to_plan(client) # type: ignore[operator]
+ key=expr(k).to_plan(client), # type: ignore[operator]
+ value=v.to_plan(client),
)
for k, v in assignments.items()
]
diff --git a/python/pyspark/sql/connect/plan.py b/python/pyspark/sql/connect/plan.py
index e3ac645d0ceb..8b91517c85fc 100644
--- a/python/pyspark/sql/connect/plan.py
+++ b/python/pyspark/sql/connect/plan.py
@@ -297,9 +297,7 @@ class LogicalPlan:
HTML representation of this :class:`LogicalPlan`.
"""
params = self._parameters_to_print(signature(self.__class__.__init__).parameters)
- pretty_params = [
- f"\n {name}: " f"{param} <br/>" for name, param in params.items()
- ]
+ pretty_params = [f"\n {name}: {param} <br/>" for name, param in params.items()]
if len(pretty_params) == 0:
pretty_str = ""
else:
@@ -1017,7 +1015,9 @@ class Join(LogicalPlan):
self._collect_references(
[]
if on is None or isinstance(on, str)
- else [on] if isinstance(on, Column) else [c for c in on if isinstance(c, Column)]
+ else [on]
+ if isinstance(on, Column)
+ else [c for c in on if isinstance(c, Column)]
),
)
self.left = cast(LogicalPlan, left)
@@ -1970,9 +1970,7 @@ class WriteOperation(LogicalPlan):
if self.table_save_method is not None:
tsm = self.table_save_method.lower()
if tsm == "save_as_table":
- plan.write_operation.table.save_method = (
- proto.WriteOperation.SaveTable.TableSaveMethod.TABLE_SAVE_METHOD_SAVE_AS_TABLE
- )
+ plan.write_operation.table.save_method = proto.WriteOperation.SaveTable.TableSaveMethod.TABLE_SAVE_METHOD_SAVE_AS_TABLE
elif tsm == "insert_into":
plan.write_operation.table.save_method = (
proto.WriteOperation.SaveTable.TableSaveMethod.TABLE_SAVE_METHOD_INSERT_INTO
@@ -2715,7 +2713,9 @@ class PythonUDTF:
self._return_type: Optional[DataType] = (
None
if return_type is None
- else UnparsedDataType(return_type) if isinstance(return_type, str) else return_type
+ else UnparsedDataType(return_type)
+ if isinstance(return_type, str)
+ else return_type
)
self._eval_type = eval_type
self._python_ver = python_ver
@@ -2741,8 +2741,7 @@ class PythonUDTF:
def __repr__(self) -> str:
return (
- f"PythonUDTF({self._name}, {self._return_type}, "
- f"{self._eval_type}, {self._python_ver})"
+ f"PythonUDTF({self._name}, {self._return_type}, {self._eval_type}, {self._python_ver})"
)
diff --git a/python/pyspark/sql/connect/proto/base_pb2.py b/python/pyspark/sql/connect/proto/base_pb2.py
index a17d27544ac4..a77c61ca6d2b 100644
--- a/python/pyspark/sql/connect/proto/base_pb2.py
+++ b/python/pyspark/sql/connect/proto/base_pb2.py
@@ -44,6 +44,7 @@ from pyspark.sql.connect.proto import types_pb2 as spark_dot_connect_dot_types__
from pyspark.sql.connect.proto import ml_pb2 as spark_dot_connect_dot_ml__pb2
from pyspark.sql.connect.proto import pipelines_pb2 as spark_dot_connect_dot_pipelines__pb2
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x18spark/connect/base.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1cspark/connect/commands.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto\x1a\x16spark/connect/ml.proto\x1a\x1dspark/connect/pipelines.proto"\xe3\x03\n\x04Plan\x12-\n\x04root\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationH\x00R\x04root\x12\x32\n\x07\x63ommand\x18\x02 \x01(\x0b\x32\x16.spark.conn [...]
)
@@ -53,20 +54,20 @@ _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "pyspark.sql.connect.proto.base_pb2", _globals)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals["DESCRIPTOR"]._serialized_options = (
- b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
- )
- _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY"]._loaded_options = (
- None
- )
+ _globals[
+ "DESCRIPTOR"
+ ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
+ _globals[
+ "_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY"
+ ]._loaded_options = None
_globals[
"_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY"
]._serialized_options = b"8\001"
_globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._loaded_options = None
_globals["_ARTIFACTSTATUSESRESPONSE_STATUSESENTRY"]._serialized_options = b"8\001"
- _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY"]._loaded_options = (
- None
- )
+ _globals[
+ "_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY"
+ ]._loaded_options = None
_globals[
"_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY"
]._serialized_options = b"8\001"
@@ -159,9 +160,9 @@ if not _descriptor._USE_C_DESCRIPTORS:
_globals[
"_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY"
]._serialized_start = 8937
- _globals["_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY"]._serialized_end = (
- 9060
- )
+ _globals[
+ "_EXECUTEPLANRESPONSE_METRICS_METRICOBJECT_EXECUTIONMETRICSENTRY"
+ ]._serialized_end = 9060
_globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_start = 9062
_globals["_EXECUTEPLANRESPONSE_METRICS_METRICVALUE"]._serialized_end = 9150
_globals["_EXECUTEPLANRESPONSE_OBSERVEDMETRICS"]._serialized_start = 9153
@@ -255,9 +256,9 @@ if not _descriptor._USE_C_DESCRIPTORS:
_globals[
"_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY"
]._serialized_start = 17234
- _globals["_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY"]._serialized_end = (
- 17302
- )
+ _globals[
+ "_FETCHERRORDETAILSRESPONSE_SPARKTHROWABLE_MESSAGEPARAMETERSENTRY"
+ ]._serialized_end = 17302
_globals["_FETCHERRORDETAILSRESPONSE_BREAKINGCHANGEINFO"]._serialized_start = 17360
_globals["_FETCHERRORDETAILSRESPONSE_BREAKINGCHANGEINFO"]._serialized_end = 17610
_globals["_FETCHERRORDETAILSRESPONSE_MITIGATIONCONFIG"]._serialized_start = 17612
diff --git a/python/pyspark/sql/connect/proto/base_pb2.pyi b/python/pyspark/sql/connect/proto/base_pb2.pyi
index f464344d44f5..2db3132cd0c0 100644
--- a/python/pyspark/sql/connect/proto/base_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/base_pb2.pyi
@@ -201,7 +201,6 @@ class UserContext(google.protobuf.message.Message):
messages into this message. Extensions are stored as a `repeated` type to be able to
handle multiple active extensions.
"""
-
def __init__(
self,
*,
@@ -230,7 +229,6 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
@property
def plan(self) -> global___Plan:
"""(Required) The logical plan to be analyzed."""
-
def __init__(
self,
*,
@@ -354,7 +352,6 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
@property
def plan(self) -> global___Plan:
"""(Required) The logical plan to be analyzed."""
-
def __init__(
self,
*,
@@ -372,7 +369,6 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
@property
def plan(self) -> global___Plan:
"""(Required) The logical plan to be analyzed."""
-
def __init__(
self,
*,
@@ -390,7 +386,6 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
@property
def plan(self) -> global___Plan:
"""(Required) The logical plan to be analyzed."""
-
def __init__(
self,
*,
@@ -433,11 +428,9 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
@property
def target_plan(self) -> global___Plan:
"""(Required) The plan to be compared."""
-
@property
def other_plan(self) -> global___Plan:
"""(Required) The other plan to be compared."""
-
def __init__(
self,
*,
@@ -464,7 +457,6 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
@property
def plan(self) -> global___Plan:
"""(Required) The logical plan to get a hashCode."""
-
def __init__(
self,
*,
@@ -483,11 +475,9 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
@property
def relation(self) -> pyspark.sql.connect.proto.relations_pb2.Relation:
"""(Required) The logical plan to persist."""
-
@property
def storage_level(self) -> pyspark.sql.connect.proto.common_pb2.StorageLevel:
"""(Optional) The storage level."""
-
def __init__(
self,
*,
@@ -559,7 +549,6 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
@property
def relation(self) -> pyspark.sql.connect.proto.relations_pb2.Relation:
"""(Required) The logical plan to get the storage level."""
-
def __init__(
self,
*,
@@ -781,7 +770,9 @@ class AnalyzePlanRequest(google.protobuf.message.Message):
self, oneof_group: typing_extensions.Literal["_client_type", b"_client_type"]
) -> typing_extensions.Literal["client_type"] | None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["analyze", b"analyze"]) -> (
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["analyze", b"analyze"]
+ ) -> (
typing_extensions.Literal[
"schema",
"explain",
@@ -894,7 +885,6 @@ class AnalyzePlanResponse(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""A best-effort snapshot of the files that compose this Dataset"""
-
def __init__(
self,
*,
@@ -983,7 +973,6 @@ class AnalyzePlanResponse(google.protobuf.message.Message):
@property
def storage_level(self) -> pyspark.sql.connect.proto.common_pb2.StorageLevel:
"""(Required) The StorageLevel as a result of get_storage_level request."""
-
def __init__(
self,
*,
@@ -1153,7 +1142,9 @@ class AnalyzePlanResponse(google.protobuf.message.Message):
b"unpersist",
],
) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["result", b"result"]) -> (
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["result", b"result"]
+ ) -> (
typing_extensions.Literal[
"schema",
"explain",
@@ -1193,7 +1184,6 @@ class ExecutePlanRequest(google.protobuf.message.Message):
@property
def extension(self) -> google.protobuf.any_pb2.Any:
"""Extension type for request options"""
-
def __init__(
self,
*,
@@ -1286,7 +1276,6 @@ class ExecutePlanRequest(google.protobuf.message.Message):
"""Repeated element for options that can be passed to the request. This element is currently
unused but allows to pass in an extension value used for arbitrary options.
"""
-
@property
def tags(
self,
@@ -1295,7 +1284,6 @@ class ExecutePlanRequest(google.protobuf.message.Message):
Tags cannot contain ',' character and cannot be empty strings.
Used by Interrupt with interrupt.tag.
"""
-
def __init__(
self,
*,
@@ -1305,9 +1293,8 @@ class ExecutePlanRequest(google.protobuf.message.Message):
operation_id: builtins.str | None = ...,
plan: global___Plan | None = ...,
client_type: builtins.str | None = ...,
- request_options: (
- collections.abc.Iterable[global___ExecutePlanRequest.RequestOption] | None
- ) = ...,
+ request_options: collections.abc.Iterable[global___ExecutePlanRequest.RequestOption]
+ | None = ...,
tags: collections.abc.Iterable[builtins.str] | None = ...,
) -> None: ...
def HasField(
@@ -1536,12 +1523,10 @@ class ExecutePlanResponse(google.protobuf.message.Message):
name: builtins.str = ...,
plan_id: builtins.int = ...,
parent: builtins.int = ...,
- execution_metrics: (
- collections.abc.Mapping[
- builtins.str, global___ExecutePlanResponse.Metrics.MetricValue
- ]
- | None
- ) = ...,
+ execution_metrics: collections.abc.Mapping[
+ builtins.str, global___ExecutePlanResponse.Metrics.MetricValue
+ ]
+ | None = ...,
) -> None: ...
def ClearField(
self,
@@ -1590,9 +1575,8 @@ class ExecutePlanResponse(google.protobuf.message.Message):
def __init__(
self,
*,
- metrics: (
- collections.abc.Iterable[global___ExecutePlanResponse.Metrics.MetricObject] | None
- ) = ...,
+ metrics: collections.abc.Iterable[global___ExecutePlanResponse.Metrics.MetricObject]
+ | None = ...,
) -> None: ...
def ClearField(
self, field_name: typing_extensions.Literal["metrics", b"metrics"]
@@ -1632,17 +1616,14 @@ class ExecutePlanResponse(google.protobuf.message.Message):
"""A list of errors that occurred while collecting the observed metrics.
If the length is 0, it means no errors occurred.
"""
-
def __init__(
self,
*,
name: builtins.str = ...,
- values: (
- collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
- ]
- | None
- ) = ...,
+ values: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+ ]
+ | None = ...,
keys: collections.abc.Iterable[builtins.str] | None = ...,
plan_id: builtins.int = ...,
root_error_idx: builtins.int | None = ...,
@@ -1746,10 +1727,10 @@ class ExecutePlanResponse(google.protobuf.message.Message):
def __init__(
self,
*,
- stages: (
- collections.abc.Iterable[global___ExecutePlanResponse.ExecutionProgress.StageInfo]
- | None
- ) = ...,
+ stages: collections.abc.Iterable[
+ global___ExecutePlanResponse.ExecutionProgress.StageInfo
+ ]
+ | None = ...,
num_inflight_tasks: builtins.int = ...,
) -> None: ...
def ClearField(
@@ -1802,69 +1783,56 @@ class ExecutePlanResponse(google.protobuf.message.Message):
@property
def sql_command_result(self) -> global___ExecutePlanResponse.SqlCommandResult:
"""Special case for executing SQL commands."""
-
@property
def write_stream_operation_start_result(
self,
) -> pyspark.sql.connect.proto.commands_pb2.WriteStreamOperationStartResult:
"""Response for a streaming query."""
-
@property
def streaming_query_command_result(
self,
) -> pyspark.sql.connect.proto.commands_pb2.StreamingQueryCommandResult:
"""Response for commands on a streaming query."""
-
@property
def get_resources_command_result(
self,
) -> pyspark.sql.connect.proto.commands_pb2.GetResourcesCommandResult:
"""Response for 'SparkContext.resources'."""
-
@property
def streaming_query_manager_command_result(
self,
) -> pyspark.sql.connect.proto.commands_pb2.StreamingQueryManagerCommandResult:
"""Response for commands on the streaming query manager."""
-
@property
def streaming_query_listener_events_result(
self,
) -> pyspark.sql.connect.proto.commands_pb2.StreamingQueryListenerEventsResult:
"""Response for commands on the client side streaming query listener."""
-
@property
def result_complete(self) -> global___ExecutePlanResponse.ResultComplete:
"""Response type informing if the stream is complete in reattachable execution."""
-
@property
def create_resource_profile_command_result(
self,
) -> pyspark.sql.connect.proto.commands_pb2.CreateResourceProfileCommandResult:
"""Response for command that creates ResourceProfile."""
-
@property
def execution_progress(self) -> global___ExecutePlanResponse.ExecutionProgress:
"""(Optional) Intermediate query progress reports."""
-
@property
def checkpoint_command_result(self) -> global___CheckpointCommandResult:
"""Response for command that checkpoints a DataFrame."""
-
@property
def ml_command_result(self) -> pyspark.sql.connect.proto.ml_pb2.MlCommandResult:
"""ML command response"""
-
@property
def pipeline_event_result(self) -> pyspark.sql.connect.proto.pipelines_pb2.PipelineEventResult:
"""Response containing pipeline event that is streamed back to the client during a pipeline run"""
-
@property
def pipeline_command_result(
self,
) -> pyspark.sql.connect.proto.pipelines_pb2.PipelineCommandResult:
"""Pipeline command response"""
-
@property
def pipeline_query_function_execution_signal(
self,
@@ -1872,17 +1840,14 @@ class ExecutePlanResponse(google.protobuf.message.Message):
"""A signal from the server to the client to execute the query function for a flow, and to
register its result with the server.
"""
-
@property
def extension(self) -> google.protobuf.any_pb2.Any:
"""Support arbitrary result objects."""
-
@property
def metrics(self) -> global___ExecutePlanResponse.Metrics:
"""Metrics for the query execution. Typically, this field is only present in the last
batch of results and then represent the overall state of the query execution.
"""
-
@property
def observed_metrics(
self,
@@ -1890,11 +1855,9 @@ class ExecutePlanResponse(google.protobuf.message.Message):
global___ExecutePlanResponse.ObservedMetrics
]:
"""The metrics observed during the execution of the query plan."""
-
@property
def schema(self) -> pyspark.sql.connect.proto.types_pb2.DataType:
"""(Optional) The Spark schema. This field is available when `collect` is called."""
-
def __init__(
self,
*,
@@ -1904,42 +1867,32 @@ class ExecutePlanResponse(google.protobuf.message.Message):
response_id: builtins.str = ...,
arrow_batch: global___ExecutePlanResponse.ArrowBatch | None = ...,
sql_command_result: global___ExecutePlanResponse.SqlCommandResult | None = ...,
- write_stream_operation_start_result: (
- pyspark.sql.connect.proto.commands_pb2.WriteStreamOperationStartResult | None
- ) = ...,
- streaming_query_command_result: (
- pyspark.sql.connect.proto.commands_pb2.StreamingQueryCommandResult | None
- ) = ...,
- get_resources_command_result: (
- pyspark.sql.connect.proto.commands_pb2.GetResourcesCommandResult | None
- ) = ...,
- streaming_query_manager_command_result: (
- pyspark.sql.connect.proto.commands_pb2.StreamingQueryManagerCommandResult | None
- ) = ...,
- streaming_query_listener_events_result: (
- pyspark.sql.connect.proto.commands_pb2.StreamingQueryListenerEventsResult | None
- ) = ...,
+ write_stream_operation_start_result: pyspark.sql.connect.proto.commands_pb2.WriteStreamOperationStartResult
+ | None = ...,
+ streaming_query_command_result: pyspark.sql.connect.proto.commands_pb2.StreamingQueryCommandResult
+ | None = ...,
+ get_resources_command_result: pyspark.sql.connect.proto.commands_pb2.GetResourcesCommandResult
+ | None = ...,
+ streaming_query_manager_command_result: pyspark.sql.connect.proto.commands_pb2.StreamingQueryManagerCommandResult
+ | None = ...,
+ streaming_query_listener_events_result: pyspark.sql.connect.proto.commands_pb2.StreamingQueryListenerEventsResult
+ | None = ...,
result_complete: global___ExecutePlanResponse.ResultComplete | None = ...,
- create_resource_profile_command_result: (
- pyspark.sql.connect.proto.commands_pb2.CreateResourceProfileCommandResult | None
- ) = ...,
+ create_resource_profile_command_result: pyspark.sql.connect.proto.commands_pb2.CreateResourceProfileCommandResult
+ | None = ...,
execution_progress: global___ExecutePlanResponse.ExecutionProgress | None = ...,
checkpoint_command_result: global___CheckpointCommandResult | None = ...,
ml_command_result: pyspark.sql.connect.proto.ml_pb2.MlCommandResult | None = ...,
- pipeline_event_result: (
- pyspark.sql.connect.proto.pipelines_pb2.PipelineEventResult | None
- ) = ...,
- pipeline_command_result: (
- pyspark.sql.connect.proto.pipelines_pb2.PipelineCommandResult | None
- ) = ...,
- pipeline_query_function_execution_signal: (
- pyspark.sql.connect.proto.pipelines_pb2.PipelineQueryFunctionExecutionSignal | None
- ) = ...,
+ pipeline_event_result: pyspark.sql.connect.proto.pipelines_pb2.PipelineEventResult
+ | None = ...,
+ pipeline_command_result: pyspark.sql.connect.proto.pipelines_pb2.PipelineCommandResult
+ | None = ...,
+ pipeline_query_function_execution_signal: pyspark.sql.connect.proto.pipelines_pb2.PipelineQueryFunctionExecutionSignal
+ | None = ...,
extension: google.protobuf.any_pb2.Any | None = ...,
metrics: global___ExecutePlanResponse.Metrics | None = ...,
- observed_metrics: (
- collections.abc.Iterable[global___ExecutePlanResponse.ObservedMetrics] | None
- ) = ...,
+ observed_metrics: collections.abc.Iterable[global___ExecutePlanResponse.ObservedMetrics]
+ | None = ...,
schema: pyspark.sql.connect.proto.types_pb2.DataType | None = ...,
) -> None: ...
def HasField(
@@ -2227,7 +2180,6 @@ class ConfigRequest(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Required) The config keys to get."""
-
def __init__(
self,
*,
@@ -2244,7 +2196,6 @@ class ConfigRequest(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___KeyValue]:
"""(Required) The config key-value pairs to get. The value will be used as the default value."""
-
def __init__(
self,
*,
@@ -2261,7 +2212,6 @@ class ConfigRequest(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Required) The config keys to get optionally."""
-
def __init__(
self,
*,
@@ -2299,7 +2249,6 @@ class ConfigRequest(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Required) The config keys to unset."""
-
def __init__(
self,
*,
@@ -2316,7 +2265,6 @@ class ConfigRequest(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Required) The config keys to check the config is modifiable."""
-
def __init__(
self,
*,
@@ -2346,7 +2294,6 @@ class ConfigRequest(google.protobuf.message.Message):
@property
def user_context(self) -> global___UserContext:
"""(Required) User context"""
-
@property
def operation(self) -> global___ConfigRequest.Operation:
"""(Required) The operation for the config."""
@@ -2439,7 +2386,6 @@ class ConfigResponse(google.protobuf.message.Message):
Available when the operation is 'Get', 'GetWithDefault', 'GetOption', 'GetAll'.
Also available for the operation 'IsModifiable' with boolean string "true" and "false".
"""
-
@property
def warnings(
self,
@@ -2448,7 +2394,6 @@ class ConfigResponse(google.protobuf.message.Message):
Warning messages for deprecated or unsupported configurations.
"""
-
def __init__(
self,
*,
@@ -2519,7 +2464,6 @@ class AddArtifactsRequest(google.protobuf.message.Message):
@property
def data(self) -> global___AddArtifactsRequest.ArtifactChunk:
"""A single data chunk."""
-
def __init__(
self,
*,
@@ -2548,9 +2492,8 @@ class AddArtifactsRequest(google.protobuf.message.Message):
def __init__(
self,
*,
- artifacts: (
- collections.abc.Iterable[global___AddArtifactsRequest.SingleChunkArtifact] | None
- ) = ...,
+ artifacts: collections.abc.Iterable[global___AddArtifactsRequest.SingleChunkArtifact]
+ | None = ...,
) -> None: ...
def ClearField(
self, field_name: typing_extensions.Literal["artifacts", b"artifacts"]
@@ -2581,7 +2524,6 @@ class AddArtifactsRequest(google.protobuf.message.Message):
@property
def initial_chunk(self) -> global___AddArtifactsRequest.ArtifactChunk:
"""The first/initial chunk."""
-
def __init__(
self,
*,
@@ -2644,13 +2586,11 @@ class AddArtifactsRequest(google.protobuf.message.Message):
The server side is notified about the total size of the large artifact as well as the
number of chunks to expect.
"""
-
@property
def chunk(self) -> global___AddArtifactsRequest.ArtifactChunk:
"""A chunk of an artifact excluding metadata. This can be any chunk of a large artifact
excluding the first chunk (which is included in `BeginChunkedArtifact`).
"""
-
def __init__(
self,
*,
@@ -2778,15 +2718,13 @@ class AddArtifactsResponse(google.protobuf.message.Message):
global___AddArtifactsResponse.ArtifactSummary
]:
"""The list of artifact(s) seen by the server."""
-
def __init__(
self,
*,
session_id: builtins.str = ...,
server_side_session_id: builtins.str = ...,
- artifacts: (
- collections.abc.Iterable[global___AddArtifactsResponse.ArtifactSummary] | None
- ) = ...,
+ artifacts: collections.abc.Iterable[global___AddArtifactsResponse.ArtifactSummary]
+ | None = ...,
) -> None: ...
def ClearField(
self,
@@ -2845,7 +2783,6 @@ class ArtifactStatusesRequest(google.protobuf.message.Message):
The relative path of the file on the server's filesystem will be the same as the name of
the provided artifact)
"""
-
def __init__(
self,
*,
@@ -2962,16 +2899,15 @@ class ArtifactStatusesResponse(google.protobuf.message.Message):
builtins.str, global___ArtifactStatusesResponse.ArtifactStatus
]:
"""A map of artifact names to their statuses."""
-
def __init__(
self,
*,
session_id: builtins.str = ...,
server_side_session_id: builtins.str = ...,
- statuses: (
- collections.abc.Mapping[builtins.str, global___ArtifactStatusesResponse.ArtifactStatus]
- | None
- ) = ...,
+ statuses: collections.abc.Mapping[
+ builtins.str, global___ArtifactStatusesResponse.ArtifactStatus
+ ]
+ | None = ...,
) -> None: ...
def ClearField(
self,
@@ -3147,7 +3083,6 @@ class InterruptResponse(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""Operation ids of the executions which were interrupted."""
-
def __init__(
self,
*,
@@ -3941,7 +3876,6 @@ class FetchErrorDetailsResponse(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""The message parameters for the error framework."""
-
@property
def query_contexts(
self,
@@ -3956,19 +3890,18 @@ class FetchErrorDetailsResponse(google.protobuf.message.Message):
@property
def breaking_change_info(self) -> global___FetchErrorDetailsResponse.BreakingChangeInfo:
"""Additional information if the error was caused by a breaking change."""
-
def __init__(
self,
*,
error_class: builtins.str | None = ...,
message_parameters: collections.abc.Mapping[builtins.str, builtins.str] | None = ...,
- query_contexts: (
- collections.abc.Iterable[global___FetchErrorDetailsResponse.QueryContext] | None
- ) = ...,
+ query_contexts: collections.abc.Iterable[
+ global___FetchErrorDetailsResponse.QueryContext
+ ]
+ | None = ...,
sql_state: builtins.str | None = ...,
- breaking_change_info: (
- global___FetchErrorDetailsResponse.BreakingChangeInfo | None
- ) = ...,
+ breaking_change_info: global___FetchErrorDetailsResponse.BreakingChangeInfo
+ | None = ...,
) -> None: ...
def HasField(
self,
@@ -4039,7 +3972,6 @@ class FetchErrorDetailsResponse(google.protobuf.message.Message):
"""A message explaining how the user can migrate their job to work
with the breaking change.
"""
-
@property
def mitigation_config(self) -> global___FetchErrorDetailsResponse.MitigationConfig:
"""A spark config flag that can be used to mitigate the breaking change."""
@@ -4144,16 +4076,15 @@ class FetchErrorDetailsResponse(google.protobuf.message.Message):
@property
def spark_throwable(self) -> global___FetchErrorDetailsResponse.SparkThrowable:
"""The structured data of a SparkThrowable exception."""
-
def __init__(
self,
*,
error_type_hierarchy: collections.abc.Iterable[builtins.str] | None = ...,
message: builtins.str = ...,
- stack_trace: (
- collections.abc.Iterable[global___FetchErrorDetailsResponse.StackTraceElement]
- | None
- ) = ...,
+ stack_trace: collections.abc.Iterable[
+ global___FetchErrorDetailsResponse.StackTraceElement
+ ]
+ | None = ...,
cause_idx: builtins.int | None = ...,
spark_throwable: global___FetchErrorDetailsResponse.SparkThrowable | None = ...,
) -> None: ...
@@ -4216,7 +4147,6 @@ class FetchErrorDetailsResponse(google.protobuf.message.Message):
global___FetchErrorDetailsResponse.Error
]:
"""A list of errors."""
-
def __init__(
self,
*,
@@ -4259,7 +4189,6 @@ class CheckpointCommandResult(google.protobuf.message.Message):
@property
def relation(self) -> pyspark.sql.connect.proto.relations_pb2.CachedRemoteRelation:
"""(Required) The logical plan checkpointed."""
-
def __init__(
self,
*,
@@ -4440,7 +4369,6 @@ class GetStatusRequest(google.protobuf.message.Message):
"""Get status of operations with these operation_ids.
If unset or empty, returns status of all operations in the session.
"""
-
@property
def extensions(
self,
@@ -4448,7 +4376,6 @@ class GetStatusRequest(google.protobuf.message.Message):
google.protobuf.any_pb2.Any
]:
"""Extension point for custom operation-level status requests."""
-
def __init__(
self,
*,
@@ -4500,7 +4427,6 @@ class GetStatusRequest(google.protobuf.message.Message):
Get status of operations in the session.
"""
-
@property
def extensions(
self,
@@ -4508,7 +4434,6 @@ class GetStatusRequest(google.protobuf.message.Message):
google.protobuf.any_pb2.Any
]:
"""Extension point for custom status request types."""
-
def __init__(
self,
*,
@@ -4643,7 +4568,6 @@ class GetStatusResponse(google.protobuf.message.Message):
google.protobuf.any_pb2.Any
]:
"""Extension point for custom operation-level status fields."""
-
def __init__(
self,
*,
@@ -4675,7 +4599,6 @@ class GetStatusResponse(google.protobuf.message.Message):
global___GetStatusResponse.OperationStatus
]:
"""Status information about requested operations."""
-
@property
def extensions(
self,
@@ -4683,15 +4606,13 @@ class GetStatusResponse(google.protobuf.message.Message):
google.protobuf.any_pb2.Any
]:
"""Extension point for custom status response types."""
-
def __init__(
self,
*,
session_id: builtins.str = ...,
server_side_session_id: builtins.str = ...,
- operation_statuses: (
- collections.abc.Iterable[global___GetStatusResponse.OperationStatus] | None
- ) = ...,
+ operation_statuses: collections.abc.Iterable[global___GetStatusResponse.OperationStatus]
+ | None = ...,
extensions: collections.abc.Iterable[google.protobuf.any_pb2.Any] | None = ...,
) -> None: ...
def ClearField(
diff --git a/python/pyspark/sql/connect/proto/catalog_pb2.py b/python/pyspark/sql/connect/proto/catalog_pb2.py
index 122e55770338..c286def280ce 100644
--- a/python/pyspark/sql/connect/proto/catalog_pb2.py
+++ b/python/pyspark/sql/connect/proto/catalog_pb2.py
@@ -38,6 +38,7 @@ _sym_db = _symbol_database.Default()
from pyspark.sql.connect.proto import common_pb2 as spark_dot_connect_dot_common__pb2
from pyspark.sql.connect.proto import types_pb2 as spark_dot_connect_dot_types__pb2
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x1bspark/connect/catalog.proto\x12\rspark.connect\x1a\x1aspark/connect/common.proto\x1a\x19spark/connect/types.proto"\xc6\x0e\n\x07\x43\x61talog\x12K\n\x10\x63urrent_database\x18\x01 \x01(\x0b\x32\x1e.spark.connect.CurrentDatabaseH\x00R\x0f\x63urrentDatabase\x12U\n\x14set_current_database\x18\x02 \x01(\x0b\x32!.spark.connect.SetCurrentDatabaseH\x00R\x12setCurrentDatabase\x12\x45\n\x0elist_databases\x18\x03 \x01(\x0b\x32\x1c.spark.connect.ListDatabasesH\x00R\rlistDatabases\x12<\n [...]
)
@@ -49,9 +50,9 @@ _builder.BuildTopDescriptorsAndMessages(
)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals["DESCRIPTOR"]._serialized_options = (
- b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
- )
+ _globals[
+ "DESCRIPTOR"
+ ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
_globals["_CREATEEXTERNALTABLE_OPTIONSENTRY"]._loaded_options = None
_globals["_CREATEEXTERNALTABLE_OPTIONSENTRY"]._serialized_options = b"8\001"
_globals["_CREATETABLE_OPTIONSENTRY"]._loaded_options = None
diff --git a/python/pyspark/sql/connect/proto/catalog_pb2.pyi b/python/pyspark/sql/connect/proto/catalog_pb2.pyi
index 72f4ddaf8888..90864adbd5ca 100644
--- a/python/pyspark/sql/connect/proto/catalog_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/catalog_pb2.pyi
@@ -282,7 +282,9 @@ class Catalog(google.protobuf.message.Message):
b"uncache_table",
],
) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["cat_type", b"cat_type"]) -> (
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["cat_type", b"cat_type"]
+ ) -> (
typing_extensions.Literal[
"current_database",
"set_current_database",
@@ -707,13 +709,11 @@ class CreateExternalTable(google.protobuf.message.Message):
@property
def schema(self) -> pyspark.sql.connect.proto.types_pb2.DataType:
"""(Optional)"""
-
@property
def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""Options could be empty for valid data source format.
The map key is case insensitive.
"""
-
def __init__(
self,
*,
@@ -815,13 +815,11 @@ class CreateTable(google.protobuf.message.Message):
@property
def schema(self) -> pyspark.sql.connect.proto.types_pb2.DataType:
"""(Optional)"""
-
@property
def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""Options could be empty for valid data source format.
The map key is case insensitive.
"""
-
def __init__(
self,
*,
@@ -985,7 +983,6 @@ class CacheTable(google.protobuf.message.Message):
@property
def storage_level(self) -> pyspark.sql.connect.proto.common_pb2.StorageLevel:
"""(Optional)"""
-
def __init__(
self,
*,
diff --git a/python/pyspark/sql/connect/proto/commands_pb2.py b/python/pyspark/sql/connect/proto/commands_pb2.py
index 60cbc3f1eb83..a1f005adf9e9 100644
--- a/python/pyspark/sql/connect/proto/commands_pb2.py
+++ b/python/pyspark/sql/connect/proto/commands_pb2.py
@@ -42,6 +42,7 @@ from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_rel
from pyspark.sql.connect.proto import ml_pb2 as spark_dot_connect_dot_ml__pb2
from pyspark.sql.connect.proto import pipelines_pb2 as spark_dot_connect_dot_pipelines__pb2
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x1cspark/connect/commands.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1aspark/connect/common.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/relations.proto\x1a\x16spark/connect/ml.proto\x1a\x1dspark/connect/pipelines.proto"\xfb\x0e\n\x07\x43ommand\x12]\n\x11register_function\x18\x01 \x01(\x0b\x32..spark.connect.CommonInlineUserDefinedFunctionH\x00R\x10registerFunction\x12H\n\x0fwrite_operation\x18\x02 \x01(\x0b\x32\x1d.spark.connect.WriteOpe [...]
)
@@ -53,9 +54,9 @@ _builder.BuildTopDescriptorsAndMessages(
)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals["DESCRIPTOR"]._serialized_options = (
- b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
- )
+ _globals[
+ "DESCRIPTOR"
+ ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
_globals["_SQLCOMMAND_ARGSENTRY"]._loaded_options = None
_globals["_SQLCOMMAND_ARGSENTRY"]._serialized_options = b"8\001"
_globals["_SQLCOMMAND_NAMEDARGUMENTSENTRY"]._loaded_options = None
@@ -154,9 +155,9 @@ if not _descriptor._USE_C_DESCRIPTORS:
_globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_ACTIVERESULT"]._serialized_end = 9542
_globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE"]._serialized_start = 9544
_globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYINSTANCE"]._serialized_end = 9659
- _globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT"]._serialized_start = (
- 9661
- )
+ _globals[
+ "_STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT"
+ ]._serialized_start = 9661
_globals["_STREAMINGQUERYMANAGERCOMMANDRESULT_AWAITANYTERMINATIONRESULT"]._serialized_end = 9720
_globals[
"_STREAMINGQUERYMANAGERCOMMANDRESULT_STREAMINGQUERYLISTENERINSTANCE"
diff --git a/python/pyspark/sql/connect/proto/commands_pb2.pyi b/python/pyspark/sql/connect/proto/commands_pb2.pyi
index 2505dad6e6dd..e0ead65e8922 100644
--- a/python/pyspark/sql/connect/proto/commands_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/commands_pb2.pyi
@@ -162,13 +162,11 @@ class Command(google.protobuf.message.Message):
"""This field is used to mark extensions to the protocol. When plugins generate arbitrary
Commands they can add them here. During the planning the correct resolution is done.
"""
-
def __init__(
self,
*,
- register_function: (
- pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction | None
- ) = ...,
+ register_function: pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction
+ | None = ...,
write_operation: global___WriteOperation | None = ...,
create_dataframe_view: global___CreateDataFrameViewCommand | None = ...,
write_operation_v2: global___WriteOperationV2 | None = ...,
@@ -177,20 +175,16 @@ class Command(google.protobuf.message.Message):
streaming_query_command: global___StreamingQueryCommand | None = ...,
get_resources_command: global___GetResourcesCommand | None = ...,
streaming_query_manager_command: global___StreamingQueryManagerCommand | None = ...,
- register_table_function: (
- pyspark.sql.connect.proto.relations_pb2.CommonInlineUserDefinedTableFunction | None
- ) = ...,
- streaming_query_listener_bus_command: (
- global___StreamingQueryListenerBusCommand | None
- ) = ...,
- register_data_source: (
- pyspark.sql.connect.proto.relations_pb2.CommonInlineUserDefinedDataSource | None
- ) = ...,
+ register_table_function: pyspark.sql.connect.proto.relations_pb2.CommonInlineUserDefinedTableFunction
+ | None = ...,
+ streaming_query_listener_bus_command: global___StreamingQueryListenerBusCommand
+ | None = ...,
+ register_data_source: pyspark.sql.connect.proto.relations_pb2.CommonInlineUserDefinedDataSource
+ | None = ...,
create_resource_profile_command: global___CreateResourceProfileCommand | None = ...,
checkpoint_command: global___CheckpointCommand | None = ...,
- remove_cached_remote_relation_command: (
- global___RemoveCachedRemoteRelationCommand | None
- ) = ...,
+ remove_cached_remote_relation_command: global___RemoveCachedRemoteRelationCommand
+ | None = ...,
merge_into_table_command: global___MergeIntoTableCommand | None = ...,
ml_command: pyspark.sql.connect.proto.ml_pb2.MlCommand | None = ...,
execute_external_command: global___ExecuteExternalCommand | None = ...,
@@ -389,7 +383,6 @@ class SqlCommand(google.protobuf.message.Message):
builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
]:
"""(Optional) A map of parameter names to literal expressions."""
-
@property
def pos_args(
self,
@@ -397,7 +390,6 @@ class SqlCommand(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
]:
"""(Optional) A sequence of literal expressions for positional parameters in the SQL query text."""
-
@property
def named_arguments(
self,
@@ -407,7 +399,6 @@ class SqlCommand(google.protobuf.message.Message):
"""(Optional) A map of parameter names to expressions.
It cannot coexist with `pos_arguments`.
"""
-
@property
def pos_arguments(
self,
@@ -417,34 +408,29 @@ class SqlCommand(google.protobuf.message.Message):
"""(Optional) A sequence of expressions for positional parameters in the SQL query text.
It cannot coexist with `named_arguments`.
"""
-
@property
def input(self) -> pyspark.sql.connect.proto.relations_pb2.Relation:
"""(Optional) The relation that this SQL command will be built on."""
-
def __init__(
self,
*,
sql: builtins.str = ...,
- args: (
- collections.abc.Mapping[
- builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
- ]
- | None
- ) = ...,
- pos_args: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression.Literal]
- | None
- ) = ...,
- named_arguments: (
- collections.abc.Mapping[
- builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None
- ) = ...,
- pos_arguments: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
+ args: collections.abc.Mapping[
+ builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+ ]
+ | None = ...,
+ pos_args: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+ ]
+ | None = ...,
+ named_arguments: collections.abc.Mapping[
+ builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
+ pos_arguments: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
input: pyspark.sql.connect.proto.relations_pb2.Relation | None = ...,
) -> None: ...
def HasField(
@@ -650,29 +636,24 @@ class WriteOperation(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) List of columns to sort the output by."""
-
@property
def partitioning_columns(
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) List of columns for partitioning."""
-
@property
def bucket_by(self) -> global___WriteOperation.BucketBy:
"""(Optional) Bucketing specification. Bucketing must set the number of buckets and the columns
to bucket by.
"""
-
@property
def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""(Optional) A list of configuration options."""
-
@property
def clustering_columns(
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) Columns used for clustering the table."""
-
def __init__(
self,
*,
@@ -840,11 +821,9 @@ class WriteOperationV2(google.protobuf.message.Message):
"""(Optional) List of columns for partitioning for output table created by `create`,
`createOrReplace`, or `replace`
"""
-
@property
def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""(Optional) A list of configuration options."""
-
@property
def table_properties(
self,
@@ -855,22 +834,21 @@ class WriteOperationV2(google.protobuf.message.Message):
@property
def overwrite_condition(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression:
"""(Optional) A condition for overwrite saving mode"""
-
@property
def clustering_columns(
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) Columns used for clustering the table."""
-
def __init__(
self,
*,
input: pyspark.sql.connect.proto.relations_pb2.Relation | None = ...,
table_name: builtins.str = ...,
provider: builtins.str | None = ...,
- partitioning_columns: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
+ partitioning_columns: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
options: collections.abc.Mapping[builtins.str, builtins.str] | None = ...,
table_properties: collections.abc.Mapping[builtins.str, builtins.str] | None = ...,
mode: global___WriteOperationV2.Mode.ValueType = ...,
@@ -994,7 +972,6 @@ class WriteStreamOperationStart(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) Columns used for clustering the table."""
-
def __init__(
self,
*,
@@ -1090,7 +1067,9 @@ class WriteStreamOperationStart(google.protobuf.message.Message):
self, oneof_group: typing_extensions.Literal["sink_destination", b"sink_destination"]
) -> typing_extensions.Literal["path", "table_name"] | None: ...
@typing.overload
- def WhichOneof(self, oneof_group: typing_extensions.Literal["trigger", b"trigger"]) -> (
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["trigger", b"trigger"]
+ ) -> (
typing_extensions.Literal[
"processing_time_interval",
"available_now",
@@ -1309,7 +1288,6 @@ class StreamingQueryCommand(google.protobuf.message.Message):
@property
def await_termination(self) -> global___StreamingQueryCommand.AwaitTerminationCommand:
"""awaitTermination() API. Waits for the termination of the query."""
-
def __init__(
self,
*,
@@ -1373,7 +1351,9 @@ class StreamingQueryCommand(google.protobuf.message.Message):
b"stop",
],
) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["command", b"command"]) -> (
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["command", b"command"]
+ ) -> (
typing_extensions.Literal[
"status",
"last_progress",
@@ -1437,7 +1417,6 @@ class StreamingQueryCommandResult(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""Progress reports as an array of json strings."""
-
def __init__(
self,
*,
@@ -1555,7 +1534,6 @@ class StreamingQueryCommandResult(google.protobuf.message.Message):
@property
def query_id(self) -> global___StreamingQueryInstanceId:
"""(Required) Query instance id. See `StreamingQueryInstanceId`."""
-
@property
def status(self) -> global___StreamingQueryCommandResult.StatusResult: ...
@property
@@ -1673,9 +1651,8 @@ class StreamingQueryManagerCommand(google.protobuf.message.Message):
self,
*,
listener_payload: builtins.bytes = ...,
- python_listener_payload: (
- pyspark.sql.connect.proto.expressions_pb2.PythonUDF | None
- ) = ...,
+ python_listener_payload: pyspark.sql.connect.proto.expressions_pb2.PythonUDF
+ | None = ...,
id: builtins.str = ...,
) -> None: ...
def HasField(
@@ -1728,7 +1705,6 @@ class StreamingQueryManagerCommand(google.protobuf.message.Message):
@property
def add_listener(self) -> global___StreamingQueryManagerCommand.StreamingQueryListenerCommand:
"""addListener API."""
-
@property
def remove_listener(
self,
@@ -1741,16 +1717,13 @@ class StreamingQueryManagerCommand(google.protobuf.message.Message):
*,
active: builtins.bool = ...,
get_query: builtins.str = ...,
- await_any_termination: (
- global___StreamingQueryManagerCommand.AwaitAnyTerminationCommand | None
- ) = ...,
+ await_any_termination: global___StreamingQueryManagerCommand.AwaitAnyTerminationCommand
+ | None = ...,
reset_terminated: builtins.bool = ...,
- add_listener: (
- global___StreamingQueryManagerCommand.StreamingQueryListenerCommand | None
- ) = ...,
- remove_listener: (
- global___StreamingQueryManagerCommand.StreamingQueryListenerCommand | None
- ) = ...,
+ add_listener: global___StreamingQueryManagerCommand.StreamingQueryListenerCommand
+ | None = ...,
+ remove_listener: global___StreamingQueryManagerCommand.StreamingQueryListenerCommand
+ | None = ...,
list_listeners: builtins.bool = ...,
) -> None: ...
def HasField(
@@ -1795,7 +1768,9 @@ class StreamingQueryManagerCommand(google.protobuf.message.Message):
b"reset_terminated",
],
) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["command", b"command"]) -> (
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["command", b"command"]
+ ) -> (
typing_extensions.Literal[
"active",
"get_query",
@@ -1828,12 +1803,10 @@ class StreamingQueryManagerCommandResult(google.protobuf.message.Message):
def __init__(
self,
*,
- active_queries: (
- collections.abc.Iterable[
- global___StreamingQueryManagerCommandResult.StreamingQueryInstance
- ]
- | None
- ) = ...,
+ active_queries: collections.abc.Iterable[
+ global___StreamingQueryManagerCommandResult.StreamingQueryInstance
+ ]
+ | None = ...,
) -> None: ...
def ClearField(
self, field_name: typing_extensions.Literal["active_queries", b"active_queries"]
@@ -1904,7 +1877,6 @@ class StreamingQueryManagerCommandResult(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Required) Reference IDs of listener instances."""
-
def __init__(
self,
*,
@@ -1941,15 +1913,13 @@ class StreamingQueryManagerCommandResult(google.protobuf.message.Message):
*,
active: global___StreamingQueryManagerCommandResult.ActiveResult | None = ...,
query: global___StreamingQueryManagerCommandResult.StreamingQueryInstance | None = ...,
- await_any_termination: (
- global___StreamingQueryManagerCommandResult.AwaitAnyTerminationResult | None
- ) = ...,
+ await_any_termination: global___StreamingQueryManagerCommandResult.AwaitAnyTerminationResult
+ | None = ...,
reset_terminated: builtins.bool = ...,
add_listener: builtins.bool = ...,
remove_listener: builtins.bool = ...,
- list_listeners: (
- global___StreamingQueryManagerCommandResult.ListStreamingQueryListenerResult | None
- ) = ...,
+ list_listeners: global___StreamingQueryManagerCommandResult.ListStreamingQueryListenerResult
+ | None = ...,
) -> None: ...
def HasField(
self,
@@ -1993,7 +1963,9 @@ class StreamingQueryManagerCommandResult(google.protobuf.message.Message):
b"result_type",
],
) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["result_type", b"result_type"]) -> (
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["result_type", b"result_type"]
+ ) -> (
typing_extensions.Literal[
"active",
"query",
@@ -2179,12 +2151,10 @@ class GetResourcesCommandResult(google.protobuf.message.Message):
def __init__(
self,
*,
- resources: (
- collections.abc.Mapping[
- builtins.str, pyspark.sql.connect.proto.common_pb2.ResourceInformation
- ]
- | None
- ) = ...,
+ resources: collections.abc.Mapping[
+ builtins.str, pyspark.sql.connect.proto.common_pb2.ResourceInformation
+ ]
+ | None = ...,
) -> None: ...
def ClearField(
self, field_name: typing_extensions.Literal["resources", b"resources"]
@@ -2201,7 +2171,6 @@ class CreateResourceProfileCommand(google.protobuf.message.Message):
@property
def profile(self) -> pyspark.sql.connect.proto.common_pb2.ResourceProfile:
"""(Required) The ResourceProfile to be built on the server-side."""
-
def __init__(
self,
*,
@@ -2242,7 +2211,6 @@ class RemoveCachedRemoteRelationCommand(google.protobuf.message.Message):
@property
def relation(self) -> pyspark.sql.connect.proto.relations_pb2.CachedRemoteRelation:
"""(Required) The remote to be related"""
-
def __init__(
self,
*,
@@ -2276,7 +2244,6 @@ class CheckpointCommand(google.protobuf.message.Message):
@property
def storage_level(self) -> pyspark.sql.connect.proto.common_pb2.StorageLevel:
"""(Optional) For local checkpoint, the storage level to use."""
-
def __init__(
self,
*,
@@ -2332,11 +2299,9 @@ class MergeIntoTableCommand(google.protobuf.message.Message):
@property
def source_table_plan(self) -> pyspark.sql.connect.proto.relations_pb2.Relation:
"""(Required) The relation of the source table."""
-
@property
def merge_condition(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression:
"""(Required) The condition to match the source and target."""
-
@property
def match_actions(
self,
@@ -2344,7 +2309,6 @@ class MergeIntoTableCommand(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Optional) The actions to be taken when the condition is matched."""
-
@property
def not_matched_actions(
self,
@@ -2352,7 +2316,6 @@ class MergeIntoTableCommand(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Optional) The actions to be taken when the condition is not matched."""
-
@property
def not_matched_by_source_actions(
self,
@@ -2368,15 +2331,18 @@ class MergeIntoTableCommand(google.protobuf.message.Message):
target_table_name: builtins.str = ...,
source_table_plan: pyspark.sql.connect.proto.relations_pb2.Relation | None = ...,
merge_condition: pyspark.sql.connect.proto.expressions_pb2.Expression | None = ...,
- match_actions: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
- not_matched_actions: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
- not_matched_by_source_actions: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
+ match_actions: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
+ not_matched_actions: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
+ not_matched_by_source_actions: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
with_schema_evolution: builtins.bool = ...,
) -> None: ...
def HasField(
@@ -2439,7 +2405,6 @@ class ExecuteExternalCommand(google.protobuf.message.Message):
@property
def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""(Optional) The options for the runner."""
-
def __init__(
self,
*,
diff --git a/python/pyspark/sql/connect/proto/common_pb2.py b/python/pyspark/sql/connect/proto/common_pb2.py
index c8108b627f6f..4eb5145d4425 100644
--- a/python/pyspark/sql/connect/proto/common_pb2.py
+++ b/python/pyspark/sql/connect/proto/common_pb2.py
@@ -46,9 +46,9 @@ _builder.BuildTopDescriptorsAndMessages(
)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals["DESCRIPTOR"]._serialized_options = (
- b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
- )
+ _globals[
+ "DESCRIPTOR"
+ ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
_globals["_RESOURCEPROFILE_EXECUTORRESOURCESENTRY"]._loaded_options = None
_globals["_RESOURCEPROFILE_EXECUTORRESOURCESENTRY"]._serialized_options = b"8\001"
_globals["_RESOURCEPROFILE_TASKRESOURCESENTRY"]._loaded_options = None
diff --git a/python/pyspark/sql/connect/proto/common_pb2.pyi b/python/pyspark/sql/connect/proto/common_pb2.pyi
index 4854a6e0af77..8c5315674a59 100644
--- a/python/pyspark/sql/connect/proto/common_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/common_pb2.pyi
@@ -112,7 +112,6 @@ class ResourceInformation(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Required) An array of strings describing the addresses of the resource."""
-
def __init__(
self,
*,
@@ -275,7 +274,6 @@ class ResourceProfile(google.protobuf.message.Message):
"""(Optional) Resource requests for executors. Mapped from the resource name
(e.g., cores, memory, CPU) to its specific request.
"""
-
@property
def task_resources(
self,
@@ -283,16 +281,13 @@ class ResourceProfile(google.protobuf.message.Message):
"""(Optional) Resource requests for tasks. Mapped from the resource name
(e.g., cores, memory, CPU) to its specific request.
"""
-
def __init__(
self,
*,
- executor_resources: (
- collections.abc.Mapping[builtins.str, global___ExecutorResourceRequest] | None
- ) = ...,
- task_resources: (
- collections.abc.Mapping[builtins.str, global___TaskResourceRequest] | None
- ) = ...,
+ executor_resources: collections.abc.Mapping[builtins.str, global___ExecutorResourceRequest]
+ | None = ...,
+ task_resources: collections.abc.Mapping[builtins.str, global___TaskResourceRequest]
+ | None = ...,
) -> None: ...
def ClearField(
self,
@@ -390,7 +385,6 @@ class JvmOrigin(google.protobuf.message.Message):
global___StackTraceElement
]:
"""(Optional) Stack trace."""
-
def __init__(
self,
*,
diff --git a/python/pyspark/sql/connect/proto/example_plugins_pb2.py b/python/pyspark/sql/connect/proto/example_plugins_pb2.py
index a100ddb9bd55..61d70cccda3a 100644
--- a/python/pyspark/sql/connect/proto/example_plugins_pb2.py
+++ b/python/pyspark/sql/connect/proto/example_plugins_pb2.py
@@ -38,6 +38,7 @@ _sym_db = _symbol_database.Default()
from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_relations__pb2
from pyspark.sql.connect.proto import expressions_pb2 as spark_dot_connect_dot_expressions__pb2
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n#spark/connect/example_plugins.proto\x12\rspark.connect\x1a\x1dspark/connect/relations.proto\x1a\x1fspark/connect/expressions.proto"i\n\x15\x45xamplePluginRelation\x12-\n\x05input\x18\x01 \x01(\x0b\x32\x17.spark.connect.RelationR\x05input\x12!\n\x0c\x63ustom_field\x18\x02 \x01(\tR\x0b\x63ustomField"m\n\x17\x45xamplePluginExpression\x12/\n\x05\x63hild\x18\x01 \x01(\x0b\x32\x19.spark.connect.ExpressionR\x05\x63hild\x12!\n\x0c\x63ustom_field\x18\x02 \x01(\tR\x0b\x63ustomField"9\n\x1 [...]
)
@@ -49,9 +50,9 @@ _builder.BuildTopDescriptorsAndMessages(
)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals["DESCRIPTOR"]._serialized_options = (
- b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
- )
+ _globals[
+ "DESCRIPTOR"
+ ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
_globals["_EXAMPLEPLUGINRELATION"]._serialized_start = 118
_globals["_EXAMPLEPLUGINRELATION"]._serialized_end = 223
_globals["_EXAMPLEPLUGINEXPRESSION"]._serialized_start = 225
diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.py b/python/pyspark/sql/connect/proto/expressions_pb2.py
index 8e2087da18a0..cf9647b7c674 100644
--- a/python/pyspark/sql/connect/proto/expressions_pb2.py
+++ b/python/pyspark/sql/connect/proto/expressions_pb2.py
@@ -39,6 +39,7 @@ from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2
from pyspark.sql.connect.proto import types_pb2 as spark_dot_connect_dot_types__pb2
from pyspark.sql.connect.proto import common_pb2 as spark_dot_connect_dot_common__pb2
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x1fspark/connect/expressions.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x19spark/connect/types.proto\x1a\x1aspark/connect/common.proto"\x92\x38\n\nExpression\x12\x37\n\x06\x63ommon\x18\x12 \x01(\x0b\x32\x1f.spark.connect.ExpressionCommonR\x06\x63ommon\x12=\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralH\x00R\x07literal\x12\x62\n\x14unresolved_attribute\x18\x02 \x01(\x0b\x32-.spark.connect.Expression.UnresolvedAttributeH\x00R\x13unresolved [...]
)
@@ -50,9 +51,9 @@ _builder.BuildTopDescriptorsAndMessages(
)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals["DESCRIPTOR"]._serialized_options = (
- b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
- )
+ _globals[
+ "DESCRIPTOR"
+ ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
_globals["_EXPRESSION_LITERAL_ARRAY"].fields_by_name["element_type"]._loaded_options = None
_globals["_EXPRESSION_LITERAL_ARRAY"].fields_by_name[
"element_type"
diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.pyi b/python/pyspark/sql/connect/proto/expressions_pb2.pyi
index 87b71cd5e7ca..99c9f13b2c4b 100644
--- a/python/pyspark/sql/connect/proto/expressions_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/expressions_pb2.pyi
@@ -114,7 +114,6 @@ class Expression(google.protobuf.message.Message):
@property
def value(self) -> global___Expression:
"""This is an expression for future proofing. We are expecting literals on the server side."""
-
def __init__(
self,
*,
@@ -160,11 +159,9 @@ class Expression(google.protobuf.message.Message):
@property
def lower(self) -> global___Expression.Window.WindowFrame.FrameBoundary:
"""(Required) The lower bound of the frame."""
-
@property
def upper(self) -> global___Expression.Window.WindowFrame.FrameBoundary:
"""(Required) The upper bound of the frame."""
-
def __init__(
self,
*,
@@ -189,7 +186,6 @@ class Expression(google.protobuf.message.Message):
@property
def window_function(self) -> global___Expression:
"""(Required) The window function."""
-
@property
def partition_spec(
self,
@@ -197,7 +193,6 @@ class Expression(google.protobuf.message.Message):
global___Expression
]:
"""(Optional) The way that input rows are partitioned."""
-
@property
def order_spec(
self,
@@ -205,14 +200,12 @@ class Expression(google.protobuf.message.Message):
global___Expression.SortOrder
]:
"""(Optional) Ordering of rows in a partition."""
-
@property
def frame_spec(self) -> global___Expression.Window.WindowFrame:
"""(Optional) Window frame in a partition.
If not set, it will be treated as 'UnspecifiedFrame'.
"""
-
def __init__(
self,
*,
@@ -327,7 +320,6 @@ class Expression(google.protobuf.message.Message):
@property
def child(self) -> global___Expression:
"""(Required) The expression that evaluates to the partition ID."""
-
def __init__(
self,
*,
@@ -370,7 +362,6 @@ class Expression(google.protobuf.message.Message):
@property
def expr(self) -> global___Expression:
"""(Required) the expression to be casted."""
-
@property
def type(self) -> pyspark.sql.connect.proto.types_pb2.DataType: ...
type_str: builtins.str
@@ -512,7 +503,6 @@ class Expression(google.protobuf.message.Message):
This field is deprecated since Spark 4.1+. Use data_type field instead.
"""
-
@property
def elements(
self,
@@ -520,7 +510,6 @@ class Expression(google.protobuf.message.Message):
global___Expression.Literal
]:
"""The literal values that make up the array elements."""
-
def __init__(
self,
*,
@@ -550,7 +539,6 @@ class Expression(google.protobuf.message.Message):
This field is deprecated since Spark 4.1+. Use data_type field instead.
"""
-
@property
def value_type(self) -> pyspark.sql.connect.proto.types_pb2.DataType:
"""(Deprecated) The value type of the map.
@@ -558,7 +546,6 @@ class Expression(google.protobuf.message.Message):
This field is deprecated since Spark 4.1+ and should only be set
if the data_type field is not set. Use data_type field instead.
"""
-
@property
def keys(
self,
@@ -566,7 +553,6 @@ class Expression(google.protobuf.message.Message):
global___Expression.Literal
]:
"""The literal keys that make up the map."""
-
@property
def values(
self,
@@ -574,7 +560,6 @@ class Expression(google.protobuf.message.Message):
global___Expression.Literal
]:
"""The literal values that make up the map."""
-
def __init__(
self,
*,
@@ -615,7 +600,6 @@ class Expression(google.protobuf.message.Message):
This field is deprecated since Spark 4.1+ because using DataType as the type of a struct
is ambiguous. Use data_type field instead.
"""
-
@property
def elements(
self,
@@ -623,7 +607,6 @@ class Expression(google.protobuf.message.Message):
global___Expression.Literal
]:
"""The literal values that make up the struct elements."""
-
def __init__(
self,
*,
@@ -810,7 +793,6 @@ class Expression(google.protobuf.message.Message):
If the data_type field is not set at the root level, the data type will be
inferred or retrieved from the deprecated data type fields using best efforts.
"""
-
def __init__(
self,
*,
@@ -1225,13 +1207,11 @@ class Expression(google.protobuf.message.Message):
"""(Required) The expression to extract value from, can be
Map, Array, Struct or array of Structs.
"""
-
@property
def extraction(self) -> global___Expression:
"""(Required) The expression to describe the extraction, can be
key of Map, index of Array, field name of Struct.
"""
-
def __init__(
self,
*,
@@ -1266,7 +1246,6 @@ class Expression(google.protobuf.message.Message):
When not set, it means this field will be dropped.
"""
-
def __init__(
self,
*,
@@ -1301,7 +1280,6 @@ class Expression(google.protobuf.message.Message):
@property
def expr(self) -> global___Expression:
"""(Required) The expression that alias will be added on."""
-
@property
def name(
self,
@@ -1347,7 +1325,6 @@ class Expression(google.protobuf.message.Message):
The function body should use 'UnresolvedAttribute' as arguments, the sever side will
replace 'UnresolvedAttribute' with 'UnresolvedNamedLambdaVariable'.
"""
-
@property
def arguments(
self,
@@ -1355,14 +1332,12 @@ class Expression(google.protobuf.message.Message):
global___Expression.UnresolvedNamedLambdaVariable
]:
"""(Required) Function variables. Must contains 1 ~ 3 variables."""
-
def __init__(
self,
*,
function: global___Expression | None = ...,
- arguments: (
- collections.abc.Iterable[global___Expression.UnresolvedNamedLambdaVariable] | None
- ) = ...,
+ arguments: collections.abc.Iterable[global___Expression.UnresolvedNamedLambdaVariable]
+ | None = ...,
) -> None: ...
def HasField(
self, field_name: typing_extensions.Literal["function", b"function"]
@@ -1383,7 +1358,6 @@ class Expression(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Required) a list of name parts for the variable. Must not be empty."""
-
def __init__(
self,
*,
@@ -1467,7 +1441,6 @@ class Expression(google.protobuf.message.Message):
"""This field is used to mark extensions to the protocol. When plugins generate arbitrary
relations they can add them here. During the planning the correct resolution is done.
"""
-
def __init__(
self,
*,
@@ -1485,9 +1458,8 @@ class Expression(google.protobuf.message.Message):
window: global___Expression.Window | None = ...,
unresolved_extract_value: global___Expression.UnresolvedExtractValue | None = ...,
update_fields: global___Expression.UpdateFields | None = ...,
- unresolved_named_lambda_variable: (
- global___Expression.UnresolvedNamedLambdaVariable | None
- ) = ...,
+ unresolved_named_lambda_variable: global___Expression.UnresolvedNamedLambdaVariable
+ | None = ...,
common_inline_user_defined_function: global___CommonInlineUserDefinedFunction | None = ...,
call_function: global___CallFunction | None = ...,
named_argument_expression: global___NamedArgumentExpression | None = ...,
@@ -1603,7 +1575,9 @@ class Expression(google.protobuf.message.Message):
b"window",
],
) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["expr_type", b"expr_type"]) -> (
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["expr_type", b"expr_type"]
+ ) -> (
typing_extensions.Literal[
"literal",
"unresolved_attribute",
@@ -1640,7 +1614,6 @@ class ExpressionCommon(google.protobuf.message.Message):
@property
def origin(self) -> pyspark.sql.connect.proto.common_pb2.Origin:
"""(Required) Keep the information of the origin for this expression such as stacktrace."""
-
def __init__(
self,
*,
@@ -1672,7 +1645,6 @@ class CommonInlineUserDefinedFunction(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Expression]:
"""(Optional) Function arguments. Empty arguments are allowed."""
-
@property
def python_udf(self) -> global___PythonUDF: ...
@property
@@ -1754,7 +1726,6 @@ class PythonUDF(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) Additional includes for the Python UDF."""
-
def __init__(
self,
*,
@@ -1802,7 +1773,6 @@ class ScalarScalaUDF(google.protobuf.message.Message):
pyspark.sql.connect.proto.types_pb2.DataType
]:
"""(Optional) Input type(s) of the UDF"""
-
@property
def outputType(self) -> pyspark.sql.connect.proto.types_pb2.DataType:
"""(Required) Output type of the UDF"""
@@ -1814,9 +1784,8 @@ class ScalarScalaUDF(google.protobuf.message.Message):
self,
*,
payload: builtins.bytes = ...,
- inputTypes: (
- collections.abc.Iterable[pyspark.sql.connect.proto.types_pb2.DataType] | None
- ) = ...,
+ inputTypes: collections.abc.Iterable[pyspark.sql.connect.proto.types_pb2.DataType]
+ | None = ...,
outputType: pyspark.sql.connect.proto.types_pb2.DataType | None = ...,
nullable: builtins.bool = ...,
aggregate: builtins.bool = ...,
@@ -1894,7 +1863,6 @@ class TypedAggregateExpression(google.protobuf.message.Message):
@property
def scalar_scala_udf(self) -> global___ScalarScalaUDF:
"""(Required) The aggregate function object packed into bytes."""
-
def __init__(
self,
*,
@@ -1921,7 +1889,6 @@ class CallFunction(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Expression]:
"""(Optional) Function arguments. Empty arguments are allowed."""
-
def __init__(
self,
*,
@@ -1947,7 +1914,6 @@ class NamedArgumentExpression(google.protobuf.message.Message):
@property
def value(self) -> global___Expression:
"""(Required) The value expression of the named argument."""
-
def __init__(
self,
*,
@@ -2000,11 +1966,9 @@ class MergeAction(google.protobuf.message.Message):
@property
def key(self) -> global___Expression:
"""(Required) The key of the assignment."""
-
@property
def value(self) -> global___Expression:
"""(Required) The value of the assignment."""
-
def __init__(
self,
*,
@@ -2026,7 +1990,6 @@ class MergeAction(google.protobuf.message.Message):
@property
def condition(self) -> global___Expression:
"""(Optional) The condition expression of the merge action."""
-
@property
def assignments(
self,
@@ -2034,7 +1997,6 @@ class MergeAction(google.protobuf.message.Message):
global___MergeAction.Assignment
]:
"""(Optional) The assignments of the merge action. Required for ActionTypes INSERT and UPDATE."""
-
def __init__(
self,
*,
@@ -2109,7 +2071,6 @@ class SubqueryExpression(google.protobuf.message.Message):
global___Expression
]:
"""(Optional) The way that input rows are partitioned."""
-
@property
def order_spec(
self,
@@ -2166,13 +2127,11 @@ class SubqueryExpression(google.protobuf.message.Message):
@property
def table_arg_options(self) -> global___SubqueryExpression.TableArgOptions:
"""(Optional) Options specific to table arguments."""
-
@property
def in_subquery_values(
self,
) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Expression]:
"""(Optional) IN subquery values."""
-
def __init__(
self,
*,
diff --git a/python/pyspark/sql/connect/proto/ml_common_pb2.py b/python/pyspark/sql/connect/proto/ml_common_pb2.py
index dda976b333d9..2a5edc0ac93d 100644
--- a/python/pyspark/sql/connect/proto/ml_common_pb2.py
+++ b/python/pyspark/sql/connect/proto/ml_common_pb2.py
@@ -37,6 +37,7 @@ _sym_db = _symbol_database.Default()
from pyspark.sql.connect.proto import expressions_pb2 as spark_dot_connect_dot_expressions__pb2
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x1dspark/connect/ml_common.proto\x12\rspark.connect\x1a\x1fspark/connect/expressions.proto"\xa5\x01\n\x08MlParams\x12;\n\x06params\x18\x01 \x03(\x0b\x32#.spark.connect.MlParams.ParamsEntryR\x06params\x1a\\\n\x0bParamsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x05value:\x02\x38\x01"\x90\x02\n\nMlOperator\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x10\n\x03uid\x18\x02 \x01(\tR\x03uid\x12:\n\x04type [...]
)
@@ -48,9 +49,9 @@ _builder.BuildTopDescriptorsAndMessages(
)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals["DESCRIPTOR"]._serialized_options = (
- b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
- )
+ _globals[
+ "DESCRIPTOR"
+ ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
_globals["_MLPARAMS_PARAMSENTRY"]._loaded_options = None
_globals["_MLPARAMS_PARAMSENTRY"]._serialized_options = b"8\001"
_globals["_MLPARAMS"]._serialized_start = 82
diff --git a/python/pyspark/sql/connect/proto/ml_common_pb2.pyi b/python/pyspark/sql/connect/proto/ml_common_pb2.pyi
index 23be14b7634e..52a072088dc4 100644
--- a/python/pyspark/sql/connect/proto/ml_common_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/ml_common_pb2.pyi
@@ -85,16 +85,13 @@ class MlParams(google.protobuf.message.Message):
builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
]:
"""User-supplied params"""
-
def __init__(
self,
*,
- params: (
- collections.abc.Mapping[
- builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
- ]
- | None
- ) = ...,
+ params: collections.abc.Mapping[
+ builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+ ]
+ | None = ...,
) -> None: ...
def ClearField(self, field_name: typing_extensions.Literal["params", b"params"]) -> None: ...
diff --git a/python/pyspark/sql/connect/proto/ml_pb2.py b/python/pyspark/sql/connect/proto/ml_pb2.py
index 9cbc85c65b6d..6b80d6bdfa7a 100644
--- a/python/pyspark/sql/connect/proto/ml_pb2.py
+++ b/python/pyspark/sql/connect/proto/ml_pb2.py
@@ -39,6 +39,7 @@ from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_rel
from pyspark.sql.connect.proto import expressions_pb2 as spark_dot_connect_dot_expressions__pb2
from pyspark.sql.connect.proto import ml_common_pb2 as spark_dot_connect_dot_ml__common__pb2
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x16spark/connect/ml.proto\x12\rspark.connect\x1a\x1dspark/connect/relations.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/ml_common.proto"\xc7\x0e\n\tMlCommand\x12\x30\n\x03\x66it\x18\x01 \x01(\x0b\x32\x1c.spark.connect.MlCommand.FitH\x00R\x03\x66it\x12,\n\x05\x66\x65tch\x18\x02 \x01(\x0b\x32\x14.spark.connect.FetchH\x00R\x05\x66\x65tch\x12\x39\n\x06\x64\x65lete\x18\x03 \x01(\x0b\x32\x1f.spark.connect.MlCommand.DeleteH\x00R\x06\x64\x65lete\x12\x36\n\x05write\ [...]
)
@@ -48,9 +49,9 @@ _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "pyspark.sql.connect.proto.ml_pb2", _globals)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals["DESCRIPTOR"]._serialized_options = (
- b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
- )
+ _globals[
+ "DESCRIPTOR"
+ ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
_globals["_MLCOMMAND_WRITE_OPTIONSENTRY"]._loaded_options = None
_globals["_MLCOMMAND_WRITE_OPTIONSENTRY"]._serialized_options = b"8\001"
_globals["_MLCOMMAND"]._serialized_start = 137
diff --git a/python/pyspark/sql/connect/proto/ml_pb2.pyi b/python/pyspark/sql/connect/proto/ml_pb2.pyi
index 759bde72134a..38156e722170 100644
--- a/python/pyspark/sql/connect/proto/ml_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/ml_pb2.pyi
@@ -68,15 +68,12 @@ class MlCommand(google.protobuf.message.Message):
@property
def estimator(self) -> pyspark.sql.connect.proto.ml_common_pb2.MlOperator:
"""(Required) Estimator information (its type should be OPERATOR_TYPE_ESTIMATOR)"""
-
@property
def params(self) -> pyspark.sql.connect.proto.ml_common_pb2.MlParams:
"""(Optional) parameters of the Estimator"""
-
@property
def dataset(self) -> pyspark.sql.connect.proto.relations_pb2.Relation:
"""(Required) the training dataset"""
-
def __init__(
self,
*,
@@ -136,9 +133,8 @@ class MlCommand(google.protobuf.message.Message):
def __init__(
self,
*,
- obj_refs: (
- collections.abc.Iterable[pyspark.sql.connect.proto.ml_common_pb2.ObjectRef] | None
- ) = ...,
+ obj_refs: collections.abc.Iterable[pyspark.sql.connect.proto.ml_common_pb2.ObjectRef]
+ | None = ...,
evict_only: builtins.bool | None = ...,
) -> None: ...
def HasField(
@@ -206,11 +202,9 @@ class MlCommand(google.protobuf.message.Message):
@property
def operator(self) -> pyspark.sql.connect.proto.ml_common_pb2.MlOperator:
"""Estimator or evaluator"""
-
@property
def obj_ref(self) -> pyspark.sql.connect.proto.ml_common_pb2.ObjectRef:
"""The cached model"""
-
@property
def params(self) -> pyspark.sql.connect.proto.ml_common_pb2.MlParams:
"""(Optional) The parameters of operator which could be estimator/evaluator or a cached model"""
@@ -223,7 +217,6 @@ class MlCommand(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""(Optional) The options of the writer"""
-
def __init__(
self,
*,
@@ -325,15 +318,12 @@ class MlCommand(google.protobuf.message.Message):
@property
def evaluator(self) -> pyspark.sql.connect.proto.ml_common_pb2.MlOperator:
"""(Required) Evaluator information (its type should be OPERATOR_TYPE_EVALUATOR)"""
-
@property
def params(self) -> pyspark.sql.connect.proto.ml_common_pb2.MlParams:
"""(Optional) parameters of the Evaluator"""
-
@property
def dataset(self) -> pyspark.sql.connect.proto.relations_pb2.Relation:
"""(Required) the evaluating dataset"""
-
def __init__(
self,
*,
@@ -517,7 +507,9 @@ class MlCommand(google.protobuf.message.Message):
b"write",
],
) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["command", b"command"]) -> (
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["command", b"command"]
+ ) -> (
typing_extensions.Literal[
"fit",
"fetch",
@@ -647,7 +639,6 @@ class MlCommandResult(google.protobuf.message.Message):
@property
def operator_info(self) -> global___MlCommandResult.MlOperatorInfo:
"""Operator information"""
-
def __init__(
self,
*,
diff --git a/python/pyspark/sql/connect/proto/pipelines_pb2.py b/python/pyspark/sql/connect/proto/pipelines_pb2.py
index 712c00d3d11f..4cca17b39cd6 100644
--- a/python/pyspark/sql/connect/proto/pipelines_pb2.py
+++ b/python/pyspark/sql/connect/proto/pipelines_pb2.py
@@ -41,6 +41,7 @@ from pyspark.sql.connect.proto import common_pb2 as spark_dot_connect_dot_common
from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_relations__pb2
from pyspark.sql.connect.proto import types_pb2 as spark_dot_connect_dot_types__pb2
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x1dspark/connect/pipelines.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1aspark/connect/common.proto\x1a\x1dspark/connect/relations.proto\x1a\x19spark/connect/types.proto"\xa4\'\n\x0fPipelineCommand\x12h\n\x15\x63reate_dataflow_graph\x18\x01 \x01(\x0b\x32\x32.spark.connect.PipelineCommand.CreateDataflowGraphH\x00R\x13\x63reateDataflowGraph\x12R\n\rdefine_output\x18\x02 \x01(\x0b\x32+.spark.connect.PipelineCommand.DefineOut [...]
)
@@ -52,21 +53,21 @@ _builder.BuildTopDescriptorsAndMessages(
)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals["DESCRIPTOR"]._serialized_options = (
- b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
- )
+ _globals[
+ "DESCRIPTOR"
+ ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
_globals["_PIPELINECOMMAND_CREATEDATAFLOWGRAPH_SQLCONFENTRY"]._loaded_options = None
_globals["_PIPELINECOMMAND_CREATEDATAFLOWGRAPH_SQLCONFENTRY"]._serialized_options = b"8\001"
- _globals["_PIPELINECOMMAND_DEFINEOUTPUT_TABLEDETAILS_TABLEPROPERTIESENTRY"]._loaded_options = (
- None
- )
+ _globals[
+ "_PIPELINECOMMAND_DEFINEOUTPUT_TABLEDETAILS_TABLEPROPERTIESENTRY"
+ ]._loaded_options = None
_globals[
"_PIPELINECOMMAND_DEFINEOUTPUT_TABLEDETAILS_TABLEPROPERTIESENTRY"
]._serialized_options = b"8\001"
_globals["_PIPELINECOMMAND_DEFINEOUTPUT_SINKDETAILS_OPTIONSENTRY"]._loaded_options = None
- _globals["_PIPELINECOMMAND_DEFINEOUTPUT_SINKDETAILS_OPTIONSENTRY"]._serialized_options = (
- b"8\001"
- )
+ _globals[
+ "_PIPELINECOMMAND_DEFINEOUTPUT_SINKDETAILS_OPTIONSENTRY"
+ ]._serialized_options = b"8\001"
_globals["_PIPELINECOMMAND_DEFINEFLOW_SQLCONFENTRY"]._loaded_options = None
_globals["_PIPELINECOMMAND_DEFINEFLOW_SQLCONFENTRY"]._serialized_options = b"8\001"
_globals["_PIPELINECOMMAND_DEFINEFLOWQUERYFUNCTIONRESULT"].fields_by_name[
@@ -102,9 +103,9 @@ if not _descriptor._USE_C_DESCRIPTORS:
_globals[
"_PIPELINECOMMAND_DEFINEOUTPUT_TABLEDETAILS_TABLEPROPERTIESENTRY"
]._serialized_start = 2532
- _globals["_PIPELINECOMMAND_DEFINEOUTPUT_TABLEDETAILS_TABLEPROPERTIESENTRY"]._serialized_end = (
- 2598
- )
+ _globals[
+ "_PIPELINECOMMAND_DEFINEOUTPUT_TABLEDETAILS_TABLEPROPERTIESENTRY"
+ ]._serialized_end = 2598
_globals["_PIPELINECOMMAND_DEFINEOUTPUT_SINKDETAILS"]._serialized_start = 2622
_globals["_PIPELINECOMMAND_DEFINEOUTPUT_SINKDETAILS"]._serialized_end = 2831
_globals["_PIPELINECOMMAND_DEFINEOUTPUT_SINKDETAILS_OPTIONSENTRY"]._serialized_start = 2762
diff --git a/python/pyspark/sql/connect/proto/pipelines_pb2.pyi b/python/pyspark/sql/connect/proto/pipelines_pb2.pyi
index e986e801acc5..ee628ddd2419 100644
--- a/python/pyspark/sql/connect/proto/pipelines_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/pipelines_pb2.pyi
@@ -129,7 +129,6 @@ class PipelineCommand(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""SQL configurations for all flows in this graph."""
-
def __init__(
self,
*,
@@ -248,7 +247,6 @@ class PipelineCommand(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""Optional table properties."""
-
@property
def partition_cols(
self,
@@ -264,7 +262,6 @@ class PipelineCommand(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""Optional cluster columns for the table."""
-
def __init__(
self,
*,
@@ -390,7 +387,6 @@ class PipelineCommand(google.protobuf.message.Message):
@property
def source_code_location(self) -> global___SourceCodeLocation:
"""The location in source code that this output was defined."""
-
@property
def table_details(self) -> global___PipelineCommand.DefineOutput.TableDetails: ...
@property
@@ -537,7 +533,6 @@ class PipelineCommand(google.protobuf.message.Message):
"""An unresolved relation that defines the dataset's flow. Empty if the query function
that defines the flow cannot be analyzed at the time of flow definition.
"""
-
def __init__(
self,
*,
@@ -613,7 +608,6 @@ class PipelineCommand(google.protobuf.message.Message):
@property
def source_code_location(self) -> global___SourceCodeLocation:
"""The location in source code that this flow was defined."""
-
@property
def relation_flow_details(
self,
@@ -636,9 +630,8 @@ class PipelineCommand(google.protobuf.message.Message):
sql_conf: collections.abc.Mapping[builtins.str, builtins.str] | None = ...,
client_id: builtins.str | None = ...,
source_code_location: global___SourceCodeLocation | None = ...,
- relation_flow_details: (
- global___PipelineCommand.DefineFlow.WriteRelationFlowDetails | None
- ) = ...,
+ relation_flow_details: global___PipelineCommand.DefineFlow.WriteRelationFlowDetails
+ | None = ...,
extension: google.protobuf.any_pb2.Any | None = ...,
once: builtins.bool | None = ...,
) -> None: ...
@@ -761,7 +754,6 @@ class PipelineCommand(google.protobuf.message.Message):
@property
def define_output(self) -> global___PipelineCommand.DefineOutput:
"""The output (table or materialized view or sink) definition."""
-
@property
def define_flows(
self,
@@ -780,14 +772,12 @@ class PipelineCommand(google.protobuf.message.Message):
google.protobuf.any_pb2.Any
]:
"""Reserved field for protocol extensions."""
-
def __init__(
self,
*,
define_output: global___PipelineCommand.DefineOutput | None = ...,
- define_flows: (
- collections.abc.Iterable[global___PipelineCommand.DefineFlow] | None
- ) = ...,
+ define_flows: collections.abc.Iterable[global___PipelineCommand.DefineFlow]
+ | None = ...,
full_refresh: builtins.bool | None = ...,
storage: builtins.str | None = ...,
extension: collections.abc.Iterable[google.protobuf.any_pb2.Any] | None = ...,
@@ -1098,7 +1088,6 @@ class PipelineCommand(google.protobuf.message.Message):
@property
def relation(self) -> pyspark.sql.connect.proto.relations_pb2.Relation:
"""An unresolved relation that defines the dataset's flow."""
-
def __init__(
self,
*,
@@ -1206,7 +1195,6 @@ class PipelineCommand(google.protobuf.message.Message):
that are not yet defined in this version of the proto. During planning, the
engine will resolve and dispatch the concrete command contained in this field.
"""
-
def __init__(
self,
*,
@@ -1216,12 +1204,10 @@ class PipelineCommand(google.protobuf.message.Message):
drop_dataflow_graph: global___PipelineCommand.DropDataflowGraph | None = ...,
start_run: global___PipelineCommand.StartRun | None = ...,
define_sql_graph_elements: global___PipelineCommand.DefineSqlGraphElements | None = ...,
- get_query_function_execution_signal_stream: (
- global___PipelineCommand.GetQueryFunctionExecutionSignalStream | None
- ) = ...,
- define_flow_query_function_result: (
- global___PipelineCommand.DefineFlowQueryFunctionResult | None
- ) = ...,
+ get_query_function_execution_signal_stream: global___PipelineCommand.GetQueryFunctionExecutionSignalStream
+ | None = ...,
+ define_flow_query_function_result: global___PipelineCommand.DefineFlowQueryFunctionResult
+ | None = ...,
execute_output_flows: global___PipelineCommand.ExecuteOutputFlows | None = ...,
extension: google.protobuf.any_pb2.Any | None = ...,
) -> None: ...
@@ -1345,13 +1331,11 @@ class PipelineCommandResult(google.protobuf.message.Message):
@property
def resolved_identifier(self) -> pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier:
"""Resolved identifier of the output"""
-
def __init__(
self,
*,
- resolved_identifier: (
- pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier | None
- ) = ...,
+ resolved_identifier: pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier
+ | None = ...,
) -> None: ...
def HasField(
self,
@@ -1383,13 +1367,11 @@ class PipelineCommandResult(google.protobuf.message.Message):
@property
def resolved_identifier(self) -> pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier:
"""Resolved identifier of the flow"""
-
def __init__(
self,
*,
- resolved_identifier: (
- pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier | None
- ) = ...,
+ resolved_identifier: pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier
+ | None = ...,
) -> None: ...
def HasField(
self,
@@ -1428,9 +1410,8 @@ class PipelineCommandResult(google.protobuf.message.Message):
def __init__(
self,
*,
- create_dataflow_graph_result: (
- global___PipelineCommandResult.CreateDataflowGraphResult | None
- ) = ...,
+ create_dataflow_graph_result: global___PipelineCommandResult.CreateDataflowGraphResult
+ | None = ...,
define_output_result: global___PipelineCommandResult.DefineOutputResult | None = ...,
define_flow_result: global___PipelineCommandResult.DefineFlowResult | None = ...,
) -> None: ...
@@ -1551,7 +1532,6 @@ class SourceCodeLocation(google.protobuf.message.Message):
that are not yet defined in this version of the proto. During planning, the
engine will resolve and dispatch the concrete command contained in this field.
"""
-
def __init__(
self,
*,
@@ -1628,7 +1608,6 @@ class PipelineQueryFunctionExecutionSignal(google.protobuf.message.Message):
This field is deprecated since Spark 4.2+. Use flow_identifiers field instead.
"""
-
@property
def flow_identifiers(
self,
@@ -1636,14 +1615,14 @@ class PipelineQueryFunctionExecutionSignal(google.protobuf.message.Message):
pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier
]:
"""The identifier of flows that are ready to be re-evaluated"""
-
def __init__(
self,
*,
flow_names: collections.abc.Iterable[builtins.str] | None = ...,
- flow_identifiers: (
- collections.abc.Iterable[pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier] | None
- ) = ...,
+ flow_identifiers: collections.abc.Iterable[
+ pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier
+ ]
+ | None = ...,
) -> None: ...
def ClearField(
self,
@@ -1676,7 +1655,6 @@ class PipelineAnalysisContext(google.protobuf.message.Message):
@property
def flow_identifier(self) -> pyspark.sql.connect.proto.common_pb2.ResolvedIdentifier:
"""The identifier of the Flow involved in this analysis"""
-
@property
def extension(
self,
@@ -1684,7 +1662,6 @@ class PipelineAnalysisContext(google.protobuf.message.Message):
google.protobuf.any_pb2.Any
]:
"""Reserved field for protocol extensions."""
-
def __init__(
self,
*,
diff --git a/python/pyspark/sql/connect/proto/relations_pb2.py b/python/pyspark/sql/connect/proto/relations_pb2.py
index 36ad7def3696..dfa511c3ba39 100644
--- a/python/pyspark/sql/connect/proto/relations_pb2.py
+++ b/python/pyspark/sql/connect/proto/relations_pb2.py
@@ -42,6 +42,7 @@ from pyspark.sql.connect.proto import catalog_pb2 as spark_dot_connect_dot_catal
from pyspark.sql.connect.proto import common_pb2 as spark_dot_connect_dot_common__pb2
from pyspark.sql.connect.proto import ml_common_pb2 as spark_dot_connect_dot_ml__common__pb2
+
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x19spark/connect/types.proto\x1a\x1bspark/connect/catalog.proto\x1a\x1aspark/connect/common.proto\x1a\x1dspark/connect/ml_common.proto"\x8c\x1e\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x [...]
)
@@ -53,9 +54,9 @@ _builder.BuildTopDescriptorsAndMessages(
)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals["DESCRIPTOR"]._serialized_options = (
- b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
- )
+ _globals[
+ "DESCRIPTOR"
+ ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
_globals["_RELATIONCOMMON"].fields_by_name["source_info"]._loaded_options = None
_globals["_RELATIONCOMMON"].fields_by_name["source_info"]._serialized_options = b"\030\001"
_globals["_SQL_ARGSENTRY"]._loaded_options = None
diff --git a/python/pyspark/sql/connect/proto/relations_pb2.pyi b/python/pyspark/sql/connect/proto/relations_pb2.pyi
index ff6e02850731..d77517af29ea 100644
--- a/python/pyspark/sql/connect/proto/relations_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/relations_pb2.pyi
@@ -222,7 +222,6 @@ class Relation(google.protobuf.message.Message):
@property
def fill_na(self) -> global___NAFill:
"""NA functions"""
-
@property
def drop_na(self) -> global___NADrop: ...
@property
@@ -230,7 +229,6 @@ class Relation(google.protobuf.message.Message):
@property
def summary(self) -> global___StatSummary:
"""stat functions"""
-
@property
def crosstab(self) -> global___StatCrosstab: ...
@property
@@ -248,17 +246,14 @@ class Relation(google.protobuf.message.Message):
@property
def catalog(self) -> pyspark.sql.connect.proto.catalog_pb2.Catalog:
"""Catalog API (experimental / unstable)"""
-
@property
def ml_relation(self) -> global___MlRelation:
"""ML relation"""
-
@property
def extension(self) -> google.protobuf.any_pb2.Any:
"""This field is used to mark extensions to the protocol. When plugins generate arbitrary
relations they can add them here. During the planning the correct resolution is done.
"""
-
@property
def unknown(self) -> global___Unknown: ...
def __init__(
@@ -301,13 +296,11 @@ class Relation(google.protobuf.message.Message):
html_string: global___HtmlString | None = ...,
cached_local_relation: global___CachedLocalRelation | None = ...,
cached_remote_relation: global___CachedRemoteRelation | None = ...,
- common_inline_user_defined_table_function: (
- global___CommonInlineUserDefinedTableFunction | None
- ) = ...,
+ common_inline_user_defined_table_function: global___CommonInlineUserDefinedTableFunction
+ | None = ...,
as_of_join: global___AsOfJoin | None = ...,
- common_inline_user_defined_data_source: (
- global___CommonInlineUserDefinedDataSource | None
- ) = ...,
+ common_inline_user_defined_data_source: global___CommonInlineUserDefinedDataSource
+ | None = ...,
with_relations: global___WithRelations | None = ...,
transpose: global___Transpose | None = ...,
unresolved_table_valued_function: global___UnresolvedTableValuedFunction | None = ...,
@@ -583,7 +576,9 @@ class Relation(google.protobuf.message.Message):
b"with_watermark",
],
) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["rel_type", b"rel_type"]) -> (
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["rel_type", b"rel_type"]
+ ) -> (
typing_extensions.Literal[
"read",
"project",
@@ -669,19 +664,15 @@ class MlRelation(google.protobuf.message.Message):
@property
def obj_ref(self) -> pyspark.sql.connect.proto.ml_common_pb2.ObjectRef:
"""Object reference"""
-
@property
def transformer(self) -> pyspark.sql.connect.proto.ml_common_pb2.MlOperator:
"""Could be an ML transformer like VectorAssembler"""
-
@property
def input(self) -> global___Relation:
"""the input dataframe"""
-
@property
def params(self) -> pyspark.sql.connect.proto.ml_common_pb2.MlParams:
"""the operator specific parameters"""
-
def __init__(
self,
*,
@@ -734,7 +725,6 @@ class MlRelation(google.protobuf.message.Message):
@property
def model_summary_dataset(self) -> global___Relation:
"""(Optional) the dataset for restoring the model summary"""
-
def __init__(
self,
*,
@@ -841,7 +831,6 @@ class Fetch(google.protobuf.message.Message):
global___Fetch.Method.Args
]:
"""(Optional) the arguments of the method"""
-
def __init__(
self,
*,
@@ -857,13 +846,11 @@ class Fetch(google.protobuf.message.Message):
@property
def obj_ref(self) -> pyspark.sql.connect.proto.ml_common_pb2.ObjectRef:
"""(Required) reference to the object on the server side"""
-
@property
def methods(
self,
) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Fetch.Method]:
"""(Required) the calling method chains"""
-
def __init__(
self,
*,
@@ -905,7 +892,6 @@ class RelationCommon(google.protobuf.message.Message):
@property
def origin(self) -> pyspark.sql.connect.proto.common_pb2.Origin:
"""(Optional) Keep the information of the origin for this expression such as stacktrace."""
-
def __init__(
self,
*,
@@ -999,7 +985,6 @@ class SQL(google.protobuf.message.Message):
builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
]:
"""(Optional) A map of parameter names to literal expressions."""
-
@property
def pos_args(
self,
@@ -1007,7 +992,6 @@ class SQL(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
]:
"""(Optional) A sequence of literal expressions for positional parameters in the SQL query text."""
-
@property
def named_arguments(
self,
@@ -1017,7 +1001,6 @@ class SQL(google.protobuf.message.Message):
"""(Optional) A map of parameter names to expressions.
It cannot coexist with `pos_arguments`.
"""
-
@property
def pos_arguments(
self,
@@ -1027,30 +1010,26 @@ class SQL(google.protobuf.message.Message):
"""(Optional) A sequence of expressions for positional parameters in the SQL query text.
It cannot coexist with `named_arguments`.
"""
-
def __init__(
self,
*,
query: builtins.str = ...,
- args: (
- collections.abc.Mapping[
- builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
- ]
- | None
- ) = ...,
- pos_args: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression.Literal]
- | None
- ) = ...,
- named_arguments: (
- collections.abc.Mapping[
- builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression
- ]
- | None
- ) = ...,
- pos_arguments: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
+ args: collections.abc.Mapping[
+ builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+ ]
+ | None = ...,
+ pos_args: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+ ]
+ | None = ...,
+ named_arguments: collections.abc.Mapping[
+ builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
+ pos_arguments: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
) -> None: ...
def ClearField(
self,
@@ -1089,7 +1068,6 @@ class WithRelations(google.protobuf.message.Message):
"""(Required) Plan at the root of the query tree. This plan is expected to contain one or more
references. Those references get expanded later on by the engine.
"""
-
@property
def references(
self,
@@ -1097,7 +1075,6 @@ class WithRelations(google.protobuf.message.Message):
"""(Required) Plans referenced by the root plan. Relations in this list are also allowed to
contain references to other relations in this list, as long they do not form cycles.
"""
-
def __init__(
self,
*,
@@ -1147,7 +1124,6 @@ class Read(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""Options for the named table. The map key is case insensitive."""
-
def __init__(
self,
*,
@@ -1205,13 +1181,11 @@ class Read(google.protobuf.message.Message):
data source format. This options could be empty for valid data source format.
The map key is case insensitive.
"""
-
@property
def paths(
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) A list of path for file-system backed data sources."""
-
@property
def predicates(
self,
@@ -1346,7 +1320,6 @@ class Project(google.protobuf.message.Message):
For example, `SELECT ABS(-1)` is valid plan without an input plan.
"""
-
@property
def expressions(
self,
@@ -1354,14 +1327,12 @@ class Project(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Required) A Project requires at least one expression."""
-
def __init__(
self,
*,
input: global___Relation | None = ...,
- expressions: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
+ expressions: collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
+ | None = ...,
) -> None: ...
def HasField(
self, field_name: typing_extensions.Literal["input", b"input"]
@@ -1385,11 +1356,9 @@ class Filter(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) Input relation for a Filter."""
-
@property
def condition(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression:
"""(Required) A Filter must have a condition expression."""
-
def __init__(
self,
*,
@@ -1472,11 +1441,9 @@ class Join(google.protobuf.message.Message):
@property
def left(self) -> global___Relation:
"""(Required) Left input relation for a Join."""
-
@property
def right(self) -> global___Relation:
"""(Required) Right input relation for a Join."""
-
@property
def join_condition(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression:
"""(Optional) The join condition. Could be unset when `using_columns` is utilized.
@@ -1495,11 +1462,9 @@ class Join(google.protobuf.message.Message):
This field does not co-exist with join_condition.
"""
-
@property
def join_data_type(self) -> global___Join.JoinDataType:
"""(Optional) Only used by joinWith. Set the left and right join data types."""
-
def __init__(
self,
*,
@@ -1586,7 +1551,6 @@ class SetOperation(google.protobuf.message.Message):
@property
def left_input(self) -> global___Relation:
"""(Required) Left input relation for a Set operation."""
-
@property
def right_input(self) -> global___Relation:
"""(Required) Right input relation for a Set operation."""
@@ -1798,7 +1762,6 @@ class Aggregate(google.protobuf.message.Message):
@property
def col(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression:
"""(Required) The column to pivot"""
-
@property
def values(
self,
@@ -1810,17 +1773,14 @@ class Aggregate(google.protobuf.message.Message):
Note that if it is empty, the server side will immediately trigger a job to collect
the distinct values of the column.
"""
-
def __init__(
self,
*,
col: pyspark.sql.connect.proto.expressions_pb2.Expression | None = ...,
- values: (
- collections.abc.Iterable[
- pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
- ]
- | None
- ) = ...,
+ values: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+ ]
+ | None = ...,
) -> None: ...
def HasField(
self, field_name: typing_extensions.Literal["col", b"col"]
@@ -1840,14 +1800,13 @@ class Aggregate(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Required) Individual grouping set"""
-
def __init__(
self,
*,
- grouping_set: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
- | None
- ) = ...,
+ grouping_set: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
) -> None: ...
def ClearField(
self, field_name: typing_extensions.Literal["grouping_set", b"grouping_set"]
@@ -1871,7 +1830,6 @@ class Aggregate(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Required) Expressions for grouping keys"""
-
@property
def aggregate_expressions(
self,
@@ -1879,11 +1837,9 @@ class Aggregate(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Required) List of values that will be translated to columns in the output DataFrame."""
-
@property
def pivot(self) -> global___Aggregate.Pivot:
"""(Optional) Pivots a column of the current `DataFrame` and performs the specified aggregation."""
-
@property
def grouping_sets(
self,
@@ -1891,18 +1847,19 @@ class Aggregate(google.protobuf.message.Message):
global___Aggregate.GroupingSets
]:
"""(Optional) List of values that will be translated to columns in the output DataFrame."""
-
def __init__(
self,
*,
input: global___Relation | None = ...,
group_type: global___Aggregate.GroupType.ValueType = ...,
- grouping_expressions: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
- aggregate_expressions: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
+ grouping_expressions: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
+ aggregate_expressions: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
pivot: global___Aggregate.Pivot | None = ...,
grouping_sets: collections.abc.Iterable[global___Aggregate.GroupingSets] | None = ...,
) -> None: ...
@@ -1940,7 +1897,6 @@ class Sort(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) Input relation for a Sort."""
-
@property
def order(
self,
@@ -1954,10 +1910,10 @@ class Sort(google.protobuf.message.Message):
self,
*,
input: global___Relation | None = ...,
- order: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression.SortOrder]
- | None
- ) = ...,
+ order: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression.SortOrder
+ ]
+ | None = ...,
is_global: builtins.bool | None = ...,
) -> None: ...
def HasField(
@@ -1996,7 +1952,6 @@ class Drop(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
-
@property
def columns(
self,
@@ -2004,20 +1959,17 @@ class Drop(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Optional) columns to drop."""
-
@property
def column_names(
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) names of columns to drop."""
-
def __init__(
self,
*,
input: global___Relation | None = ...,
- columns: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
+ columns: collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
+ | None = ...,
column_names: collections.abc.Iterable[builtins.str] | None = ...,
) -> None: ...
def HasField(
@@ -2046,7 +1998,6 @@ class Deduplicate(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) Input relation for a Deduplicate."""
-
@property
def column_names(
self,
@@ -2417,7 +2368,6 @@ class SubqueryAlias(google.protobuf.message.Message):
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) Qualifier of the alias."""
-
def __init__(
self,
*,
@@ -2584,7 +2534,6 @@ class StatSummary(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
-
@property
def statistics(
self,
@@ -2604,7 +2553,6 @@ class StatSummary(google.protobuf.message.Message):
If no statistics are given, this function computes 'count', 'mean', 'stddev', 'min',
'approximate quartiles' (percentiles at 25%, 50%, and 75%), and 'max'.
"""
-
def __init__(
self,
*,
@@ -2633,13 +2581,11 @@ class StatDescribe(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
-
@property
def cols(
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) Columns to compute statistics on."""
-
def __init__(
self,
*,
@@ -2804,13 +2750,11 @@ class StatApproxQuantile(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
-
@property
def cols(
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Required) The names of the numerical columns."""
-
@property
def probabilities(
self,
@@ -2867,7 +2811,6 @@ class StatFreqItems(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
-
@property
def cols(
self,
@@ -2942,11 +2885,9 @@ class StatSampleBy(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
-
@property
def col(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression:
"""(Required) The column that defines strata."""
-
@property
def fractions(
self,
@@ -3017,13 +2958,11 @@ class NAFill(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
-
@property
def cols(
self,
) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]:
"""(Optional) Optional list of column names to consider."""
-
@property
def values(
self,
@@ -3035,16 +2974,15 @@ class NAFill(google.protobuf.message.Message):
Should contain at least 1 item.
Only 4 data types are supported now: bool, long, double, string
"""
-
def __init__(
self,
*,
input: global___Relation | None = ...,
cols: collections.abc.Iterable[builtins.str] | None = ...,
- values: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression.Literal]
- | None
- ) = ...,
+ values: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+ ]
+ | None = ...,
) -> None: ...
def HasField(
self, field_name: typing_extensions.Literal["input", b"input"]
@@ -3071,7 +3009,6 @@ class NADrop(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
-
@property
def cols(
self,
@@ -3146,14 +3083,12 @@ class NAReplace(google.protobuf.message.Message):
Only 4 data types are supported now: null, bool, double, string.
"""
-
@property
def new_value(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression.Literal:
"""(Required) The new value.
Should be of the same data type with the old value.
"""
-
def __init__(
self,
*,
@@ -3179,7 +3114,6 @@ class NAReplace(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
-
@property
def cols(
self,
@@ -3188,7 +3122,6 @@ class NAReplace(google.protobuf.message.Message):
When it is empty, all the type-compatible columns in the input relation will be considered.
"""
-
@property
def replacements(
self,
@@ -3196,7 +3129,6 @@ class NAReplace(google.protobuf.message.Message):
global___NAReplace.Replacement
]:
"""(Optional) The value replacement mapping."""
-
def __init__(
self,
*,
@@ -3226,7 +3158,6 @@ class ToDF(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation of RenameColumnsBySameLengthNames."""
-
@property
def column_names(
self,
@@ -3236,7 +3167,6 @@ class ToDF(google.protobuf.message.Message):
The number of columns of the input relation must be equal to the length
of this field. If this is not true, an exception will be returned.
"""
-
def __init__(
self,
*,
@@ -3303,7 +3233,6 @@ class WithColumnsRenamed(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
-
@property
def rename_columns_map(
self,
@@ -3315,7 +3244,6 @@ class WithColumnsRenamed(google.protobuf.message.Message):
does not require that all input relation column names to present as keys.
duplicated B are not allowed.
"""
-
@property
def renames(
self,
@@ -3351,7 +3279,6 @@ class WithColumns(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
-
@property
def aliases(
self,
@@ -3368,15 +3295,14 @@ class WithColumns(google.protobuf.message.Message):
An exception is thrown when duplicated names are present in the mapping.
"""
-
def __init__(
self,
*,
input: global___Relation | None = ...,
- aliases: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression.Alias]
- | None
- ) = ...,
+ aliases: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression.Alias
+ ]
+ | None = ...,
) -> None: ...
def HasField(
self, field_name: typing_extensions.Literal["input", b"input"]
@@ -3444,15 +3370,13 @@ class Hint(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Optional) Hint parameters."""
-
def __init__(
self,
*,
input: global___Relation | None = ...,
name: builtins.str = ...,
- parameters: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
+ parameters: collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
+ | None = ...,
) -> None: ...
def HasField(
self, field_name: typing_extensions.Literal["input", b"input"]
@@ -3484,10 +3408,8 @@ class Unpivot(google.protobuf.message.Message):
def __init__(
self,
*,
- values: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
- | None
- ) = ...,
+ values: collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
+ | None = ...,
) -> None: ...
def ClearField(
self, field_name: typing_extensions.Literal["values", b"values"]
@@ -3501,7 +3423,6 @@ class Unpivot(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
-
@property
def ids(
self,
@@ -3509,7 +3430,6 @@ class Unpivot(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Required) Id columns."""
-
@property
def values(self) -> global___Unpivot.Values:
"""(Optional) Value columns to unpivot."""
@@ -3521,9 +3441,8 @@ class Unpivot(google.protobuf.message.Message):
self,
*,
input: global___Relation | None = ...,
- ids: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
+ ids: collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
+ | None = ...,
values: global___Unpivot.Values | None = ...,
variable_column_name: builtins.str = ...,
value_column_name: builtins.str = ...,
@@ -3570,7 +3489,6 @@ class Transpose(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
-
@property
def index_columns(
self,
@@ -3580,14 +3498,14 @@ class Transpose(google.protobuf.message.Message):
"""(Optional) A list of columns that will be treated as the indices.
Only single column is supported now.
"""
-
def __init__(
self,
*,
input: global___Relation | None = ...,
- index_columns: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
+ index_columns: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
) -> None: ...
def HasField(
self, field_name: typing_extensions.Literal["input", b"input"]
@@ -3613,14 +3531,12 @@ class UnresolvedTableValuedFunction(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Optional) Function arguments. Empty arguments are allowed."""
-
def __init__(
self,
*,
function_name: builtins.str = ...,
- arguments: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
+ arguments: collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
+ | None = ...,
) -> None: ...
def ClearField(
self,
@@ -3639,14 +3555,12 @@ class ToSchema(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
-
@property
def schema(self) -> pyspark.sql.connect.proto.types_pb2.DataType:
"""(Required) The user provided schema.
The Sever side will update the dataframe with this schema.
"""
-
def __init__(
self,
*,
@@ -3671,7 +3585,6 @@ class RepartitionByExpression(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) The input relation."""
-
@property
def partition_exprs(
self,
@@ -3685,9 +3598,10 @@ class RepartitionByExpression(google.protobuf.message.Message):
self,
*,
input: global___Relation | None = ...,
- partition_exprs: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
+ partition_exprs: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
num_partitions: builtins.int | None = ...,
) -> None: ...
def HasField(
@@ -3730,7 +3644,6 @@ class MapPartitions(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) Input relation for a mapPartitions-equivalent API: mapInPandas, mapInArrow."""
-
@property
def func(self) -> pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction:
"""(Required) Input user-defined function."""
@@ -3742,9 +3655,8 @@ class MapPartitions(google.protobuf.message.Message):
self,
*,
input: global___Relation | None = ...,
- func: (
- pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction | None
- ) = ...,
+ func: pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction
+ | None = ...,
is_barrier: builtins.bool | None = ...,
profile_id: builtins.int | None = ...,
) -> None: ...
@@ -3810,7 +3722,6 @@ class GroupMap(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) Input relation for Group Map API: apply, applyInPandas."""
-
@property
def grouping_expressions(
self,
@@ -3818,11 +3729,9 @@ class GroupMap(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Required) Expressions for grouping keys."""
-
@property
def func(self) -> pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction:
"""(Required) Input user-defined function."""
-
@property
def sorting_expressions(
self,
@@ -3830,13 +3739,11 @@ class GroupMap(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Optional) Expressions for sorting. Only used by Scala Sorted Group Map API."""
-
@property
def initial_input(self) -> global___Relation:
"""Below fields are only used by (Flat)MapGroupsWithState
(Optional) Input relation for initial State.
"""
-
@property
def initial_grouping_expressions(
self,
@@ -3853,30 +3760,30 @@ class GroupMap(google.protobuf.message.Message):
@property
def state_schema(self) -> pyspark.sql.connect.proto.types_pb2.DataType:
"""(Optional) The schema for the grouped state."""
-
@property
def transform_with_state_info(self) -> global___TransformWithStateInfo:
"""Below fields are used by TransformWithState and TransformWithStateInPandas
(Optional) TransformWithState related parameters.
"""
-
def __init__(
self,
*,
input: global___Relation | None = ...,
- grouping_expressions: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
- func: (
- pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction | None
- ) = ...,
- sorting_expressions: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
+ grouping_expressions: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
+ func: pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction
+ | None = ...,
+ sorting_expressions: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
initial_input: global___Relation | None = ...,
- initial_grouping_expressions: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
+ initial_grouping_expressions: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
is_map_groups_with_state: builtins.bool | None = ...,
output_mode: builtins.str | None = ...,
timeout_conf: builtins.str | None = ...,
@@ -3997,7 +3904,6 @@ class TransformWithStateInfo(google.protobuf.message.Message):
"""(Optional) Schema for the output DataFrame.
Only required used for TransformWithStateInPandas.
"""
-
def __init__(
self,
*,
@@ -4060,7 +3966,6 @@ class CoGroupMap(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) One input relation for CoGroup Map API - applyInPandas."""
-
@property
def input_grouping_expressions(
self,
@@ -4068,11 +3973,9 @@ class CoGroupMap(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""Expressions for grouping keys of the first input relation."""
-
@property
def other(self) -> global___Relation:
"""(Required) The other input relation."""
-
@property
def other_grouping_expressions(
self,
@@ -4080,11 +3983,9 @@ class CoGroupMap(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""Expressions for grouping keys of the other input relation."""
-
@property
def func(self) -> pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction:
"""(Required) Input user-defined function."""
-
@property
def input_sorting_expressions(
self,
@@ -4092,7 +3993,6 @@ class CoGroupMap(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Optional) Expressions for sorting. Only used by Scala Sorted CoGroup Map API."""
-
@property
def other_sorting_expressions(
self,
@@ -4100,27 +4000,29 @@ class CoGroupMap(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Optional) Expressions for sorting. Only used by Scala Sorted CoGroup Map API."""
-
def __init__(
self,
*,
input: global___Relation | None = ...,
- input_grouping_expressions: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
+ input_grouping_expressions: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
other: global___Relation | None = ...,
- other_grouping_expressions: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
- func: (
- pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction | None
- ) = ...,
- input_sorting_expressions: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
- other_sorting_expressions: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
+ other_grouping_expressions: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
+ func: pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction
+ | None = ...,
+ input_sorting_expressions: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
+ other_sorting_expressions: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
) -> None: ...
def HasField(
self,
@@ -4163,7 +4065,6 @@ class ApplyInPandasWithState(google.protobuf.message.Message):
@property
def input(self) -> global___Relation:
"""(Required) Input relation for applyInPandasWithState."""
-
@property
def grouping_expressions(
self,
@@ -4171,7 +4072,6 @@ class ApplyInPandasWithState(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Required) Expressions for grouping keys."""
-
@property
def func(self) -> pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction:
"""(Required) Input user-defined function."""
@@ -4187,12 +4087,12 @@ class ApplyInPandasWithState(google.protobuf.message.Message):
self,
*,
input: global___Relation | None = ...,
- grouping_expressions: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
- func: (
- pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction | None
- ) = ...,
+ grouping_expressions: collections.abc.Iterable[
+ pyspark.sql.connect.proto.expressions_pb2.Expression
+ ]
+ | None = ...,
+ func: pyspark.sql.connect.proto.expressions_pb2.CommonInlineUserDefinedFunction
+ | None = ...,
output_schema: builtins.str = ...,
state_schema: builtins.str = ...,
output_mode: builtins.str = ...,
@@ -4241,7 +4141,6 @@ class CommonInlineUserDefinedTableFunction(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Optional) Function input arguments. Empty arguments are allowed."""
-
@property
def python_udtf(self) -> global___PythonUDTF: ...
def __init__(
@@ -4249,9 +4148,8 @@ class CommonInlineUserDefinedTableFunction(google.protobuf.message.Message):
*,
function_name: builtins.str = ...,
deterministic: builtins.bool = ...,
- arguments: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
+ arguments: collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
+ | None = ...,
python_udtf: global___PythonUDTF | None = ...,
) -> None: ...
def HasField(
@@ -4412,15 +4310,13 @@ class CollectMetrics(google.protobuf.message.Message):
pyspark.sql.connect.proto.expressions_pb2.Expression
]:
"""(Required) The metric sequence."""
-
def __init__(
self,
*,
input: global___Relation | None = ...,
name: builtins.str = ...,
- metrics: (
- collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression] | None
- ) = ...,
+ metrics: collections.abc.Iterable[pyspark.sql.connect.proto.expressions_pb2.Expression]
+ | None = ...,
) -> None: ...
def HasField(
self, field_name: typing_extensions.Literal["input", b"input"]
@@ -4484,11 +4380,9 @@ class Parse(google.protobuf.message.Message):
@property
def schema(self) -> pyspark.sql.connect.proto.types_pb2.DataType:
"""(Optional) DataType representing the schema. If not set, Spark will infer the schema."""
-
@property
def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]:
"""Options for the csv/json parser. The map key is case insensitive."""
-
def __init__(
self,
*,
@@ -4545,26 +4439,21 @@ class AsOfJoin(google.protobuf.message.Message):
@property
def left(self) -> global___Relation:
"""(Required) Left input relation for a Join."""
-
@property
def right(self) -> global___Relation:
"""(Required) Right input relation for a Join."""
-
@property
def left_as_of(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression:
"""(Required) Field to join on in left DataFrame"""
-
@property
def right_as_of(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression:
"""(Required) Field to join on in right DataFrame"""
-
@property
def join_expr(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression:
"""(Optional) The join condition. Could be unset when `using_columns` is utilized.
This field does not co-exist with using_columns.
"""
-
@property
def using_columns(
self,
@@ -4658,11 +4547,9 @@ class LateralJoin(google.protobuf.message.Message):
@property
def left(self) -> global___Relation:
"""(Required) Left input relation for a Join."""
-
@property
def right(self) -> global___Relation:
"""(Required) Right input relation for a Join."""
-
@property
def join_condition(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression:
"""(Optional) The join condition."""
diff --git a/python/pyspark/sql/connect/proto/types_pb2.py b/python/pyspark/sql/connect/proto/types_pb2.py
index 0d7f8c3c9d58..224c55660842 100644
--- a/python/pyspark/sql/connect/proto/types_pb2.py
+++ b/python/pyspark/sql/connect/proto/types_pb2.py
@@ -44,9 +44,9 @@ _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals)
_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, "pyspark.sql.connect.proto.types_pb2", _globals)
if not _descriptor._USE_C_DESCRIPTORS:
_globals["DESCRIPTOR"]._loaded_options = None
- _globals["DESCRIPTOR"]._serialized_options = (
- b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
- )
+ _globals[
+ "DESCRIPTOR"
+ ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
_globals["_DATATYPE"]._serialized_start = 45
_globals["_DATATYPE"]._serialized_end = 4869
_globals["_DATATYPE_BOOLEAN"]._serialized_start = 1778
diff --git a/python/pyspark/sql/connect/proto/types_pb2.pyi b/python/pyspark/sql/connect/proto/types_pb2.pyi
index c90b9f660a7a..f2c727b26db9 100644
--- a/python/pyspark/sql/connect/proto/types_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/types_pb2.pyi
@@ -750,7 +750,6 @@ class DataType(google.protobuf.message.Message):
@property
def sql_type(self) -> global___DataType:
"""Required for Python UDT"""
-
def __init__(
self,
*,
@@ -876,7 +875,6 @@ class DataType(google.protobuf.message.Message):
@property
def byte(self) -> global___DataType.Byte:
"""Numeric types"""
-
@property
def short(self) -> global___DataType.Short: ...
@property
@@ -892,7 +890,6 @@ class DataType(google.protobuf.message.Message):
@property
def string(self) -> global___DataType.String:
"""String types"""
-
@property
def char(self) -> global___DataType.Char: ...
@property
@@ -900,7 +897,6 @@ class DataType(google.protobuf.message.Message):
@property
def date(self) -> global___DataType.Date:
"""Datatime types"""
-
@property
def timestamp(self) -> global___DataType.Timestamp: ...
@property
@@ -908,7 +904,6 @@ class DataType(google.protobuf.message.Message):
@property
def calendar_interval(self) -> global___DataType.CalendarInterval:
"""Interval types"""
-
@property
def year_month_interval(self) -> global___DataType.YearMonthInterval: ...
@property
@@ -916,7 +911,6 @@ class DataType(google.protobuf.message.Message):
@property
def array(self) -> global___DataType.Array:
"""Complex types"""
-
@property
def struct(self) -> global___DataType.Struct: ...
@property
@@ -926,17 +920,14 @@ class DataType(google.protobuf.message.Message):
@property
def udt(self) -> global___DataType.UDT:
"""UserDefinedType"""
-
@property
def geometry(self) -> global___DataType.Geometry:
"""Geospatial types"""
-
@property
def geography(self) -> global___DataType.Geography: ...
@property
def unparsed(self) -> global___DataType.Unparsed:
"""UnparsedDataType"""
-
@property
def time(self) -> global___DataType.Time: ...
def __init__(
@@ -1097,7 +1088,9 @@ class DataType(google.protobuf.message.Message):
b"year_month_interval",
],
) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["kind", b"kind"]) -> (
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["kind", b"kind"]
+ ) -> (
typing_extensions.Literal[
"null",
"binary",
diff --git a/python/pyspark/sql/connect/readwriter.py b/python/pyspark/sql/connect/readwriter.py
index 25cadbcf1a3b..923b719a4894 100644
--- a/python/pyspark/sql/connect/readwriter.py
+++ b/python/pyspark/sql/connect/readwriter.py
@@ -443,9 +443,9 @@ class DataFrameReader(OptionUtils):
if column is not None:
assert lowerBound is not None, "lowerBound can not be None when ``column`` is specified"
assert upperBound is not None, "upperBound can not be None when ``column`` is specified"
- assert (
- numPartitions is not None
- ), "numPartitions can not be None when ``column`` is specified"
+ assert numPartitions is not None, (
+ "numPartitions can not be None when ``column`` is specified"
+ )
self.options(
partitionColumn=column,
lowerBound=lowerBound,
diff --git a/python/pyspark/sql/connect/session.py b/python/pyspark/sql/connect/session.py
index b72dbdfadddd..4538646b9b51 100644
--- a/python/pyspark/sql/connect/session.py
+++ b/python/pyspark/sql/connect/session.py
@@ -609,7 +609,9 @@ class SparkSession:
(
TimestampType()
if is_datetime64_dtype(t) or isinstance(t, pd.DatetimeTZDtype)
- else DayTimeIntervalType() if is_timedelta64_dtype(t) else None
+ else DayTimeIntervalType()
+ if is_timedelta64_dtype(t)
+ else None
)
for t in data.dtypes
]
diff --git a/python/pyspark/sql/connect/streaming/worker/foreach_batch_worker.py b/python/pyspark/sql/connect/streaming/worker/foreach_batch_worker.py
index 4343195f1972..5aa7ba29ad6c 100644
--- a/python/pyspark/sql/connect/streaming/worker/foreach_batch_worker.py
+++ b/python/pyspark/sql/connect/streaming/worker/foreach_batch_worker.py
@@ -65,7 +65,7 @@ def main(infile: IO, outfile: IO) -> None:
connect_url = os.environ["SPARK_CONNECT_LOCAL_URL"]
session_id = utf8_deserializer.loads(infile)
- print(f"{log_name} is starting with " f"url {connect_url} and sessionId {session_id}.")
+ print(f"{log_name} is starting with url {connect_url} and sessionId {session_id}.")
# To attach to the existing SparkSession, we're setting the session_id in the URL.
connect_url = connect_url + ";session_id=" + session_id
diff --git a/python/pyspark/sql/connect/udtf.py b/python/pyspark/sql/connect/udtf.py
index bc0c057a61fd..fa99c331084a 100644
--- a/python/pyspark/sql/connect/udtf.py
+++ b/python/pyspark/sql/connect/udtf.py
@@ -159,7 +159,9 @@ class UserDefinedTableFunction:
self.returnType: Optional[DataType] = (
None
if returnType is None
- else UnparsedDataType(returnType) if isinstance(returnType, str) else returnType
+ else UnparsedDataType(returnType)
+ if isinstance(returnType, str)
+ else returnType
)
self._name = name or func.__name__
self.evalType = evalType
diff --git a/python/pyspark/sql/context.py b/python/pyspark/sql/context.py
index 1fb7397b0fa1..7145b27f2cf3 100644
--- a/python/pyspark/sql/context.py
+++ b/python/pyspark/sql/context.py
@@ -166,8 +166,7 @@ class SQLContext:
cls: Type["SQLContext"], sc: "SparkContext", **static_conf: Any
) -> "SQLContext":
if (
- cls._instantiatedContext is None
- or SQLContext._instantiatedContext._sc._jsc is None # type: ignore[union-attr]
+ cls._instantiatedContext is None or SQLContext._instantiatedContext._sc._jsc is None # type: ignore[union-attr]
):
assert sc._jvm is not None
# There can be only one running Spark context. That will automatically
diff --git a/python/pyspark/sql/conversion.py b/python/pyspark/sql/conversion.py
index a4e1a9836d81..602ebec8eb80 100644
--- a/python/pyspark/sql/conversion.py
+++ b/python/pyspark/sql/conversion.py
@@ -1037,9 +1037,9 @@ class ArrowTableToRowsConversion:
dataType.elementType, none_on_identity=True, binary_as_bytes=binary_as_bytes
)
- assert (
- element_conv is not None
- ), f"_need_converter() returned True for ArrayType of {dataType.elementType}"
+ assert element_conv is not None, (
+ f"_need_converter() returned True for ArrayType of {dataType.elementType}"
+ )
def convert_array(value: Any) -> Any:
if value is None:
diff --git a/python/pyspark/sql/datasource_internal.py b/python/pyspark/sql/datasource_internal.py
index cae2fbef6ba2..e71dfb25ef9c 100644
--- a/python/pyspark/sql/datasource_internal.py
+++ b/python/pyspark/sql/datasource_internal.py
@@ -119,9 +119,9 @@ class _SimpleStreamReaderWrapper(DataSourceStreamReader):
def latestOffset(self, start: dict, limit: ReadLimit) -> dict:
assert start is not None, "start offset should not be None"
- assert isinstance(
- limit, ReadAllAvailable
- ), "simple stream reader does not support read limit"
+ assert isinstance(limit, ReadAllAvailable), (
+ "simple stream reader does not support read limit"
+ )
iter, end = self.simple_reader.read(start)
self.add_result_to_cache(start, end, iter)
@@ -163,7 +163,8 @@ class _SimpleStreamReaderWrapper(DataSourceStreamReader):
return it
def read(
- self, input_partition: SimpleInputPartition # type: ignore[override]
+ self,
+ input_partition: SimpleInputPartition, # type: ignore[override]
) -> Iterator[Tuple]:
return self.simple_reader.readBetweenOffsets(input_partition.start, input_partition.end)
diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py
index 9dbc5a6aa65f..1c79c074cb3b 100644
--- a/python/pyspark/sql/functions/builtin.py
+++ b/python/pyspark/sql/functions/builtin.py
@@ -8117,7 +8117,7 @@ def struct(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]])
@_try_remote_functions
def struct(
- *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]]
+ *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]],
) -> Column:
"""Creates a new struct column.
@@ -18319,7 +18319,7 @@ def create_map(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...
@_try_remote_functions
def create_map(
- *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]]
+ *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]],
) -> Column:
"""
Map function: Creates a new map column from an even number of input columns or
@@ -18484,7 +18484,7 @@ def array(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]) -
@_try_remote_functions
def array(
- *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]]
+ *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]],
) -> Column:
"""
Collection function: Creates a new array column from the input columns or column names.
@@ -23105,7 +23105,7 @@ def map_concat(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...
@_try_remote_functions
def map_concat(
- *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]]
+ *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]],
) -> Column:
"""
Map function: Returns the union of all given maps.
diff --git a/python/pyspark/sql/internal.py b/python/pyspark/sql/internal.py
index dd9ebbcdc182..b18ee6616804 100644
--- a/python/pyspark/sql/internal.py
+++ b/python/pyspark/sql/internal.py
@@ -42,7 +42,8 @@ class InternalFunction:
sc = SparkContext._active_spark_context
return Column(
sc._jvm.PythonSQLUtils.internalFn( # type: ignore
- name, _to_seq(sc, cols, _to_java_column) # type: ignore
+ name,
+ _to_seq(sc, cols, _to_java_column), # type: ignore
)
)
diff --git a/python/pyspark/sql/pandas/group_ops.py b/python/pyspark/sql/pandas/group_ops.py
index b78db4c70313..bbbe55ef5037 100644
--- a/python/pyspark/sql/pandas/group_ops.py
+++ b/python/pyspark/sql/pandas/group_ops.py
@@ -948,9 +948,7 @@ class PandasGroupedOpsMixin:
eval_type = PythonEvalType.SQL_GROUPED_MAP_ARROW_UDF
# The usage of the pandas_udf is internal so type checking is disabled.
- udf = pandas_udf(
- func, returnType=schema, functionType=eval_type
- ) # type: ignore[call-overload]
+ udf = pandas_udf(func, returnType=schema, functionType=eval_type) # type: ignore[call-overload]
df = self._df
udf_column = udf(*[df[col] for col in df.columns])
jdf = self._jgd.flatMapGroupsInArrow(udf_column._jc)
diff --git a/python/pyspark/sql/pandas/serializers.py b/python/pyspark/sql/pandas/serializers.py
index b2e118c00e9e..01e4a8163489 100644
--- a/python/pyspark/sql/pandas/serializers.py
+++ b/python/pyspark/sql/pandas/serializers.py
@@ -296,9 +296,9 @@ class ArrowStreamArrowUDTFSerializer(ArrowStreamUDTFSerializer):
def apply_type_coercion():
for batch, arrow_return_type in iterator:
- assert isinstance(
- arrow_return_type, pa.StructType
- ), f"Expected pa.StructType, got {type(arrow_return_type)}"
+ assert isinstance(arrow_return_type, pa.StructType), (
+ f"Expected pa.StructType, got {type(arrow_return_type)}"
+ )
# Handle empty struct case specially
if batch.num_columns == 0:
diff --git a/python/pyspark/sql/pandas/typehints.py b/python/pyspark/sql/pandas/typehints.py
index 328d703b3889..4490748568e7 100644
--- a/python/pyspark/sql/pandas/typehints.py
+++ b/python/pyspark/sql/pandas/typehints.py
@@ -200,7 +200,9 @@ def infer_pandas_eval_type(
return None
-def infer_arrow_eval_type(sig: Signature, type_hints: Dict[str, Any]) -> Optional[
+def infer_arrow_eval_type(
+ sig: Signature, type_hints: Dict[str, Any]
+) -> Optional[
Union[
"ArrowScalarUDFType",
"ArrowScalarIterUDFType",
diff --git a/python/pyspark/sql/plot/core.py b/python/pyspark/sql/plot/core.py
index c3fdaeb3aa07..99ebcec29fe0 100644
--- a/python/pyspark/sql/plot/core.py
+++ b/python/pyspark/sql/plot/core.py
@@ -530,9 +530,9 @@ class PySparkKdePlotBase:
ind: Sequence[float],
) -> Column:
# refers to org.apache.spark.mllib.stat.KernelDensity
- assert bw_method is not None and isinstance(
- bw_method, (int, float)
- ), "'bw_method' must be set as a scalar number."
+ assert bw_method is not None and isinstance(bw_method, (int, float)), (
+ "'bw_method' must be set as a scalar number."
+ )
assert ind is not None, "'ind' must be a scalar array."
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index c62f508e72e3..bb8e51b60134 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -1160,9 +1160,9 @@ class DataFrameReader(OptionUtils):
if column is not None:
assert lowerBound is not None, "lowerBound can not be None when ``column`` is specified"
assert upperBound is not None, "upperBound can not be None when ``column`` is specified"
- assert (
- numPartitions is not None
- ), "numPartitions can not be None when ``column`` is specified"
+ assert numPartitions is not None, (
+ "numPartitions can not be None when ``column`` is specified"
+ )
return self._df(
self._jreader.jdbc(
url, table, column, int(lowerBound), int(upperBound), int(numPartitions), jprop
diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py
index 3b1415210baf..0942bad119f5 100644
--- a/python/pyspark/sql/session.py
+++ b/python/pyspark/sql/session.py
@@ -1645,7 +1645,10 @@ class SparkSession(SparkConversionMixin):
data, schema, samplingRatio, verifySchema
)
return self._create_dataframe(
- data, schema, samplingRatio, verifySchema # type: ignore[arg-type]
+ data,
+ schema, # type: ignore[arg-type]
+ samplingRatio,
+ verifySchema,
)
def _create_dataframe(
@@ -1689,7 +1692,8 @@ class SparkSession(SparkConversionMixin):
rdd, struct = self._createFromRDD(data.map(prepare), schema, samplingRatio)
else:
rdd, struct = self._createFromLocal(
- map(prepare, data), schema # type: ignore[arg-type]
+ map(prepare, data), # type: ignore[arg-type]
+ schema,
)
assert self._jvm is not None
jrdd = self._jvm.SerDeUtil.toJavaArray(rdd._to_java_object_rdd())
diff --git a/python/pyspark/sql/streaming/list_state_client.py b/python/pyspark/sql/streaming/list_state_client.py
index 89de69ed0625..509e1cfb60b8 100644
--- a/python/pyspark/sql/streaming/list_state_client.py
+++ b/python/pyspark/sql/streaming/list_state_client.py
@@ -57,9 +57,7 @@ class ListStateClient:
return False
else:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(
- f"Error checking value state exists: " f"{response_message[1]}"
- )
+ raise PySparkRuntimeError(f"Error checking value state exists: {response_message[1]}")
def get(self, state_name: str, iterator_id: str) -> Tuple[Tuple, bool]:
import pyspark.sql.streaming.proto.StateMessage_pb2 as stateMessage
@@ -125,7 +123,7 @@ class ListStateClient:
status = response_message[0]
if status != 0:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(f"Error updating value state: " f"{response_message[1]}")
+ raise PySparkRuntimeError(f"Error updating value state: {response_message[1]}")
def append_list(self, state_name: str, values: List[Tuple]) -> None:
import pyspark.sql.streaming.proto.StateMessage_pb2 as stateMessage
@@ -148,7 +146,7 @@ class ListStateClient:
status = response_message[0]
if status != 0:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(f"Error updating value state: " f"{response_message[1]}")
+ raise PySparkRuntimeError(f"Error updating value state: {response_message[1]}")
def put(self, state_name: str, values: List[Tuple]) -> None:
import pyspark.sql.streaming.proto.StateMessage_pb2 as stateMessage
@@ -170,7 +168,7 @@ class ListStateClient:
status = response_message[0]
if status != 0:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(f"Error updating value state: " f"{response_message[1]}")
+ raise PySparkRuntimeError(f"Error updating value state: {response_message[1]}")
def clear(self, state_name: str) -> None:
import pyspark.sql.streaming.proto.StateMessage_pb2 as stateMessage
@@ -185,7 +183,7 @@ class ListStateClient:
status = response_message[0]
if status != 0:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(f"Error clearing value state: " f"{response_message[1]}")
+ raise PySparkRuntimeError(f"Error clearing value state: {response_message[1]}")
class ListStateIterator:
diff --git a/python/pyspark/sql/streaming/listener.py b/python/pyspark/sql/streaming/listener.py
index 40d1bdee6043..effd4550d842 100644
--- a/python/pyspark/sql/streaming/listener.py
+++ b/python/pyspark/sql/streaming/listener.py
@@ -64,7 +64,8 @@ class StreamingQueryListener(ABC):
"""
def _set_spark_session(
- self, session: "SparkSession" # type: ignore[name-defined] # noqa: F821
+ self,
+ session: "SparkSession", # type: ignore[name-defined] # noqa: F821
) -> None:
if self.spark is None:
self.spark = session
@@ -134,10 +135,8 @@ class StreamingQueryListener(ABC):
if hasattr(self, "_jlistenerobj"):
return self._jlistenerobj
- self._jlistenerobj: "JavaObject" = (
- SparkContext._jvm.PythonStreamingQueryListenerWrapper( # type: ignore[union-attr]
- JStreamingQueryListener(self)
- )
+ self._jlistenerobj: "JavaObject" = SparkContext._jvm.PythonStreamingQueryListenerWrapper( # type: ignore[union-attr]
+ JStreamingQueryListener(self)
)
return self._jlistenerobj
diff --git a/python/pyspark/sql/streaming/map_state_client.py b/python/pyspark/sql/streaming/map_state_client.py
index f3cd550a8312..0484ef792cf3 100644
--- a/python/pyspark/sql/streaming/map_state_client.py
+++ b/python/pyspark/sql/streaming/map_state_client.py
@@ -283,7 +283,7 @@ class MapStateClient:
status = response_message[0]
if status != 0:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(f"Error clearing map state: " f"{response_message[1]}")
+ raise PySparkRuntimeError(f"Error clearing map state: {response_message[1]}")
class MapStateIterator:
diff --git a/python/pyspark/sql/streaming/proto/StateMessage_pb2.pyi b/python/pyspark/sql/streaming/proto/StateMessage_pb2.pyi
index affaccb4c17a..59a28e48a2f7 100644
--- a/python/pyspark/sql/streaming/proto/StateMessage_pb2.pyi
+++ b/python/pyspark/sql/streaming/proto/StateMessage_pb2.pyi
@@ -142,7 +142,9 @@ class StateRequest(google.protobuf.message.Message):
b"version",
],
) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["method", b"method"]) -> (
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["method", b"method"]
+ ) -> (
typing_extensions.Literal[
"statefulProcessorCall",
"stateVariableRequest",
@@ -502,7 +504,9 @@ class StatefulProcessorCall(google.protobuf.message.Message):
b"timerStateCall",
],
) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["method", b"method"]) -> (
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["method", b"method"]
+ ) -> (
typing_extensions.Literal[
"setHandleState",
"getValueState",
@@ -1100,7 +1104,9 @@ class MapStateCall(google.protobuf.message.Message):
b"values",
],
) -> None: ...
- def WhichOneof(self, oneof_group: typing_extensions.Literal["method", b"method"]) -> (
+ def WhichOneof(
+ self, oneof_group: typing_extensions.Literal["method", b"method"]
+ ) -> (
typing_extensions.Literal[
"exists",
"getValue",
diff --git a/python/pyspark/sql/streaming/stateful_processor_api_client.py b/python/pyspark/sql/streaming/stateful_processor_api_client.py
index 93bc11ed2b23..490ae184c273 100644
--- a/python/pyspark/sql/streaming/stateful_processor_api_client.py
+++ b/python/pyspark/sql/streaming/stateful_processor_api_client.py
@@ -113,7 +113,7 @@ class StatefulProcessorApiClient:
self.handle_state = state
else:
# TODO(SPARK-49233): Classify errors thrown by internal methods.
- raise PySparkRuntimeError(f"Error setting handle state: " f"{response_message[1]}")
+ raise PySparkRuntimeError(f"Error setting handle state: {response_message[1]}")
def set_implicit_key(self, key: Tuple) -> None:
import pyspark.sql.streaming.proto.StateMessage_pb2 as stateMessage
@@ -128,7 +128,7 @@ class StatefulProcessorApiClient:
status = response_message[0]
if status != 0:
# TODO(SPARK-49233): Classify errors thrown by internal methods.
- raise PySparkRuntimeError(f"Error setting implicit key: " f"{response_message[1]}")
+ raise PySparkRuntimeError(f"Error setting implicit key: {response_message[1]}")
def remove_implicit_key(self) -> None:
import pyspark.sql.streaming.proto.StateMessage_pb2 as stateMessage
@@ -142,7 +142,7 @@ class StatefulProcessorApiClient:
status = response_message[0]
if status != 0:
# TODO(SPARK-49233): Classify errors thrown by internal methods.
- raise PySparkRuntimeError(f"Error removing implicit key: " f"{response_message[1]}")
+ raise PySparkRuntimeError(f"Error removing implicit key: {response_message[1]}")
def get_value_state(
self, state_name: str, schema: Union[StructType, str], ttl_duration_ms: Optional[int]
@@ -165,7 +165,7 @@ class StatefulProcessorApiClient:
status = response_message[0]
if status != 0:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(f"Error initializing value state: " f"{response_message[1]}")
+ raise PySparkRuntimeError(f"Error initializing value state: {response_message[1]}")
def get_list_state(
self, state_name: str, schema: Union[StructType, str], ttl_duration_ms: Optional[int]
@@ -188,7 +188,7 @@ class StatefulProcessorApiClient:
status = response_message[0]
if status != 0:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(f"Error initializing list state: " f"{response_message[1]}")
+ raise PySparkRuntimeError(f"Error initializing list state: {response_message[1]}")
def register_timer(self, expiry_time_stamp_ms: int) -> None:
import pyspark.sql.streaming.proto.StateMessage_pb2 as stateMessage
@@ -203,7 +203,7 @@ class StatefulProcessorApiClient:
status = response_message[0]
if status != 0:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(f"Error register timer: " f"{response_message[1]}")
+ raise PySparkRuntimeError(f"Error register timer: {response_message[1]}")
def delete_timer(self, expiry_time_stamp_ms: int) -> None:
import pyspark.sql.streaming.proto.StateMessage_pb2 as stateMessage
@@ -218,7 +218,7 @@ class StatefulProcessorApiClient:
status = response_message[0]
if status != 0:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(f"Error deleting timer: " f"{response_message[1]}")
+ raise PySparkRuntimeError(f"Error deleting timer: {response_message[1]}")
def get_list_timer_row(self, iterator_id: str) -> Tuple[int, bool]:
import pyspark.sql.streaming.proto.StateMessage_pb2 as stateMessage
@@ -346,7 +346,7 @@ class StatefulProcessorApiClient:
status = response_message[0]
if status != 0:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(f"Error initializing map state: " f"{response_message[1]}")
+ raise PySparkRuntimeError(f"Error initializing map state: {response_message[1]}")
def delete_if_exists(self, state_name: str) -> None:
import pyspark.sql.streaming.proto.StateMessage_pb2 as stateMessage
@@ -361,7 +361,7 @@ class StatefulProcessorApiClient:
status = response_message[0]
if status != 0:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(f"Error deleting state: " f"{response_message[1]}")
+ raise PySparkRuntimeError(f"Error deleting state: {response_message[1]}")
def _get_batch_timestamp(self) -> int:
import pyspark.sql.streaming.proto.StateMessage_pb2 as stateMessage
@@ -378,9 +378,7 @@ class StatefulProcessorApiClient:
status = response_message[0]
if status != 0:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(
- f"Error getting processing timestamp: " f"{response_message[1]}"
- )
+ raise PySparkRuntimeError(f"Error getting processing timestamp: {response_message[1]}")
else:
timestamp = response_message[2]
return timestamp
@@ -398,9 +396,7 @@ class StatefulProcessorApiClient:
status = response_message[0]
if status != 0:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(
- f"Error getting eventtime timestamp: " f"{response_message[1]}"
- )
+ raise PySparkRuntimeError(f"Error getting eventtime timestamp: {response_message[1]}")
else:
timestamp = response_message[2]
return timestamp
@@ -581,7 +577,7 @@ class StatefulProcessorApiClient:
status = response_message[0]
if status != 0:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(f"Error parsing string schema: " f"{response_message[1]}")
+ raise PySparkRuntimeError(f"Error parsing string schema: {response_message[1]}")
else:
return StructType.fromJson(json.loads(response_message[2]))
diff --git a/python/pyspark/sql/streaming/value_state_client.py b/python/pyspark/sql/streaming/value_state_client.py
index 532a89cf92d2..6c8e0faffdfd 100644
--- a/python/pyspark/sql/streaming/value_state_client.py
+++ b/python/pyspark/sql/streaming/value_state_client.py
@@ -53,9 +53,7 @@ class ValueStateClient:
return False
else:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(
- f"Error checking value state exists: " f"{response_message[1]}"
- )
+ raise PySparkRuntimeError(f"Error checking value state exists: {response_message[1]}")
def get(self, state_name: str) -> Optional[Tuple]:
import pyspark.sql.streaming.proto.StateMessage_pb2 as stateMessage
@@ -75,7 +73,7 @@ class ValueStateClient:
return tuple(data)
else:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(f"Error getting value state: " f"{response_message[1]}")
+ raise PySparkRuntimeError(f"Error getting value state: {response_message[1]}")
def update(self, state_name: str, value: Tuple) -> None:
import pyspark.sql.streaming.proto.StateMessage_pb2 as stateMessage
@@ -93,7 +91,7 @@ class ValueStateClient:
status = response_message[0]
if status != 0:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(f"Error updating value state: " f"{response_message[1]}")
+ raise PySparkRuntimeError(f"Error updating value state: {response_message[1]}")
def clear(self, state_name: str) -> None:
import pyspark.sql.streaming.proto.StateMessage_pb2 as stateMessage
@@ -108,4 +106,4 @@ class ValueStateClient:
status = response_message[0]
if status != 0:
# TODO(SPARK-49233): Classify user facing errors.
- raise PySparkRuntimeError(f"Error clearing value state: " f"{response_message[1]}")
+ raise PySparkRuntimeError(f"Error clearing value state: {response_message[1]}")
diff --git a/python/pyspark/sql/tests/arrow/test_arrow.py b/python/pyspark/sql/tests/arrow/test_arrow.py
index db74f303e3d4..3e31ff637180 100644
--- a/python/pyspark/sql/tests/arrow/test_arrow.py
+++ b/python/pyspark/sql/tests/arrow/test_arrow.py
@@ -666,7 +666,7 @@ class ArrowTestsMixin:
self.assertEqual(len(exception.args), 1)
self.assertRegex(
exception.args[0],
- "with name '7_date_t' " "to Arrow Array \\(decimal128\\(38, 18\\)\\)",
+ "with name '7_date_t' to Arrow Array \\(decimal128\\(38, 18\\)\\)",
)
# the inner exception provides us with the incorrect types
diff --git a/python/pyspark/sql/tests/arrow/test_arrow_grouped_map.py b/python/pyspark/sql/tests/arrow/test_arrow_grouped_map.py
index 3f11c87f2d41..b29e4eaeef26 100644
--- a/python/pyspark/sql/tests/arrow/test_arrow_grouped_map.py
+++ b/python/pyspark/sql/tests/arrow/test_arrow_grouped_map.py
@@ -419,7 +419,7 @@ class ApplyInArrowTestsMixin:
[
Row(
level="WARNING",
- msg=f"arrow grouped map: {dict(id=lst, value=[v*10 for v in lst])}",
+ msg=f"arrow grouped map: {dict(id=lst, value=[v * 10 for v in lst])}",
context={"func_name": func_with_logging.__name__},
logger="test_arrow_grouped_map",
)
@@ -459,7 +459,7 @@ class ApplyInArrowTestsMixin:
[
Row(
level="WARNING",
- msg=f"arrow grouped map: {dict(id=lst, value=[v*10 for v in lst])}",
+ msg=f"arrow grouped map: {dict(id=lst, value=[v * 10 for v in lst])}",
context={"func_name": func_with_logging.__name__},
logger="test_arrow_grouped_map",
)
diff --git a/python/pyspark/sql/tests/arrow/test_arrow_udtf.py b/python/pyspark/sql/tests/arrow/test_arrow_udtf.py
index 052ee8537bee..4c3996e125d5 100644
--- a/python/pyspark/sql/tests/arrow/test_arrow_udtf.py
+++ b/python/pyspark/sql/tests/arrow/test_arrow_udtf.py
@@ -112,9 +112,9 @@ class ArrowUDTFTestsMixin:
@arrow_udtf(returnType="batch_id int, name string, count int")
class RecordBatchUDTF:
def eval(self, batch_size: "pa.Array") -> Iterator["pa.RecordBatch"]:
- assert isinstance(
- batch_size, pa.Array
- ), f"Expected pa.Array, got {type(batch_size)}"
+ assert isinstance(batch_size, pa.Array), (
+ f"Expected pa.Array, got {type(batch_size)}"
+ )
size = batch_size[0].as_py()
@@ -518,9 +518,9 @@ class ArrowUDTFTestsMixin:
@arrow_udtf(returnType="filtered_id bigint") # Use bigint to match int64
class TableArgUDTF:
def eval(self, table_data: "pa.RecordBatch") -> Iterator["pa.Table"]:
- assert isinstance(
- table_data, pa.RecordBatch
- ), f"Expected pa.RecordBatch, got {type(table_data)}"
+ assert isinstance(table_data, pa.RecordBatch), (
+ f"Expected pa.RecordBatch, got {type(table_data)}"
+ )
# Convert record batch to table to work with it more easily
table = pa.table(table_data)
@@ -555,12 +555,12 @@ class ArrowUDTFTestsMixin:
def eval(
self, table_data: "pa.RecordBatch", threshold: "pa.Array"
) -> Iterator["pa.Table"]:
- assert isinstance(
- threshold, pa.Array
- ), f"Expected pa.Array for threshold, got {type(threshold)}"
- assert isinstance(
- table_data, pa.RecordBatch
- ), f"Expected pa.RecordBatch for table_data, got {type(table_data)}"
+ assert isinstance(threshold, pa.Array), (
+ f"Expected pa.Array for threshold, got {type(threshold)}"
+ )
+ assert isinstance(table_data, pa.RecordBatch), (
+ f"Expected pa.RecordBatch for table_data, got {type(table_data)}"
+ )
threshold_val = threshold[0].as_py()
@@ -751,9 +751,9 @@ class ArrowUDTFTestsMixin:
def eval(self, table_data: "pa.RecordBatch") -> Iterator["pa.Table"]:
table = pa.table(table_data)
partition_key = pc.unique(table["partition_key"]).to_pylist()
- assert (
- len(partition_key) == 1
- ), f"Expected exactly one partition key, got {partition_key}"
+ assert len(partition_key) == 1, (
+ f"Expected exactly one partition key, got {partition_key}"
+ )
self._partition_key = partition_key[0]
self._sum += pc.sum(table["value"]).as_py()
# Don't yield here - accumulate and yield in terminate
@@ -1025,9 +1025,9 @@ class ArrowUDTFTestsMixin:
if self._product is None:
self._product = product
else:
- assert (
- self._product == product
- ), f"Mixed products {self._product} and {product} in partition"
+ assert self._product == product, (
+ f"Mixed products {self._product} and {product} in partition"
+ )
self._batches.append(table)
self._seen += table.num_rows
@@ -1124,7 +1124,8 @@ class ArrowUDTFTestsMixin:
# Create DataFrame with 5 input partitions but all data will map to partition_key=1
# range(1, 10, 1, 5) creates ids from 1 to 9 with 5 partitions
input_df = self.spark.range(1, 10, 1, 5).selectExpr(
- "1 as partition_key", "id" # constant partition key
+ "1 as partition_key",
+ "id", # constant partition key
)
self.spark.udtf.register("single_partition_udtf", SinglePartitionUDTF)
@@ -1511,12 +1512,12 @@ class ArrowUDTFTestsMixin:
def eval(
self, threshold: "pa.Array", table_data: "pa.RecordBatch"
) -> Iterator["pa.Table"]:
- assert isinstance(
- threshold, pa.Array
- ), f"Expected pa.Array for threshold, got {type(threshold)}"
- assert isinstance(
- table_data, pa.RecordBatch
- ), f"Expected pa.RecordBatch for table_data, got {type(table_data)}"
+ assert isinstance(threshold, pa.Array), (
+ f"Expected pa.Array for threshold, got {type(threshold)}"
+ )
+ assert isinstance(table_data, pa.RecordBatch), (
+ f"Expected pa.RecordBatch for table_data, got {type(table_data)}"
+ )
threshold_val = threshold[0].as_py()
@@ -1557,15 +1558,15 @@ class ArrowUDTFTestsMixin:
table_data: "pa.RecordBatch",
max_threshold: "pa.Array",
) -> Iterator["pa.Table"]:
- assert isinstance(
- min_threshold, pa.Array
- ), f"Expected pa.Array for min_threshold, got {type(min_threshold)}"
- assert isinstance(
- table_data, pa.RecordBatch
- ), f"Expected pa.RecordBatch for table_data, got {type(table_data)}"
- assert isinstance(
- max_threshold, pa.Array
- ), f"Expected pa.Array for max_threshold, got {type(max_threshold)}"
+ assert isinstance(min_threshold, pa.Array), (
+ f"Expected pa.Array for min_threshold, got {type(min_threshold)}"
+ )
+ assert isinstance(table_data, pa.RecordBatch), (
+ f"Expected pa.RecordBatch for table_data, got {type(table_data)}"
+ )
+ assert isinstance(max_threshold, pa.Array), (
+ f"Expected pa.Array for max_threshold, got {type(max_threshold)}"
+ )
min_val = min_threshold[0].as_py()
max_val = max_threshold[0].as_py()
@@ -1606,12 +1607,12 @@ class ArrowUDTFTestsMixin:
def eval(
self, table_data: "pa.RecordBatch", multiplier: "pa.Array"
) -> Iterator["pa.Table"]:
- assert isinstance(
- table_data, pa.RecordBatch
- ), f"Expected pa.RecordBatch for table_data, got {type(table_data)}"
- assert isinstance(
- multiplier, pa.Array
- ), f"Expected pa.Array for multiplier, got {type(multiplier)}"
+ assert isinstance(table_data, pa.RecordBatch), (
+ f"Expected pa.RecordBatch for table_data, got {type(table_data)}"
+ )
+ assert isinstance(multiplier, pa.Array), (
+ f"Expected pa.Array for multiplier, got {type(multiplier)}"
+ )
multiplier_val = multiplier[0].as_py()
@@ -1673,9 +1674,9 @@ class ArrowUDTFTestsMixin:
@arrow_udtf(returnType="id bigint, doubled bigint")
class TestArrowUDTFWithLogging:
def eval(self, table_data: "pa.RecordBatch") -> Iterator["pa.Table"]:
- assert isinstance(
- table_data, pa.RecordBatch
- ), f"Expected pa.RecordBatch, got {type(table_data)}"
+ assert isinstance(table_data, pa.RecordBatch), (
+ f"Expected pa.RecordBatch, got {type(table_data)}"
+ )
logger = logging.getLogger("test_arrow_udtf")
logger.warning(f"arrow udtf: {table_data.to_pydict()}")
diff --git a/python/pyspark/sql/tests/connect/test_connect_collection.py b/python/pyspark/sql/tests/connect/test_connect_collection.py
index 0c19d13d4a3f..555058513efa 100644
--- a/python/pyspark/sql/tests/connect/test_connect_collection.py
+++ b/python/pyspark/sql/tests/connect/test_connect_collection.py
@@ -358,9 +358,9 @@ class SparkConnectCollectionTests(ReusedMixedTestCase, PandasOnSparkTestUtils):
count = 0
for row in iterator:
# This will raise an exception if the type is not as expected
- assert isinstance(
- row.b, expected_type
- ), f"Expected {expected_type_name}, got {type(row.b).__name__}"
+ assert isinstance(row.b, expected_type), (
+ f"Expected {expected_type_name}, got {type(row.b).__name__}"
+ )
count += 1
# Ensure we actually processed rows
assert count > 0, "No rows were processed"
diff --git a/python/pyspark/sql/tests/connect/test_connect_stat.py b/python/pyspark/sql/tests/connect/test_connect_stat.py
index f4bf21158d86..6239fde6e76b 100644
--- a/python/pyspark/sql/tests/connect/test_connect_stat.py
+++ b/python/pyspark/sql/tests/connect/test_connect_stat.py
@@ -213,7 +213,7 @@ class SparkConnectStatTests(SparkConnectSQLTestCase):
},
)
with self.assertRaises(ValueError) as context:
- self.connect.read.table(self.tbl_name2).stat.corr("col1", "col3", "spearman"),
+ (self.connect.read.table(self.tbl_name2).stat.corr("col1", "col3", "spearman"),)
self.assertTrue(
"Currently only the calculation of the Pearson Correlation "
+ "coefficient is supported."
diff --git a/python/pyspark/sql/tests/connect/test_parity_udtf.py b/python/pyspark/sql/tests/connect/test_parity_udtf.py
index 4169f4b1f1f4..2ab0e733a4be 100644
--- a/python/pyspark/sql/tests/connect/test_parity_udtf.py
+++ b/python/pyspark/sql/tests/connect/test_parity_udtf.py
@@ -58,7 +58,7 @@ class UDTFParityTests(BaseUDTFTestsMixin, ReusedConnectTestCase):
@udtf(returnType="int")
class TestUDTF:
def eval(self, a: int):
- yield a + 1,
+ yield (a + 1,)
with self.assertRaisesRegex(InvalidPlanInput, "Invalid.*type"):
TestUDTF(lit(1)).collect()
@@ -115,7 +115,7 @@ class LegacyArrowUDTFParityTests(LegacyUDTFArrowTestsMixin, UDTFParityTests):
class TestUDTF:
def eval(self):
df.collect()
- yield 1,
+ yield (1,)
with self.assertRaisesRegex(PythonException, "NO_ACTIVE_SESSION"):
TestUDTF().collect()
@@ -149,7 +149,7 @@ class ArrowUDTFParityTests(UDTFArrowTestsMixin, UDTFParityTests):
class TestUDTF:
def eval(self):
df.collect()
- yield 1,
+ yield (1,)
with self.assertRaisesRegex(PythonException, "NO_ACTIVE_SESSION"):
TestUDTF().collect()
diff --git a/python/pyspark/sql/tests/pandas/helper/helper_pandas_transform_with_state.py b/python/pyspark/sql/tests/pandas/helper/helper_pandas_transform_with_state.py
index df672890b4df..6a5b65ebf958 100644
--- a/python/pyspark/sql/tests/pandas/helper/helper_pandas_transform_with_state.py
+++ b/python/pyspark/sql/tests/pandas/helper/helper_pandas_transform_with_state.py
@@ -1050,14 +1050,14 @@ class PandasListStateLargeListProcessor(StatefulProcessor):
list_size = self.list_size_state.get()
assert list_size is not None
list_size = list_size[0]
- assert list_size == len(
- elements
- ), f"list_size ({list_size}) != len(elements) ({len(elements)})"
+ assert list_size == len(elements), (
+ f"list_size ({list_size}) != len(elements) ({len(elements)})"
+ )
expected_elements_in_state = [(i, None) for i in range(list_size)]
- assert (
- elements == expected_elements_in_state
- ), f"expected {expected_elements_in_state} but got {elements}"
+ assert elements == expected_elements_in_state, (
+ f"expected {expected_elements_in_state} but got {elements}"
+ )
if key == ("0",):
# Use the operation `put`
@@ -1117,14 +1117,14 @@ class RowListStateLargeListProcessor(StatefulProcessor):
list_size = self.list_size_state.get()
assert list_size is not None
list_size = list_size[0]
- assert list_size == len(
- elements
- ), f"list_size ({list_size}) != len(elements) ({len(elements)})"
+ assert list_size == len(elements), (
+ f"list_size ({list_size}) != len(elements) ({len(elements)})"
+ )
expected_elements_in_state = [(i, None) for i in range(list_size)]
- assert (
- elements == expected_elements_in_state
- ), f"expected {expected_elements_in_state} but got {elements}"
+ assert elements == expected_elements_in_state, (
+ f"expected {expected_elements_in_state} but got {elements}"
+ )
if key == ("0",):
# Use the operation `put`
@@ -1767,9 +1767,9 @@ class PandasStatefulProcessorCompositeType(StatefulProcessor):
ids, tags, metadata, address = self.obj_state.get()
assert tags == self.TAGS, f"Tag mismatch: {tags}"
assert metadata == [Row(**m) for m in self.METADATA], f"Metadata mismatch: {metadata}"
- assert address == [
- Row(**e._asdict()) for e in self.ADDRESS
- ], f"Address mismatch: {address}"
+ assert address == [Row(**e._asdict()) for e in self.ADDRESS], (
+ f"Address mismatch: {address}"
+ )
ids = [int(x + total_temperature) for x in ids]
else:
ids = [0]
@@ -1890,9 +1890,9 @@ class RowStatefulProcessorCompositeType(StatefulProcessor):
ids, tags, metadata, address = self.obj_state.get()
assert tags == self.TAGS, f"Tag mismatch: {tags}"
assert metadata == [Row(**m) for m in self.METADATA], f"Metadata mismatch: {metadata}"
- assert address == [
- Row(**e._asdict()) for e in self.ADDRESS
- ], f"Address mismatch: {address}"
+ assert address == [Row(**e._asdict()) for e in self.ADDRESS], (
+ f"Address mismatch: {address}"
+ )
ids = [int(x + total_temperature) for x in ids]
else:
ids = [0]
diff --git a/python/pyspark/sql/tests/pandas/streaming/test_pandas_transform_with_state_state_variable.py b/python/pyspark/sql/tests/pandas/streaming/test_pandas_transform_with_state_state_variable.py
index 52e95e93c5fe..01959ed02eab 100644
--- a/python/pyspark/sql/tests/pandas/streaming/test_pandas_transform_with_state_state_variable.py
+++ b/python/pyspark/sql/tests/pandas/streaming/test_pandas_transform_with_state_state_variable.py
@@ -706,9 +706,7 @@ class TransformWithStateStateVariableTestsMixin:
"id AS groupingKey",
"map_key.name AS mapKey",
"map_value.value.count AS mapValue",
- ).sort(
- "groupingKey"
- ).collect() == [
+ ).sort("groupingKey").collect() == [
Row(groupingKey="0", mapKey="key2", mapValue=2),
Row(groupingKey="1", mapKey="key2", mapValue=2),
]
@@ -801,9 +799,7 @@ class TransformWithStateStateVariableTestsMixin:
"key.id AS groupingKey", "list_value.temperature AS valueList"
).sort("groupingKey").withColumn(
"valueSortedList", array_sort(col("valueList"))
- ).select(
- "groupingKey", "valueSortedList"
- ).collect() == [
+ ).select("groupingKey", "valueSortedList").collect() == [
Row(groupingKey="0", valueSortedList=[20, 20, 120, 120, 222]),
Row(groupingKey="1", valueSortedList=[20, 20, 120, 120, 222]),
]
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py b/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py
index 1c0e9d6de037..e5da8b0cc040 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py
@@ -1009,7 +1009,7 @@ class ApplyInPandasTestsMixin:
[
Row(
level="WARNING",
- msg=f"pandas grouped map: {dict(id=lst, value=[v*10 for v in lst])}",
+ msg=f"pandas grouped map: {dict(id=lst, value=[v * 10 for v in lst])}",
context={"func_name": func_with_logging.__name__},
logger="test_pandas_grouped_map",
)
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_udf_grouped_agg.py b/python/pyspark/sql/tests/pandas/test_pandas_udf_grouped_agg.py
index a765c8f40575..73a44cacd5aa 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_udf_grouped_agg.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_udf_grouped_agg.py
@@ -1129,9 +1129,9 @@ class GroupedAggPandasUDFTestsMixin:
for person_series in it:
# Currently struct types are passed as Series of dicts
# In the future, they should be passed as pd.DataFrame (like scalar pandas UDFs)
- assert isinstance(
- person_series, pd.Series
- ), f"Expected Series, got {type(person_series)}"
+ assert isinstance(person_series, pd.Series), (
+ f"Expected Series, got {type(person_series)}"
+ )
# Extract age values from dicts
ages = [p["age"] for p in person_series]
total_age += sum(ages)
diff --git a/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py b/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py
index 2aecb5efe7f8..37bd6fd334a4 100644
--- a/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py
+++ b/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py
@@ -688,7 +688,7 @@ class ScalarPandasUDFTestsMixin:
yield pd.Series(1)
with self.assertRaisesRegex(
- Exception, "The length of output in Scalar iterator.*" "the length of output was 1"
+ Exception, "The length of output in Scalar iterator.*the length of output was 1"
):
df.select(iter_udf_wong_output_size(col("id"))).collect()
diff --git a/python/pyspark/sql/tests/test_dataframe.py b/python/pyspark/sql/tests/test_dataframe.py
index 9a7218ef8254..01219585424a 100644
--- a/python/pyspark/sql/tests/test_dataframe.py
+++ b/python/pyspark/sql/tests/test_dataframe.py
@@ -885,9 +885,7 @@ class DataFrameTestsMixin:
[(1, "Alice"), (2, "Bob")], ["id", "name"]
).write.mode("overwrite").saveAsTable("testcat.ns1.target")
- source = self.spark.createDataFrame(
- [(1, "Charlie"), (3, "David")], ["id", "name"]
- ) # type: DataFrame
+ source = self.spark.createDataFrame([(1, "Charlie"), (3, "David")], ["id", "name"]) # type: DataFrame
from pyspark.sql.functions import col
diff --git a/python/pyspark/sql/tests/test_python_datasource.py b/python/pyspark/sql/tests/test_python_datasource.py
index 292d64725008..5a173d3600cd 100644
--- a/python/pyspark/sql/tests/test_python_datasource.py
+++ b/python/pyspark/sql/tests/test_python_datasource.py
@@ -88,7 +88,7 @@ class BasePythonDataSourceTestsMixin:
def test_basic_data_source_reader_class(self):
class MyDataSourceReader(DataSourceReader):
def read(self, partition):
- yield None,
+ yield (None,)
reader = MyDataSourceReader()
self.assertEqual(list(reader.read(None)), [(None,)])
@@ -895,7 +895,7 @@ class BasePythonDataSourceTestsMixin:
class TestReader2(DataSourceReader):
def read(self, partition):
ctypes.string_at(0)
- yield "x",
+ yield ("x",)
self.spark.dataSource.register(TestDataSource)
diff --git a/python/pyspark/sql/tests/test_python_streaming_datasource.py b/python/pyspark/sql/tests/test_python_streaming_datasource.py
index eecbd5c455e2..f6bfbdc655f5 100644
--- a/python/pyspark/sql/tests/test_python_streaming_datasource.py
+++ b/python/pyspark/sql/tests/test_python_streaming_datasource.py
@@ -227,9 +227,9 @@ class BasePythonStreamingDataSourceTestsMixin:
if isinstance(limit, ReadAllAvailable):
end_offset = start_idx + 10
else:
- assert isinstance(
- limit, ReadMaxRows
- ), "Expected ReadMaxRows read limit but got " + str(type(limit))
+ assert isinstance(limit, ReadMaxRows), (
+ "Expected ReadMaxRows read limit but got " + str(type(limit))
+ )
end_offset = start_idx + limit.max_rows
return {"partition-1": end_offset}
@@ -266,9 +266,9 @@ class BasePythonStreamingDataSourceTestsMixin:
if isinstance(limit, ReadAllAvailable):
end_offset = start_idx + 10
else:
- assert isinstance(
- limit, ReadMaxRows
- ), "Expected ReadMaxRows read limit but got " + str(type(limit))
+ assert isinstance(limit, ReadMaxRows), (
+ "Expected ReadMaxRows read limit but got " + str(type(limit))
+ )
end_offset = min(
start_idx + limit.max_rows, self.desired_end_offset["partition-1"]
)
diff --git a/python/pyspark/sql/tests/test_udtf.py b/python/pyspark/sql/tests/test_udtf.py
index 81aec7b56a62..d567687720b2 100644
--- a/python/pyspark/sql/tests/test_udtf.py
+++ b/python/pyspark/sql/tests/test_udtf.py
@@ -90,7 +90,7 @@ class BaseUDTFTestsMixin:
def test_udtf_yield_single_row_col(self):
class TestUDTF:
def eval(self, a: int):
- yield a,
+ yield (a,)
func = udtf(TestUDTF, returnType="a: int")
assertDataFrameEqual(func(lit(1)), [Row(a=1)])
@@ -106,8 +106,8 @@ class BaseUDTFTestsMixin:
def test_udtf_yield_multi_rows(self):
class TestUDTF:
def eval(self, a: int):
- yield a,
- yield a + 1,
+ yield (a,)
+ yield (a + 1,)
func = udtf(TestUDTF, returnType="a: int")
assertDataFrameEqual(func(lit(1)), [Row(a=1), Row(a=2)])
@@ -199,8 +199,8 @@ class BaseUDTFTestsMixin:
@udtf(returnType="a: int")
class TestUDTF:
def eval(self):
- yield 1,
- yield 2,
+ yield (1,)
+ yield (2,)
assertDataFrameEqual(
self.spark.range(3, numPartitions=1).lateralJoin(TestUDTF()),
@@ -218,7 +218,7 @@ class BaseUDTFTestsMixin:
class TestUDTF:
def eval(self, i: int):
for n in range(i):
- yield n,
+ yield (n,)
assertDataFrameEqual(
self.spark.range(3, numPartitions=1).lateralJoin(TestUDTF(col("id").outer())),
@@ -275,7 +275,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType="a: struct<b: int, c: int>")
class TestUDTF:
def eval(self, a: int):
- yield (a, a + 1),
+ yield ((a, a + 1),)
assertDataFrameEqual(TestUDTF(lit(1)), [Row(a=Row(b=1, c=2))])
@@ -291,7 +291,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType=StructType().add("point", ExamplePointUDT()))
class TestUDTF:
def eval(self, x: float, y: float):
- yield ExamplePoint(x=x * 10, y=y * 10),
+ yield (ExamplePoint(x=x * 10, y=y * 10),)
assertDataFrameEqual(
TestUDTF(lit(1.0), lit(2.0)), [Row(point=ExamplePoint(x=10.0, y=20.0))]
@@ -358,7 +358,7 @@ class BaseUDTFTestsMixin:
class TestUDTF:
def eval(self, a: int):
if a > 5:
- yield a,
+ yield (a,)
func = udtf(TestUDTF, returnType="a: int")
self.spark.udtf.register("test_udtf", func)
@@ -372,7 +372,7 @@ class BaseUDTFTestsMixin:
class TestUDTF:
def eval(self, a: int):
if a > 5:
- yield a,
+ yield (a,)
self.spark.udtf.register("test_udtf", TestUDTF)
@@ -393,8 +393,8 @@ class BaseUDTFTestsMixin:
@udtf(returnType="a: int")
class TestUDTF:
def eval(self, a: int):
- yield a,
- yield None,
+ yield (a,)
+ yield (None,)
assertDataFrameEqual(TestUDTF(lit(1)), [Row(a=1), Row(a=None)])
df = self.spark.createDataFrame([(0, 1), (1, 2)], schema=["a", "b"])
@@ -407,7 +407,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType="a: int")
class TestUDTF:
def eval(self, a: int):
- yield a,
+ yield (a,)
assertDataFrameEqual(TestUDTF(lit(None)), [Row(a=None)])
self.spark.udtf.register("testUDTF", TestUDTF)
@@ -437,7 +437,7 @@ class BaseUDTFTestsMixin:
def __init__(self, a: int): ...
def eval(self, a: int):
- yield a,
+ yield (a,)
with self.assertRaisesRegex(PythonException, r".*constructor has more than one argument.*"):
TestUDTF(lit(1)).show()
@@ -446,7 +446,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType="x int")
class TestUDTF:
def eval(self, a: int):
- yield a,
+ yield (a,)
def terminate(self, a: int): ...
@@ -460,7 +460,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType="a: int, b: int")
class TestUDTF:
def eval(self, a: int):
- yield a,
+ yield (a,)
with self.assertRaisesRegex(PythonException, "UDTF_RETURN_SCHEMA_MISMATCH"):
TestUDTF(lit(1)).collect()
@@ -478,7 +478,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType=StructType())
class TestUDTF:
def eval(self):
- yield 1,
+ yield (1,)
with self.assertRaisesRegex(PythonException, "UDTF_RETURN_SCHEMA_MISMATCH"):
TestUDTF().collect()
@@ -605,7 +605,7 @@ class BaseUDTFTestsMixin:
raise Exception("error")
def eval(self):
- yield 1,
+ yield (1,)
with self.assertRaisesRegex(
PythonException,
@@ -661,7 +661,7 @@ class BaseUDTFTestsMixin:
yield a, a + 1
def terminate(self):
- yield 1,
+ yield (1,)
with self.assertRaisesRegex(PythonException, "UDTF_RETURN_SCHEMA_MISMATCH"):
TestUDTF(lit(1)).show()
@@ -669,7 +669,7 @@ class BaseUDTFTestsMixin:
def test_udtf_determinism(self):
class TestUDTF:
def eval(self, a: int):
- yield a,
+ yield (a,)
func = udtf(TestUDTF, returnType="x: int")
# The UDTF is marked as non-deterministic by default.
@@ -682,7 +682,7 @@ class BaseUDTFTestsMixin:
class RandomUDTF:
def eval(self, a: int):
- yield a + int(random.random()),
+ yield (a + int(random.random()),)
random_udtf = udtf(RandomUDTF, returnType="x: int")
assertDataFrameEqual(random_udtf(lit(1)), [Row(x=1)])
@@ -695,7 +695,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType="x: int")
class TestUDTF:
def eval(self, a: int):
- yield 1 if a > 100 else 0,
+ yield (1 if a > 100 else 0,)
assertDataFrameEqual(TestUDTF(rand(0) * 100), [Row(x=0)])
@@ -703,7 +703,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType="int")
class TestUDTF:
def eval(self, a: int):
- yield a + 1,
+ yield (a + 1,)
with self.assertRaises(PySparkTypeError) as e:
TestUDTF(lit(1)).collect()
@@ -717,7 +717,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType=MapType(StringType(), IntegerType()))
class TestUDTF:
def eval(self, a: int):
- yield a + 1,
+ yield (a + 1,)
with self.assertRaises(PySparkTypeError) as e:
TestUDTF(lit(1)).collect()
@@ -735,7 +735,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType="x: string")
class TestUDTF:
def eval(self, person):
- yield f"{person.name}: {person.age}",
+ yield (f"{person.name}: {person.age}",)
self.spark.udtf.register("test_udtf", TestUDTF)
assertDataFrameEqual(
@@ -747,7 +747,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType="x: string")
class TestUDTF:
def eval(self, args):
- yield str(args),
+ yield (str(args),)
self.spark.udtf.register("test_udtf", TestUDTF)
self.assertIn(
@@ -759,7 +759,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType="x: string")
class TestUDTF:
def eval(self, m):
- yield str(m),
+ yield (str(m),)
self.spark.udtf.register("test_udtf", TestUDTF)
assertDataFrameEqual(
@@ -771,7 +771,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType="x: struct<a:int,b:int>")
class TestUDTF:
def eval(self, x: int):
- yield {"a": x, "b": x + 1},
+ yield ({"a": x, "b": x + 1},)
assertDataFrameEqual(TestUDTF(lit(1)), [Row(x=Row(a=1, b=2))])
@@ -779,7 +779,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType="x: array<int>")
class TestUDTF:
def eval(self, x: int):
- yield [x, x + 1, x + 2],
+ yield ([x, x + 1, x + 2],)
assertDataFrameEqual(TestUDTF(lit(1)), [Row(x=[1, 2, 3])])
@@ -787,7 +787,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType="x: map<int,string>")
class TestUDTF:
def eval(self, x: int):
- yield {x: str(x)},
+ yield ({x: str(x)},)
assertDataFrameEqual(TestUDTF(lit(1)), [Row(x={1: "1"})])
@@ -917,7 +917,7 @@ class BaseUDTFTestsMixin:
def test_numeric_output_type_casting(self):
class TestUDTF:
def eval(self):
- yield 1,
+ yield (1,)
for i, (ret_type, expected) in enumerate(
[
@@ -951,7 +951,7 @@ class BaseUDTFTestsMixin:
def test_numeric_string_output_type_casting(self):
class TestUDTF:
def eval(self):
- yield "1",
+ yield ("1",)
for ret_type, expected in [
("x: boolean", [Row(x=None)]),
@@ -977,7 +977,7 @@ class BaseUDTFTestsMixin:
def test_string_output_type_casting(self):
class TestUDTF:
def eval(self):
- yield "hello",
+ yield ("hello",)
for ret_type, expected in [
("x: boolean", [Row(x=None)]),
@@ -1003,7 +1003,7 @@ class BaseUDTFTestsMixin:
def test_array_output_type_casting(self):
class TestUDTF:
def eval(self):
- yield [0, 1.1, 2],
+ yield ([0, 1.1, 2],)
for ret_type, expected in [
("x: boolean", [Row(x=None)]),
@@ -1033,7 +1033,7 @@ class BaseUDTFTestsMixin:
def test_map_output_type_casting(self):
class TestUDTF:
def eval(self):
- yield {"a": 0, "b": 1.1, "c": 2},
+ yield ({"a": 0, "b": 1.1, "c": 2},)
for ret_type, expected in [
("x: boolean", [Row(x=None)]),
@@ -1063,7 +1063,7 @@ class BaseUDTFTestsMixin:
def test_struct_output_type_casting_dict(self):
class TestUDTF:
def eval(self):
- yield {"a": 0, "b": 1.1, "c": 2},
+ yield ({"a": 0, "b": 1.1, "c": 2},)
for ret_type, expected in [
("x: boolean", [Row(x=None)]),
@@ -1096,7 +1096,7 @@ class BaseUDTFTestsMixin:
def check_struct_output_type_casting_row(self, error_type):
class TestUDTF:
def eval(self):
- yield Row(a=0, b=1.1, c=2),
+ yield (Row(a=0, b=1.1, c=2),)
err = ("PickleException", error_type)
@@ -1131,8 +1131,8 @@ class BaseUDTFTestsMixin:
def test_inconsistent_output_types(self):
class TestUDTF:
def eval(self):
- yield 1,
- yield [1, 2],
+ yield (1,)
+ yield ([1, 2],)
for ret_type, expected in [
("x: int", [Row(x=1), Row(x=None)]),
@@ -1197,7 +1197,7 @@ class BaseUDTFTestsMixin:
class TestUDTF:
def eval(self):
file_obj
- yield 1,
+ yield (1,)
with self.assertRaisesRegex(PySparkPicklingError, "UDTF_SERIALIZATION_ERROR"):
TestUDTF().collect()
@@ -1209,7 +1209,7 @@ class BaseUDTFTestsMixin:
class TestUDTF:
def eval(self):
df.collect()
- yield 1,
+ yield (1,)
with self.assertRaisesRegex(PySparkPicklingError, "UDTF_SERIALIZATION_ERROR"):
TestUDTF().collect()
@@ -1233,7 +1233,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType="a: int")
def test_udtf(a: int):
- yield a,
+ yield (a,)
self.check_error(
exception=e.exception,
@@ -1271,7 +1271,7 @@ class BaseUDTFTestsMixin:
class TestUDTF:
def eval(self, row: Row):
if row["id"] > 5:
- yield row["id"],
+ yield (row["id"],)
func = udtf(TestUDTF, returnType="a: int")
return func
@@ -1382,7 +1382,7 @@ class BaseUDTFTestsMixin:
class TestUDTF:
def eval(self, i: int, row: Row):
if row["id"] > i:
- yield row["id"],
+ yield (row["id"],)
func = udtf(TestUDTF, returnType="a: int")
self.spark.udtf.register("test_udtf", func)
@@ -1406,7 +1406,7 @@ class BaseUDTFTestsMixin:
class TestUDTF:
def eval(self, i: int, row: Row):
if row["id"] > i:
- yield row["id"],
+ yield (row["id"],)
func = udtf(TestUDTF, returnType="a: int")
self.spark.udtf.register("test_udtf", func)
@@ -1532,7 +1532,7 @@ class BaseUDTFTestsMixin:
def eval(self, x: int):
"""Evaluate the input row."""
- yield x + 1,
+ yield (x + 1,)
def terminate(self):
"""Terminate the UDTF."""
@@ -1573,7 +1573,7 @@ class BaseUDTFTestsMixin:
return AnalyzeResult(StructType().add("a", a.dataType))
def eval(self, a):
- yield a,
+ yield (a,)
func = udtf(TestUDTF)
self.spark.udtf.register("test_udtf", func)
@@ -1728,7 +1728,7 @@ class BaseUDTFTestsMixin:
def eval(self, a: Row):
if a["id"] > 5:
- yield a["id"],
+ yield (a["id"],)
func = udtf(TestUDTF)
self.spark.udtf.register("test_udtf", func)
@@ -1845,7 +1845,7 @@ class BaseUDTFTestsMixin:
def eval(self, row: Row):
assert "value" not in row
- yield row["id"],
+ yield (row["id"],)
df = self.spark.createDataFrame([(1, "a"), (2, "b"), (3, "c")], ["id", "value"])
assertDataFrameEqual(
@@ -2001,7 +2001,7 @@ class BaseUDTFTestsMixin:
return AnalyzeResult(StructType().add("a", a.dataType))
def eval(self, a):
- yield a,
+ yield (a,)
func = udtf(TestUDTF)
@@ -2064,11 +2064,11 @@ class BaseUDTFTestsMixin:
def eval(self, a):
assert colname.value == "col1"
- yield a,
+ yield (a,)
def terminate(self):
assert colname.value == "col1"
- yield 100,
+ yield (100,)
self.spark.udtf.register("test_udtf", TestUDTF)
@@ -2089,11 +2089,11 @@ class BaseUDTFTestsMixin:
def eval(self, a):
test_accum.add(10)
- yield a,
+ yield (a,)
def terminate(self):
test_accum.add(100)
- yield 100,
+ yield (100,)
self.spark.udtf.register("test_udtf", TestUDTF)
@@ -2128,11 +2128,11 @@ class BaseUDTFTestsMixin:
def eval(self, a):
assert TestUDTF.call_my_func() == "col1"
- yield a,
+ yield (a,)
def terminate(self):
assert TestUDTF.call_my_func() == "col1"
- yield 100,
+ yield (100,)
test_udtf = udtf(TestUDTF)
self.spark.udtf.register("test_udtf", test_udtf)
@@ -2168,11 +2168,11 @@ class BaseUDTFTestsMixin:
def eval(self, a):
assert TestUDTF.call_my_func() == "col1"
- yield a,
+ yield (a,)
def terminate(self):
assert TestUDTF.call_my_func() == "col1"
- yield 100,
+ yield (100,)
test_udtf = udtf(TestUDTF)
self.spark.udtf.register("test_udtf", test_udtf)
@@ -2218,11 +2218,11 @@ class BaseUDTFTestsMixin:
def eval(self, a):
assert TestUDTF.read_my_archive() == "col1"
- yield a,
+ yield (a,)
def terminate(self):
assert TestUDTF.read_my_archive() == "col1"
- yield 100,
+ yield (100,)
test_udtf = udtf(TestUDTF)
self.spark.udtf.register("test_udtf", test_udtf)
@@ -2262,11 +2262,11 @@ class BaseUDTFTestsMixin:
def eval(self, a):
assert TestUDTF.read_my_file() == "col1"
- yield a,
+ yield (a,)
def terminate(self):
assert TestUDTF.read_my_file() == "col1"
- yield 100,
+ yield (100,)
test_udtf = udtf(TestUDTF)
self.spark.udtf.register("test_udtf", test_udtf)
@@ -2282,7 +2282,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType="a: int")
class TestUDTF:
def eval(self, a, b):
- yield a,
+ yield (a,)
self.spark.udtf.register("test_udtf", TestUDTF)
@@ -2301,7 +2301,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType="a: int")
class TestUDTF:
def eval(self, a, b):
- yield a.id,
+ yield (a.id,)
self.spark.udtf.register("test_udtf", TestUDTF)
@@ -2320,7 +2320,7 @@ class BaseUDTFTestsMixin:
@udtf(returnType="a: int")
class TestUDTF:
def eval(self, a, b):
- yield a,
+ yield (a,)
self.spark.udtf.register("test_udtf", TestUDTF)
@@ -2461,7 +2461,7 @@ class BaseUDTFTestsMixin:
return AnalyzeResult(StructType().add("a", a.dataType))
def eval(self, a, b):
- yield a,
+ yield (a,)
self.spark.udtf.register("test_udtf", TestUDTF)
@@ -2899,7 +2899,7 @@ class BaseUDTFTestsMixin:
)
def eval(self, row: Row):
- yield row["id"],
+ yield (row["id"],)
df = self.spark.createDataFrame([(1,), (None,)], ["id"])
assertDataFrameEqual(
@@ -3074,9 +3074,14 @@ class BaseUDTFTestsMixin:
# TODO(SPARK-50284): Replace when an easy Python API to construct Variants is created.
def eval(self, n):
for i in range(n):
- yield i, {
- "v1": VariantVal(bytes([2, 1, 0, 0, 2, 5, 97 + i]), bytes([1, 1, 0, 1, 97]))
- }
+ yield (
+ i,
+ {
+ "v1": VariantVal(
+ bytes([2, 1, 0, 0, 2, 5, 97 + i]), bytes([1, 1, 0, 1, 97])
+ )
+ },
+ )
self.spark.udtf.register("test_udtf_struct", TestUDTFStruct)
res = self.spark.sql("select i, to_json(v.v1) from test_udtf_struct(8)")
@@ -3088,9 +3093,10 @@ class BaseUDTFTestsMixin:
# TODO(SPARK-50284): Replace when an easy Python API to construct Variants is created.
def eval(self, n):
for i in range(n):
- yield i, [
- VariantVal(bytes([2, 1, 0, 0, 2, 5, 98 + i]), bytes([1, 1, 0, 1, 97]))
- ]
+ yield (
+ i,
+ [VariantVal(bytes([2, 1, 0, 0, 2, 5, 98 + i]), bytes([1, 1, 0, 1, 97]))],
+ )
self.spark.udtf.register("test_udtf_array", TestUDTFArray)
res = self.spark.sql("select i, to_json(v[0]) from test_udtf_array(8)")
@@ -3102,9 +3108,14 @@ class BaseUDTFTestsMixin:
# TODO(SPARK-50284): Replace when an easy Python API to construct Variants is created.
def eval(self, n):
for i in range(n):
- yield i, {
- "v1": VariantVal(bytes([2, 1, 0, 0, 2, 5, 99 + i]), bytes([1, 1, 0, 1, 97]))
- }
+ yield (
+ i,
+ {
+ "v1": VariantVal(
+ bytes([2, 1, 0, 0, 2, 5, 99 + i]), bytes([1, 1, 0, 1, 97])
+ )
+ },
+ )
self.spark.udtf.register("test_udtf_struct", TestUDTFStruct)
res = self.spark.sql("select i, to_json(v['v1']) from test_udtf_struct(8)")
@@ -3128,7 +3139,7 @@ class BaseUDTFTestsMixin:
def eval(self):
import ctypes
- yield ctypes.string_at(0),
+ yield (ctypes.string_at(0),)
self._check_result_or_exception(
TestUDTF, "x: string", expected, err_type=Exception
@@ -3145,7 +3156,7 @@ class BaseUDTFTestsMixin:
return AnalyzeResult(StructType().add("x", StringType()))
def eval(self):
- yield "x",
+ yield ("x",)
self._check_result_or_exception(
TestUDTFWithAnalyze, None, expected, err_type=Exception
@@ -3181,7 +3192,7 @@ class BaseUDTFTestsMixin:
return AnalyzeResult(StructType().add("x", StringType()))
def eval(self):
- yield "x",
+ yield ("x",)
self._check_result_or_exception(
TestUDTFWithAnalyze,
@@ -3411,7 +3422,7 @@ class LegacyUDTFArrowTestsMixin(BaseUDTFTestsMixin):
def test_udtf_arrow_sql_conf(self):
class TestUDTF:
def eval(self):
- yield 1,
+ yield (1,)
old_value = self.spark.conf.get("spark.sql.execution.pythonUDTF.arrow.enabled")
try:
@@ -3473,7 +3484,7 @@ class LegacyUDTFArrowTestsMixin(BaseUDTFTestsMixin):
def test_numeric_output_type_casting(self):
class TestUDTF:
def eval(self):
- yield 1,
+ yield (1,)
err = "Exception thrown when converting pandas.Series"
@@ -3500,7 +3511,7 @@ class LegacyUDTFArrowTestsMixin(BaseUDTFTestsMixin):
def test_numeric_string_output_type_casting(self):
class TestUDTF:
def eval(self):
- yield "1",
+ yield ("1",)
err = "Exception thrown when converting pandas.Series"
@@ -3529,7 +3540,7 @@ class LegacyUDTFArrowTestsMixin(BaseUDTFTestsMixin):
def test_string_output_type_casting(self):
class TestUDTF:
def eval(self):
- yield "hello",
+ yield ("hello",)
err = "Exception thrown when converting pandas.Series"
@@ -3558,7 +3569,7 @@ class LegacyUDTFArrowTestsMixin(BaseUDTFTestsMixin):
def test_array_output_type_casting(self):
class TestUDTF:
def eval(self):
- yield [0, 1.1, 2],
+ yield ([0, 1.1, 2],)
err = "Exception thrown when converting pandas.Series"
@@ -3591,7 +3602,7 @@ class LegacyUDTFArrowTestsMixin(BaseUDTFTestsMixin):
def test_map_output_type_casting(self):
class TestUDTF:
def eval(self):
- yield {"a": 0, "b": 1.1, "c": 2},
+ yield ({"a": 0, "b": 1.1, "c": 2},)
err = "Exception thrown when converting pandas.Series"
@@ -3623,7 +3634,7 @@ class LegacyUDTFArrowTestsMixin(BaseUDTFTestsMixin):
def test_struct_output_type_casting_dict(self):
class TestUDTF:
def eval(self):
- yield {"a": 0, "b": 1.1, "c": 2},
+ yield ({"a": 0, "b": 1.1, "c": 2},)
err = "Exception thrown when converting pandas.Series"
@@ -3654,7 +3665,7 @@ class LegacyUDTFArrowTestsMixin(BaseUDTFTestsMixin):
def test_struct_output_type_casting_row(self):
class TestUDTF:
def eval(self):
- yield Row(a=0, b=1.1, c=2),
+ yield (Row(a=0, b=1.1, c=2),)
err = "Exception thrown when converting pandas.Series"
@@ -3685,8 +3696,8 @@ class LegacyUDTFArrowTestsMixin(BaseUDTFTestsMixin):
def test_inconsistent_output_types(self):
class TestUDTF:
def eval(self):
- yield 1,
- yield [1, 2],
+ yield (1,)
+ yield ([1, 2],)
for ret_type in [
"x: int",
@@ -3726,7 +3737,7 @@ class UDTFArrowTestsMixin(LegacyUDTFArrowTestsMixin):
def test_numeric_output_type_casting(self):
class TestUDTF:
def eval(self):
- yield 1,
+ yield (1,)
err = "UDTF_ARROW_DATA_CONVERSION_ERROR"
@@ -3753,7 +3764,7 @@ class UDTFArrowTestsMixin(LegacyUDTFArrowTestsMixin):
def test_numeric_string_output_type_casting(self):
class TestUDTF:
def eval(self):
- yield "1",
+ yield ("1",)
err = "UDTF_ARROW_DATA_CONVERSION_ERROR"
@@ -3782,7 +3793,7 @@ class UDTFArrowTestsMixin(LegacyUDTFArrowTestsMixin):
def test_string_output_type_casting(self):
class TestUDTF:
def eval(self):
- yield "hello",
+ yield ("hello",)
err = "UDTF_ARROW_DATA_CONVERSION_ERROR"
for ret_type, expected in [
@@ -3810,7 +3821,7 @@ class UDTFArrowTestsMixin(LegacyUDTFArrowTestsMixin):
def test_array_output_type_casting(self):
class TestUDTF:
def eval(self):
- yield [0, 1.1, 2],
+ yield ([0, 1.1, 2],)
err = "UDTF_ARROW_DATA_CONVERSION_ERROR"
for ret_type, expected in [
@@ -3842,7 +3853,7 @@ class UDTFArrowTestsMixin(LegacyUDTFArrowTestsMixin):
def test_map_output_type_casting(self):
class TestUDTF:
def eval(self):
- yield {"a": 0, "b": 1.1, "c": 2},
+ yield ({"a": 0, "b": 1.1, "c": 2},)
err = "UDTF_ARROW_DATA_CONVERSION_ERROR"
for ret_type, expected in [
@@ -3873,7 +3884,7 @@ class UDTFArrowTestsMixin(LegacyUDTFArrowTestsMixin):
def test_struct_output_type_casting_dict(self):
class TestUDTF:
def eval(self):
- yield {"a": 0, "b": 1.1, "c": 2},
+ yield ({"a": 0, "b": 1.1, "c": 2},)
err = "UDTF_ARROW_DATA_CONVERSION_ERROR"
for ret_type, expected in [
@@ -3903,7 +3914,7 @@ class UDTFArrowTestsMixin(LegacyUDTFArrowTestsMixin):
def test_struct_output_type_casting_row(self):
class TestUDTF:
def eval(self):
- yield Row(a=0, b=1.1, c=2),
+ yield (Row(a=0, b=1.1, c=2),)
err = "UDTF_ARROW_DATA_CONVERSION_ERROR"
for ret_type, expected in [
@@ -3933,8 +3944,8 @@ class UDTFArrowTestsMixin(LegacyUDTFArrowTestsMixin):
def test_inconsistent_output_types(self):
class TestUDTF:
def eval(self):
- yield 1,
- yield [1, 2],
+ yield (1,)
+ yield ([1, 2],)
for ret_type in [
"x: int",
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index 1963e42f03c4..4114b3006c82 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -1469,9 +1469,9 @@ class StructType(DataType):
else:
self.fields = fields
self.names = [f.name for f in fields]
- assert all(
- isinstance(f, StructField) for f in fields
- ), "fields should be a list of StructField"
+ assert all(isinstance(f, StructField) for f in fields), (
+ "fields should be a list of StructField"
+ )
# Precalculated list of fields that need conversion with fromInternal/toInternal functions
self._needConversion = [f.needConversion() for f in self]
self._needSerializeAnyField = any(self._needConversion)
diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py
index e24ee504642c..2c3e7009a3cf 100644
--- a/python/pyspark/sql/udf.py
+++ b/python/pyspark/sql/udf.py
@@ -221,7 +221,7 @@ class UserDefinedFunction:
raise PySparkNotImplementedError(
errorClass="NOT_IMPLEMENTED",
messageParameters={
- "feature": f"Invalid return type with scalar Pandas UDFs: " f"{returnType}"
+ "feature": f"Invalid return type with scalar Pandas UDFs: {returnType}"
},
)
elif (
@@ -234,7 +234,7 @@ class UserDefinedFunction:
raise PySparkNotImplementedError(
errorClass="NOT_IMPLEMENTED",
messageParameters={
- "feature": f"Invalid return type with scalar Arrow UDFs: " f"{returnType}"
+ "feature": f"Invalid return type with scalar Arrow UDFs: {returnType}"
},
)
elif (
@@ -274,7 +274,7 @@ class UserDefinedFunction:
raise PySparkNotImplementedError(
errorClass="NOT_IMPLEMENTED",
messageParameters={
- "feature": f"Invalid return type in mapInPandas: " f"{returnType}"
+ "feature": f"Invalid return type in mapInPandas: {returnType}"
},
)
else:
@@ -316,8 +316,7 @@ class UserDefinedFunction:
raise PySparkNotImplementedError(
errorClass="NOT_IMPLEMENTED",
messageParameters={
- "feature": f"Invalid return type in cogroup.applyInPandas: "
- f"{returnType}"
+ "feature": f"Invalid return type in cogroup.applyInPandas: {returnType}"
},
)
else:
@@ -336,8 +335,7 @@ class UserDefinedFunction:
raise PySparkNotImplementedError(
errorClass="NOT_IMPLEMENTED",
messageParameters={
- "feature": "Invalid return type in cogroup.applyInArrow: "
- f"{returnType}"
+ "feature": f"Invalid return type in cogroup.applyInArrow: {returnType}"
},
)
else:
@@ -484,7 +482,11 @@ class UserDefinedFunction:
def func(*args: Any, **kwargs: Any) -> Any:
assert memory_profiler is not None
return memory_profiler.profile(
- sub_lines, start_line, f, *args, **kwargs # type: ignore[arg-type]
+ sub_lines, # type: ignore[arg-type]
+ start_line,
+ f,
+ *args,
+ **kwargs,
)
func.__signature__ = inspect.signature(f) # type: ignore[attr-defined]
diff --git a/python/pyspark/sql/worker/analyze_udtf.py b/python/pyspark/sql/worker/analyze_udtf.py
index 3559dfff42fc..03375127ad0d 100644
--- a/python/pyspark/sql/worker/analyze_udtf.py
+++ b/python/pyspark/sql/worker/analyze_udtf.py
@@ -114,12 +114,14 @@ def _main(infile: IO, outfile: IO) -> None:
inspect.signature(handler.analyze).bind(*args, **kwargs) # type: ignore[attr-defined]
except TypeError as e:
# The UDTF call's arguments did not match the expected signature.
- raise PySparkValueError(format_error(f"""
+ raise PySparkValueError(
+ format_error(f"""
{error_prefix} because the function arguments did not match the expected
signature of the static 'analyze' method ({e}). Please update the query so that
this table function call provides arguments matching the expected signature, or
else update the table function so that its static 'analyze' method accepts the
- provided arguments, and then try the query again."""))
+ provided arguments, and then try the query again.""")
+ )
# The default context provider can't detect the class name from static methods.
def context_provider() -> dict[str, str]:
@@ -133,40 +135,50 @@ def _main(infile: IO, outfile: IO) -> None:
# Check invariants about the 'analyze' method after running it.
if not isinstance(result, AnalyzeResult):
- raise PySparkValueError(format_error(f"""
+ raise PySparkValueError(
+ format_error(f"""
{error_prefix} because the static 'analyze' method expects a result of type
pyspark.sql.udtf.AnalyzeResult, but instead this method returned a value of
- type: {type(result)}"""))
+ type: {type(result)}""")
+ )
elif not isinstance(result.schema, StructType):
- raise PySparkValueError(format_error(f"""
+ raise PySparkValueError(
+ format_error(f"""
{error_prefix} because the static 'analyze' method expects a result of type
pyspark.sql.udtf.AnalyzeResult with a 'schema' field comprising a StructType,
- but the 'schema' field had the wrong type: {type(result.schema)}"""))
+ but the 'schema' field had the wrong type: {type(result.schema)}""")
+ )
def invalid_analyze_result_field(field_name: str, expected_field: str) -> PySparkValueError:
- return PySparkValueError(format_error(f"""
+ return PySparkValueError(
+ format_error(f"""
{error_prefix} because the static 'analyze' method returned an
'AnalyzeResult' object with the '{field_name}' field set to a value besides a
list or tuple of '{expected_field}' objects. Please update the table function
- and then try the query again."""))
+ and then try the query again.""")
+ )
has_table_arg = any(arg.isTable for arg in args) or any(arg.isTable for arg in kwargs.values())
if not has_table_arg and result.withSinglePartition:
- raise PySparkValueError(format_error(f"""
+ raise PySparkValueError(
+ format_error(f"""
{error_prefix} because the static 'analyze' method returned an
'AnalyzeResult' object with the 'withSinglePartition' field set to 'true', but
the function call did not provide any table argument. Please update the query so
that it provides a table argument, or else update the table function so that its
'analyze' method returns an 'AnalyzeResult' object with the
- 'withSinglePartition' field set to 'false', and then try the query again."""))
+ 'withSinglePartition' field set to 'false', and then try the query again.""")
+ )
elif not has_table_arg and len(result.partitionBy) > 0:
- raise PySparkValueError(format_error(f"""
+ raise PySparkValueError(
+ format_error(f"""
{error_prefix} because the static 'analyze' method returned an
'AnalyzeResult' object with the 'partitionBy' list set to non-empty, but the
function call did not provide any table argument. Please update the query so
that it provides a table argument, or else update the table function so that its
'analyze' method returns an 'AnalyzeResult' object with the 'partitionBy' list
- set to empty, and then try the query again."""))
+ set to empty, and then try the query again.""")
+ )
elif not isinstance(result.partitionBy, (list, tuple)) or not all(
isinstance(val, PartitioningColumn) for val in result.partitionBy
):
diff --git a/python/pyspark/sql/worker/plan_data_source_read.py b/python/pyspark/sql/worker/plan_data_source_read.py
index c006eeb9ae31..58eceb1d3428 100644
--- a/python/pyspark/sql/worker/plan_data_source_read.py
+++ b/python/pyspark/sql/worker/plan_data_source_read.py
@@ -203,9 +203,9 @@ def write_read_func_and_partitions(
columns = [column.to_pylist() for column in batch.columns]
partition_bytes = converter(columns[0][0]) # type: ignore[misc]
- assert (
- partition_bytes is not None
- ), "The input iterator for Python data source read function is empty."
+ assert partition_bytes is not None, (
+ "The input iterator for Python data source read function is empty."
+ )
# Deserialize the partition value.
partition = pickleSer.loads(partition_bytes)
@@ -338,8 +338,7 @@ def _main(infile: IO, outfile: IO) -> None:
# Receive the configuration values.
max_arrow_batch_size = read_int(infile)
assert max_arrow_batch_size > 0, (
- "The maximum arrow batch size should be greater than 0, but got "
- f"'{max_arrow_batch_size}'"
+ f"The maximum arrow batch size should be greater than 0, but got '{max_arrow_batch_size}'"
)
enable_pushdown = read_bool(infile)
diff --git a/python/pyspark/sql/worker/write_into_data_source.py b/python/pyspark/sql/worker/write_into_data_source.py
index 7498e60fa6a4..e188bdb07f91 100644
--- a/python/pyspark/sql/worker/write_into_data_source.py
+++ b/python/pyspark/sql/worker/write_into_data_source.py
@@ -157,8 +157,7 @@ def _main(infile: IO, outfile: IO) -> None:
errorClass="DATA_SOURCE_TYPE_MISMATCH",
messageParameters={
"expected": (
- "an instance of DataSourceStreamWriter or "
- "DataSourceStreamArrowWriter"
+ "an instance of DataSourceStreamWriter or DataSourceStreamArrowWriter"
),
"actual": f"'{type(writer).__name__}'",
},
@@ -210,9 +209,7 @@ def _main(infile: IO, outfile: IO) -> None:
raise PySparkRuntimeError(
errorClass="DATA_SOURCE_TYPE_MISMATCH",
messageParameters={
- "expected": (
- "'WriterCommitMessage' as the return type of " "the `write` method"
- ),
+ "expected": ("'WriterCommitMessage' as the return type of the `write` method"),
"actual": type(res).__name__,
},
)
diff --git a/python/pyspark/streaming/dstream.py b/python/pyspark/streaming/dstream.py
index 29d56bc144fa..bfc885bb5779 100644
--- a/python/pyspark/streaming/dstream.py
+++ b/python/pyspark/streaming/dstream.py
@@ -814,7 +814,8 @@ class DStream(Generic[T_co]):
return DStream(dstream.asJavaDStream(), self._ssc, self._sc.serializer)
else:
return reduced.window(windowDuration, slideDuration).reduceByKey(
- func, numPartitions # type: ignore[arg-type]
+ func, # type: ignore[arg-type]
+ numPartitions,
)
def updateStateByKey(
diff --git a/python/pyspark/testing/utils.py b/python/pyspark/testing/utils.py
index 85ade0c66003..aa16a566dac4 100644
--- a/python/pyspark/testing/utils.py
+++ b/python/pyspark/testing/utils.py
@@ -451,12 +451,14 @@ class PySparkErrorTestUtils:
f"Expected message parameter key '{key}' was not found "
"in actual message parameters.",
)
- self.assertRegex(
- actual[key],
- value,
- f"Expected message parameter value '{value}' does not match actual message "
- f"parameter value '{actual[key]}'.",
- ),
+ (
+ self.assertRegex(
+ actual[key],
+ value,
+ f"Expected message parameter value '{value}' does not match actual message "
+ f"parameter value '{actual[key]}'.",
+ ),
+ )
else:
self.assertEqual(
expected, actual, f"Expected message parameters was '{expected}', got '{actual}'"
@@ -472,9 +474,9 @@ class PySparkErrorTestUtils:
expected, actual, f"Expected QueryContext was '{expected}', got '{actual}'"
)
if actual == QueryContextType.DataFrame:
- assert (
- fragment is not None
- ), "`fragment` is required when QueryContextType is DataFrame."
+ assert fragment is not None, (
+ "`fragment` is required when QueryContextType is DataFrame."
+ )
expected = fragment
actual = actual_context.fragment()
self.assertEqual(
diff --git a/python/pyspark/tests/test_appsubmit.py b/python/pyspark/tests/test_appsubmit.py
index 1e0c8bb87048..4777661391ec 100644
--- a/python/pyspark/tests/test_appsubmit.py
+++ b/python/pyspark/tests/test_appsubmit.py
@@ -79,7 +79,8 @@ class SparkSubmitTests(unittest.TestCase):
group_id, artifact_id, version = artifact_name.split(":")
self.createTempFile(
"%s-%s.pom" % (artifact_id, version),
- ("""
+ (
+ """
|<?xml version="1.0" encoding="UTF-8"?>
|<project xmlns="http://maven.apache.org/POM/4.0.0"
| xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
@@ -90,7 +91,9 @@ class SparkSubmitTests(unittest.TestCase):
| <artifactId>%s</artifactId>
| <version>%s</version>
|</project>
- """ % (group_id, artifact_id, version)).lstrip(),
+ """
+ % (group_id, artifact_id, version)
+ ).lstrip(),
os.path.join(group_id, artifact_id, version),
)
self.createFileInZip(
diff --git a/python/pyspark/tests/test_profiler.py b/python/pyspark/tests/test_profiler.py
index 56fd0f473308..26834dbed145 100644
--- a/python/pyspark/tests/test_profiler.py
+++ b/python/pyspark/tests/test_profiler.py
@@ -153,8 +153,7 @@ class ProfilerTests2(unittest.TestCase, PySparkErrorTestUtils):
try:
self.assertRaisesRegex(
PythonException,
- "Install the 'memory_profiler' library in the cluster to enable memory "
- "profiling",
+ "Install the 'memory_profiler' library in the cluster to enable memory profiling",
lambda: spark.range(10).select(plus_one("id")).collect(),
)
finally:
diff --git a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_cast.py b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_cast.py
index ce19458328aa..6c48e4e9461a 100644
--- a/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_cast.py
+++ b/python/pyspark/tests/upstream/pyarrow/test_pyarrow_array_cast.py
@@ -195,7 +195,7 @@ class _PyArrowCastTestBase(GoldenFileTestMixin, unittest.TestCase):
expected = golden.loc[row_name, col_name]
if expected != result:
errors.append(
- f"{row_name} -> {col_name}: " f"expected '{expected}', got '{result}'"
+ f"{row_name} -> {col_name}: expected '{expected}', got '{result}'"
)
if generating:
diff --git a/python/pyspark/util.py b/python/pyspark/util.py
index 0577c4aa6a91..99760e93789b 100644
--- a/python/pyspark/util.py
+++ b/python/pyspark/util.py
@@ -414,9 +414,9 @@ def inheritable_thread_target(f: Optional[Union[Callable, "SparkSession"]] = Non
session = f
def outer(ff: Callable) -> Callable:
- assert isinstance(
- session, RemoteSparkSession
- ), "f is expected to be SparkSession for spark connect"
+ assert isinstance(session, RemoteSparkSession), (
+ "f is expected to be SparkSession for spark connect"
+ )
thread_local = session.client.thread_local
session_client_thread_local_attrs = [
(attr, copy.deepcopy(value))
@@ -429,9 +429,9 @@ def inheritable_thread_target(f: Optional[Union[Callable, "SparkSession"]] = Non
@functools.wraps(ff)
def inner(*args: Any, **kwargs: Any) -> Any:
# Propagates the active remote spark session to the current thread.
- assert isinstance(
- session, RemoteSparkSession
- ), "f is expected to be SparkSession for spark connect"
+ assert isinstance(session, RemoteSparkSession), (
+ "f is expected to be SparkSession for spark connect"
+ )
RemoteSparkSession._set_default_and_active_session(session)
# Set thread locals in child thread.
@@ -605,7 +605,9 @@ class InheritableThread(threading.Thread):
return target(*a, **k)
super().__init__(
- target=copy_local_properties, *args, **kwargs # type: ignore[misc]
+ target=copy_local_properties,
+ *args,
+ **kwargs, # type: ignore[misc]
)
else:
super().__init__(target=target, *args, **kwargs) # type: ignore[misc]
@@ -665,13 +667,9 @@ class PythonEvalType:
SQL_GROUPED_MAP_ARROW_UDF: "ArrowGroupedMapUDFType" = 209
SQL_COGROUPED_MAP_ARROW_UDF: "ArrowCogroupedMapUDFType" = 210
SQL_TRANSFORM_WITH_STATE_PANDAS_UDF: "PandasGroupedMapUDFTransformWithStateType" = 211
- SQL_TRANSFORM_WITH_STATE_PANDAS_INIT_STATE_UDF: (
- "PandasGroupedMapUDFTransformWithStateInitStateType"
- ) = 212
+ SQL_TRANSFORM_WITH_STATE_PANDAS_INIT_STATE_UDF: "PandasGroupedMapUDFTransformWithStateInitStateType" = 212
SQL_TRANSFORM_WITH_STATE_PYTHON_ROW_UDF: "GroupedMapUDFTransformWithStateType" = 213
- SQL_TRANSFORM_WITH_STATE_PYTHON_ROW_INIT_STATE_UDF: (
- "GroupedMapUDFTransformWithStateInitStateType"
- ) = 214
+ SQL_TRANSFORM_WITH_STATE_PYTHON_ROW_INIT_STATE_UDF: "GroupedMapUDFTransformWithStateInitStateType" = 214
SQL_GROUPED_MAP_ARROW_ITER_UDF: "ArrowGroupedMapIterUDFType" = 215
SQL_GROUPED_MAP_PANDAS_ITER_UDF: "PandasGroupedMapIterUDFType" = 216
SQL_GROUPED_AGG_PANDAS_ITER_UDF: "PandasGroupedAggIterUDFType" = 217
diff --git a/python/pyspark/worker.py b/python/pyspark/worker.py
index c16581b057cd..0ef578d3715b 100644
--- a/python/pyspark/worker.py
+++ b/python/pyspark/worker.py
@@ -2369,12 +2369,8 @@ def read_udtf(pickleSer, infile, eval_type, runner_conf):
raise PySparkRuntimeError(
errorClass="UDTF_RETURN_SCHEMA_MISMATCH",
messageParameters={
- "expected": e.getMessageParameters()[
- "expected_length"
- ], # type: ignore[index]
- "actual": e.getMessageParameters()[
- "actual_length"
- ], # type: ignore[index]
+ "expected": e.getMessageParameters()["expected_length"], # type: ignore[index]
+ "actual": e.getMessageParameters()["actual_length"], # type: ignore[index]
"func": f.__name__,
},
) from e
diff --git a/python/pyspark/worker_util.py b/python/pyspark/worker_util.py
index a820400fc2e9..8c52bde6ea6c 100644
--- a/python/pyspark/worker_util.py
+++ b/python/pyspark/worker_util.py
@@ -28,9 +28,9 @@ from typing import Any, Generator, IO, Optional
import warnings
if "SPARK_TESTING" in os.environ:
- assert (
- os.environ.get("SPARK_PYTHON_RUNTIME") == "PYTHON_WORKER"
- ), "This module can only be imported in python woker"
+ assert os.environ.get("SPARK_PYTHON_RUNTIME") == "PYTHON_WORKER", (
+ "This module can only be imported in python woker"
+ )
# 'resource' is a Unix specific module.
has_resource_module = True
diff --git a/python/run-tests.py b/python/run-tests.py
index c873c26639f5..6ea48e550585 100755
--- a/python/run-tests.py
+++ b/python/run-tests.py
@@ -45,17 +45,17 @@ from sparktestsupport.shellutils import which, subprocess_check_output
from sparktestsupport.modules import all_modules, pyspark_sql # noqa
-python_modules = dict((m.name, m) for m in all_modules if m.python_test_goals if m.name != 'root')
+python_modules = dict((m.name, m) for m in all_modules if m.python_test_goals if m.name != "root")
def print_red(text):
- print('\033[31m' + text + '\033[0m')
+ print("\033[31m" + text + "\033[0m")
def get_valid_filename(s):
"""Replace whitespaces and special characters in the given string to get a valid file name."""
- s = s.strip().replace(' ', '_').replace(os.sep, '_')
- return re.sub(r'(?u)[^-\w.]', '', s)
+ s = s.strip().replace(" ", "_").replace(os.sep, "_")
+ return re.sub(r"(?u)[^-\w.]", "", s)
SKIPPED_TESTS = None
@@ -146,8 +146,8 @@ class TestRunner:
def process_buffer(self, buffer, force_flush=False):
# Process all full lines first
while (nl := buffer.find(b"\n")) != -1:
- self.output_line(buffer[:nl + 1])
- buffer = buffer[nl + 1:]
+ self.output_line(buffer[: nl + 1])
+ buffer = buffer[nl + 1 :]
# Process the remaining buffer
if b"(Pdb)" in buffer:
self.pdb_mode = True
@@ -210,7 +210,7 @@ class TestRunner:
raise subprocess.TimeoutExpired(self.cmd, self.timeout)
def thread_dump(self, pid):
- pyspark_python = self.env['PYSPARK_PYTHON']
+ pyspark_python = self.env["PYSPARK_PYTHON"]
python_path = f"{os.path.join(SPARK_HOME, 'python')}"
py4j_path = f"{os.path.join(SPARK_HOME, 'python/lib/py4j-0.10.9.9-src.zip')}"
p = subprocess.run(
@@ -228,7 +228,9 @@ class TestRunner:
# pystack or psutil not installed, that's okay
pass
else:
- LOGGER.error(f"Failed to get thread dump, exit code {p.returncode}:\n{p.stdout.decode('utf-8')}")
+ LOGGER.error(
+ f"Failed to get thread dump, exit code {p.returncode}:\n{p.stdout.decode('utf-8')}"
+ )
def run_individual_python_test(target_dir, test_name, pyspark_python, keep_test_output):
@@ -248,13 +250,15 @@ def run_individual_python_test(target_dir, test_name, pyspark_python, keep_test_
"""
env = dict(os.environ)
- env.update({
- 'SPARK_DIST_CLASSPATH': SPARK_DIST_CLASSPATH,
- 'SPARK_TESTING': '1',
- 'SPARK_PREPEND_CLASSES': '1',
- 'PYSPARK_PYTHON': which(pyspark_python),
- 'PYSPARK_DRIVER_PYTHON': which(pyspark_python),
- })
+ env.update(
+ {
+ "SPARK_DIST_CLASSPATH": SPARK_DIST_CLASSPATH,
+ "SPARK_TESTING": "1",
+ "SPARK_PREPEND_CLASSES": "1",
+ "PYSPARK_PYTHON": which(pyspark_python),
+ "PYSPARK_DRIVER_PYTHON": which(pyspark_python),
+ }
+ )
if "SPARK_CONNECT_TESTING_REMOTE" in os.environ:
env.update({"SPARK_CONNECT_TESTING_REMOTE": os.environ["SPARK_CONNECT_TESTING_REMOTE"]})
@@ -267,7 +271,7 @@ def run_individual_python_test(target_dir, test_name, pyspark_python, keep_test_
while os.path.isdir(tmp_dir):
tmp_dir = os.path.join(target_dir, str(uuid.uuid4()))
os.mkdir(tmp_dir)
- sock_dir = os.getenv('TMPDIR') or os.getenv('TEMP') or os.getenv('TMP') or '/tmp'
+ sock_dir = os.getenv("TMPDIR") or os.getenv("TEMP") or os.getenv("TMP") or "/tmp"
env["TMPDIR"] = tmp_dir
metastore_dir = os.path.join(tmp_dir, str(uuid.uuid4()))
while os.path.isdir(metastore_dir):
@@ -276,15 +280,17 @@ def run_individual_python_test(target_dir, test_name, pyspark_python, keep_test_
# Also override the JVM's temp directory and log4j conf by setting driver and executor options.
log4j2_path = os.path.join(SPARK_HOME, "python/test_support/log4j2.properties")
- java_options = "-Djava.io.tmpdir={0} -Dlog4j.configurationFile={1}".format(
- tmp_dir, log4j2_path
- )
+ java_options = "-Djava.io.tmpdir={0} -Dlog4j.configurationFile={1}".format(tmp_dir, log4j2_path)
java_options = java_options + " -Xss4M"
spark_args = [
- "--conf", "spark.driver.extraJavaOptions='{0}'".format(java_options),
- "--conf", "spark.executor.extraJavaOptions='{0}'".format(java_options),
- "--conf", "spark.sql.warehouse.dir='{0}'".format(metastore_dir),
- "--conf", "spark.python.unix.domain.socket.dir={0}".format(sock_dir),
+ "--conf",
+ "spark.driver.extraJavaOptions='{0}'".format(java_options),
+ "--conf",
+ "spark.executor.extraJavaOptions='{0}'".format(java_options),
+ "--conf",
+ "spark.sql.warehouse.dir='{0}'".format(metastore_dir),
+ "--conf",
+ "spark.python.unix.domain.socket.dir={0}".format(sock_dir),
"pyspark-shell",
]
@@ -298,10 +304,12 @@ def run_individual_python_test(target_dir, test_name, pyspark_python, keep_test_
output_prefix = get_valid_filename(pyspark_python + "__" + test_name + "__").lstrip("_")
# Delete is always set to False since the cleanup will be either done by removing the
# whole test dir, or the test output is retained.
- per_test_output = tempfile.NamedTemporaryFile(prefix=output_prefix, dir=tmp_dir,
- suffix=".log", delete=False)
+ per_test_output = tempfile.NamedTemporaryFile(
+ prefix=output_prefix, dir=tmp_dir, suffix=".log", delete=False
+ )
LOGGER.info(
- "Starting test(%s): %s (temp output: %s)", pyspark_python, test_name, per_test_output.name)
+ "Starting test(%s): %s (temp output: %s)", pyspark_python, test_name, per_test_output.name
+ )
cmd = [os.path.join(SPARK_HOME, "bin/pyspark")] + test_name.split()
start_time = time.time()
@@ -326,7 +334,7 @@ def run_individual_python_test(target_dir, test_name, pyspark_python, keep_test_
try:
per_test_output.seek(0)
with FAILURE_REPORTING_LOCK:
- with open(LOG_FILE, 'ab') as log_file:
+ with open(LOG_FILE, "ab") as log_file:
log_file.writelines(per_test_output)
# We don't want the logging lines interleave with the test output, so we read the
@@ -335,7 +343,7 @@ def run_individual_python_test(target_dir, test_name, pyspark_python, keep_test_
lines = []
for line in per_test_output:
line = line.decode("utf-8", "replace")
- if not re.match('[0-9]+', line):
+ if not re.match("[0-9]+", line):
lines.append(line)
LOGGER.error(f"{test_name} with {pyspark_python} failed:\n{''.join(lines)}")
per_test_output.close()
@@ -353,9 +361,12 @@ def run_individual_python_test(target_dir, test_name, pyspark_python, keep_test_
# Here expects skipped test output from unittest when verbosity level is
# 2 (or --verbose option is enabled).
decoded_lines = map(lambda line: line.decode("utf-8", "replace"), iter(per_test_output))
- skipped_tests = list(filter(
- lambda line: re.search(r'test_.* \(pyspark\..*\) ... (skip|SKIP)', line),
- decoded_lines))
+ skipped_tests = list(
+ filter(
+ lambda line: re.search(r"test_.* \(pyspark\..*\) ... (skip|SKIP)", line),
+ decoded_lines,
+ )
+ )
skipped_counts = len(skipped_tests)
if skipped_counts > 0:
key = (pyspark_python, test_name)
@@ -364,18 +375,24 @@ def run_individual_python_test(target_dir, test_name, pyspark_python, keep_test_
per_test_output.close()
except BaseException:
import traceback
- print_red("\nGot an exception while trying to store "
- "skipped test output:\n%s" % traceback.format_exc())
+
+ print_red(
+ "\nGot an exception while trying to store "
+ "skipped test output:\n%s" % traceback.format_exc()
+ )
# Here, we use os._exit() instead of sys.exit() in order to force Python to exit even if
# this code is invoked from a thread other than the main thread.
os._exit(-1)
if skipped_counts != 0:
LOGGER.info(
- "Finished test(%s): %s (%is) ... %s tests were skipped", pyspark_python, test_name,
- duration, skipped_counts)
+ "Finished test(%s): %s (%is) ... %s tests were skipped",
+ pyspark_python,
+ test_name,
+ duration,
+ skipped_counts,
+ )
else:
- LOGGER.info(
- "Finished test(%s): %s (%is)", pyspark_python, test_name, duration)
+ LOGGER.info("Finished test(%s): %s (%is)", pyspark_python, test_name, duration)
def get_default_python_executables():
@@ -388,30 +405,32 @@ def get_default_python_executables():
def parse_opts():
- parser = ArgumentParser(
- prog="run-tests"
- )
+ parser = ArgumentParser(prog="run-tests")
parser.add_argument(
- "--python-executables", type=str, default=','.join(get_default_python_executables()),
- help="A comma-separated list of Python executables to test against (default: %(default)s)"
+ "--python-executables",
+ type=str,
+ default=",".join(get_default_python_executables()),
+ help="A comma-separated list of Python executables to test against (default: %(default)s)",
)
parser.add_argument(
- "--modules", type=str,
+ "--modules",
+ type=str,
default=",".join(sorted(python_modules.keys())),
- help="A comma-separated list of Python modules to test (default: %(default)s)"
+ help="A comma-separated list of Python modules to test (default: %(default)s)",
)
parser.add_argument(
- "-p", "--parallelism", type=int, default=4,
- help="The number of suites to test in parallel (default %(default)d)"
- )
- parser.add_argument(
- "--verbose", action="store_true",
- help="Enable additional debug logging"
+ "-p",
+ "--parallelism",
+ type=int,
+ default=4,
+ help="The number of suites to test in parallel (default %(default)d)",
)
+ parser.add_argument("--verbose", action="store_true", help="Enable additional debug logging")
group = parser.add_argument_group("Developer Options")
group.add_argument(
- "--testnames", type=str,
+ "--testnames",
+ type=str,
default=None,
help=(
"A comma-separated list of specific modules, classes and functions of doctest "
@@ -419,19 +438,24 @@ def parse_opts():
"For example, 'pyspark.sql.foo' to run the module as unittests or doctests, "
"'pyspark.sql.tests FooTests' to run the specific class of unittests, "
"'pyspark.sql.tests FooTests.test_foo' to run the specific unittest in the class. "
- "'--modules' option is ignored if they are given.")
+ "'--modules' option is ignored if they are given."
+ ),
)
group.add_argument(
- "-k", "--keep-test-output", action='store_true',
+ "-k",
+ "--keep-test-output",
+ action="store_true",
default=False,
- help=("If set to true will retain the temporary test directories. In addition, the "
- "standard output and standard error are redirected to a file in the target "
- "directory.")
+ help=(
+ "If set to true will retain the temporary test directories. In addition, the "
+ "standard output and standard error are redirected to a file in the target "
+ "directory."
+ ),
)
args, unknown = parser.parse_known_args()
if unknown:
- parser.error("Unsupported arguments: %s" % ' '.join(unknown))
+ parser.error("Unsupported arguments: %s" % " ".join(unknown))
if args.parallelism < 1:
parser.error("Parallelism cannot be less than 1")
return args
@@ -441,12 +465,14 @@ def _check_coverage(python_exec):
# Make sure if coverage is installed.
try:
subprocess_check_output(
- [python_exec, "-c", "import coverage"],
- stderr=open(os.devnull, 'w'))
+ [python_exec, "-c", "import coverage"], stderr=open(os.devnull, "w")
+ )
except BaseException:
- print_red("Coverage is not installed in Python executable '%s' "
- "but 'COVERAGE_PROCESS_START' environment variable is set, "
- "exiting." % python_exec)
+ print_red(
+ "Coverage is not installed in Python executable '%s' "
+ "but 'COVERAGE_PROCESS_START' environment variable is set, "
+ "exiting." % python_exec
+ )
sys.exit(-1)
@@ -461,21 +487,23 @@ def main():
LOGGER.info("Running PySpark tests. Output is in %s", LOG_FILE)
if os.path.exists(LOG_FILE):
os.remove(LOG_FILE)
- python_execs = opts.python_executables.split(',')
+ python_execs = opts.python_executables.split(",")
LOGGER.info("Will test against the following Python executables: %s", python_execs)
if should_test_modules:
modules_to_test = []
- for module_name in opts.modules.split(','):
+ for module_name in opts.modules.split(","):
if module_name in python_modules:
modules_to_test.append(python_modules[module_name])
else:
- print("Error: unrecognized module '%s'. Supported modules: %s" %
- (module_name, ", ".join(python_modules)))
+ print(
+ "Error: unrecognized module '%s'. Supported modules: %s"
+ % (module_name, ", ".join(python_modules))
+ )
sys.exit(-1)
LOGGER.info("Will test the following Python modules: %s", [x.name for x in modules_to_test])
else:
- testnames_to_test = opts.testnames.split(',')
+ testnames_to_test = opts.testnames.split(",")
LOGGER.info("Will test the following Python tests: %s", testnames_to_test)
task_queue = Queue.PriorityQueue()
@@ -487,17 +515,28 @@ def main():
python_implementation = subprocess_check_output(
[python_exec, "-c", "import platform; print(platform.python_implementation())"],
- universal_newlines=True).strip()
+ universal_newlines=True,
+ ).strip()
LOGGER.info("%s python_implementation is %s", python_exec, python_implementation)
- LOGGER.info("%s version is: %s", python_exec, subprocess_check_output(
- [python_exec, "--version"], stderr=subprocess.STDOUT, universal_newlines=True).strip())
+ LOGGER.info(
+ "%s version is: %s",
+ python_exec,
+ subprocess_check_output(
+ [python_exec, "--version"], stderr=subprocess.STDOUT, universal_newlines=True
+ ).strip(),
+ )
if should_test_modules:
for module in modules_to_test:
if python_implementation not in module.excluded_python_implementations:
for test_goal in module.python_test_goals:
- heavy_tests = ['pyspark.streaming.tests', 'pyspark.mllib.tests',
- 'pyspark.tests', 'pyspark.sql.tests', 'pyspark.ml.tests',
- 'pyspark.pandas.tests']
+ heavy_tests = [
+ "pyspark.streaming.tests",
+ "pyspark.mllib.tests",
+ "pyspark.tests",
+ "pyspark.sql.tests",
+ "pyspark.ml.tests",
+ "pyspark.pandas.tests",
+ ]
if any(map(lambda prefix: test_goal.startswith(prefix), heavy_tests)):
priority = 0
else:
@@ -508,7 +547,7 @@ def main():
task_queue.put((0, (python_exec, test_goal)))
# Create the target directory before starting tasks to avoid races.
- target_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'target'))
+ target_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "target"))
if not os.path.isdir(target_dir):
os.mkdir(target_dir)
@@ -519,8 +558,9 @@ def main():
except Queue.Empty:
break
try:
- run_individual_python_test(target_dir, test_goal,
- python_exec, opts.keep_test_output)
+ run_individual_python_test(
+ target_dir, test_goal, python_exec, opts.keep_test_output
+ )
finally:
task_queue.task_done()
diff --git a/python/test_coverage/sitecustomize.py b/python/test_coverage/sitecustomize.py
index f1b8e494accb..4e3b92aac73e 100644
--- a/python/test_coverage/sitecustomize.py
+++ b/python/test_coverage/sitecustomize.py
@@ -21,6 +21,7 @@
# variable is set or not. If set, it starts to run the coverage.
try:
import coverage
+
if (cov := coverage.Coverage.current()) is None:
cov = coverage.process_startup()
if cov:
@@ -31,13 +32,13 @@ try:
# the coverage data. Otherwise the worker will be killed by a signal and
# the coverage data will not be saved.
import sys
+
frame = sys._getframe(1)
if (
- frame.f_code.co_name == "manager" and
- "daemon.py" in frame.f_code.co_filename and
- "worker" in frame.f_globals
+ frame.f_code.co_name == "manager"
+ and "daemon.py" in frame.f_code.co_filename
+ and "worker" in frame.f_globals
):
-
if cov := coverage.Coverage.current():
cov.stop()
cov = coverage.process_startup(force=True)
@@ -46,11 +47,13 @@ try:
# we won't have enough time to save the coverage data. So we need to save
# the coverage data before we let the JVM know about the exception.
import pyspark.util
+
handle_worker_exception = pyspark.util.handle_worker_exception
def handle_worker_exception_wrapper(*args, **kwargs):
cov.save()
handle_worker_exception(*args, **kwargs)
+
pyspark.util.handle_worker_exception = handle_worker_exception_wrapper
def save_when_exit(func):
@@ -60,6 +63,7 @@ try:
finally:
cov.save()
return result
+
return wrapper
frame.f_globals["worker"] = save_when_exit(frame.f_globals["worker"])
diff --git a/python/test_support/userlibrary.py b/python/test_support/userlibrary.py
index 2d262ba43003..058b99bd14fa 100755
--- a/python/test_support/userlibrary.py
+++ b/python/test_support/userlibrary.py
@@ -21,6 +21,5 @@ Used to test shipping of code dependencies with SparkContext.addPyFile().
class UserClass:
-
def hello(self):
return "Hello World!"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org