You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ga...@apache.org on 2026/08/25 11:44:06 UTC
(spark) branch branch-4.x updated: [SPARK-58986][PYTHON] Sort import for python files
This is an automated email from the ASF dual-hosted git repository.
gaogaotiantian pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new b2e4eabbf777 [SPARK-58986][PYTHON] Sort import for python files
b2e4eabbf777 is described below
commit b2e4eabbf77739307605554205f1043e346d80db
Author: Tian Gao <ga...@hotmail.com>
AuthorDate: Tue Aug 25 04:39:18 2026 -0700
[SPARK-58986][PYTHON] Sort import for python files
Add `"I"` to ruff linter select so ruff will check whether import is sorted.
An auto-fix is added to `reformat-python` for import fix.
We actually enforce import order in scala code. Having sorted import makes code clean and it's easier to find issues like duplicate import. It will also make the code more deterministic, reducing the unstable state for circular import. Maintaining this state is easy because dev tool will just do it for you.
No.
CI.
No.
Closes #57921 from gaogaotiantian/sort-import.
Authored-by: Tian Gao <ga...@hotmail.com>
Signed-off-by: Tian Gao <ga...@hotmail.com>
(cherry picked from commit c95d9696402fe0f3b980df5d1b3aa947fabf35f0)
Signed-off-by: Tian Gao <ga...@hotmail.com>
---
.../examples/streaming/kinesis_wordcount_asl.py | 2 +-
dev/check-protos.py | 4 +-
dev/create-release/generate-contributors.py | 7 +-
dev/create-release/generate-llms-txt.py | 2 +-
dev/create-release/releaseutils.py | 8 +-
dev/is-changed.py | 7 +-
dev/merge_spark_pr.py | 3 +-
dev/pip-sanity-check.py | 3 +-
dev/reformat-python | 1 +
dev/run-tests.py | 9 +-
dev/sparktestsupport/modules.py | 2 +-
dev/sparktestsupport/utils.py | 3 +-
dev/structured_logging_style.py | 4 +-
examples/src/main/python/als.py | 2 +-
examples/src/main/python/avro_inputformat.py | 2 +-
examples/src/main/python/logistic_regression.py | 2 -
.../src/main/python/ml/aft_survival_regression.py | 3 +-
examples/src/main/python/ml/als_example.py | 5 +-
examples/src/main/python/ml/binarizer_example.py | 3 +-
.../main/python/ml/bisecting_k_means_example.py | 1 +
.../ml/bucketed_random_projection_lsh_example.py | 3 +-
examples/src/main/python/ml/bucketizer_example.py | 3 +-
.../src/main/python/ml/chi_square_test_example.py | 3 +-
.../src/main/python/ml/chisq_selector_example.py | 3 +-
examples/src/main/python/ml/correlation_example.py | 1 +
.../src/main/python/ml/count_vectorizer_example.py | 3 +-
examples/src/main/python/ml/cross_validator.py | 1 +
examples/src/main/python/ml/dataframe_example.py | 4 +-
examples/src/main/python/ml/dct_example.py | 1 +
.../ml/decision_tree_classification_example.py | 3 +-
.../python/ml/decision_tree_regression_example.py | 5 +-
.../main/python/ml/elementwise_product_example.py | 1 +
.../ml/estimator_transformer_param_example.py | 3 +-
.../src/main/python/ml/feature_hasher_example.py | 3 +-
.../src/main/python/ml/fm_classifier_example.py | 3 +-
.../src/main/python/ml/fm_regressor_example.py | 5 +-
examples/src/main/python/ml/fpgrowth_example.py | 1 +
.../src/main/python/ml/gaussian_mixture_example.py | 1 +
.../ml/generalized_linear_regression_example.py | 3 +-
.../ml/gradient_boosted_tree_classifier_example.py | 3 +-
.../ml/gradient_boosted_tree_regressor_example.py | 5 +-
examples/src/main/python/ml/imputer_example.py | 1 +
.../src/main/python/ml/index_to_string_example.py | 1 +
examples/src/main/python/ml/interaction_example.py | 1 +
.../main/python/ml/isotonic_regression_example.py | 1 +
examples/src/main/python/ml/kmeans_example.py | 2 +-
examples/src/main/python/ml/lda_example.py | 1 +
.../ml/linear_regression_with_elastic_net.py | 1 +
examples/src/main/python/ml/linearsvc.py | 1 +
.../ml/logistic_regression_summary_example.py | 1 +
.../ml/logistic_regression_with_elastic_net.py | 1 +
.../src/main/python/ml/max_abs_scaler_example.py | 1 +
.../src/main/python/ml/min_hash_lsh_example.py | 3 +-
.../src/main/python/ml/min_max_scaler_example.py | 1 +
...ticlass_logistic_regression_with_elastic_net.py | 1 +
.../ml/multilayer_perceptron_classification.py | 1 +
examples/src/main/python/ml/n_gram_example.py | 1 +
examples/src/main/python/ml/naive_bayes_example.py | 1 +
examples/src/main/python/ml/normalizer_example.py | 1 +
examples/src/main/python/ml/one_vs_rest_example.py | 1 +
.../src/main/python/ml/onehot_encoder_example.py | 1 +
examples/src/main/python/ml/pca_example.py | 1 +
examples/src/main/python/ml/pipeline_example.py | 1 +
.../main/python/ml/polynomial_expansion_example.py | 1 +
.../ml/power_iteration_clustering_example.py | 1 +
examples/src/main/python/ml/prefixspan_example.py | 1 +
.../main/python/ml/quantile_discretizer_example.py | 1 +
.../python/ml/random_forest_classifier_example.py | 3 +-
.../python/ml/random_forest_regressor_example.py | 5 +-
examples/src/main/python/ml/rformula_example.py | 1 +
.../src/main/python/ml/robust_scaler_example.py | 1 +
examples/src/main/python/ml/sql_transformer.py | 1 +
.../src/main/python/ml/standard_scaler_example.py | 1 +
.../main/python/ml/stopwords_remover_example.py | 1 +
.../src/main/python/ml/string_indexer_example.py | 1 +
examples/src/main/python/ml/summarizer_example.py | 7 +-
examples/src/main/python/ml/tf_idf_example.py | 3 +-
examples/src/main/python/ml/tokenizer_example.py | 7 +-
.../src/main/python/ml/train_validation_split.py | 1 +
.../ml/univariate_feature_selector_example.py | 3 +-
.../ml/variance_threshold_selector_example.py | 3 +-
.../src/main/python/ml/vector_assembler_example.py | 3 +-
.../src/main/python/ml/vector_indexer_example.py | 1 +
.../src/main/python/ml/vector_size_hint_example.py | 3 +-
.../src/main/python/ml/vector_slicer_example.py | 3 +-
examples/src/main/python/ml/word2vec_example.py | 1 +
.../mllib/binary_classification_metrics_example.py | 2 +
.../main/python/mllib/bisecting_k_means_example.py | 4 +-
examples/src/main/python/mllib/correlations.py | 1 -
.../src/main/python/mllib/correlations_example.py | 3 +-
.../mllib/decision_tree_classification_example.py | 2 +
.../mllib/decision_tree_regression_example.py | 2 +
.../python/mllib/elementwise_product_example.py | 2 +
examples/src/main/python/mllib/fpgrowth_example.py | 2 +-
.../main/python/mllib/gaussian_mixture_example.py | 4 +-
.../main/python/mllib/gaussian_mixture_model.py | 4 +-
.../gradient_boosting_classification_example.py | 2 +
.../mllib/gradient_boosting_regression_example.py | 2 +
.../python/mllib/hypothesis_testing_example.py | 2 +
...esis_testing_kolmogorov_smirnov_test_example.py | 2 +
.../python/mllib/isotonic_regression_example.py | 4 +-
examples/src/main/python/mllib/k_means_example.py | 7 +-
.../mllib/kernel_density_estimation_example.py | 2 +
.../mllib/latent_dirichlet_allocation_example.py | 2 +
.../mllib/linear_regression_with_sgd_example.py | 4 +-
.../src/main/python/mllib/logistic_regression.py | 2 +-
.../logistic_regression_with_lbfgs_example.py | 4 +-
.../python/mllib/multi_class_metrics_example.py | 7 +-
.../python/mllib/multi_label_metrics_example.py | 2 +-
.../src/main/python/mllib/naive_bayes_example.py | 2 +-
.../src/main/python/mllib/normalizer_example.py | 2 +
.../src/main/python/mllib/pca_rowmatrix_example.py | 2 +
.../mllib/power_iteration_clustering_example.py | 2 +
.../mllib/random_forest_classification_example.py | 2 +
.../mllib/random_forest_regression_example.py | 2 +
.../src/main/python/mllib/random_rdd_generation.py | 1 -
.../main/python/mllib/ranking_metrics_example.py | 4 +-
.../main/python/mllib/recommendation_example.py | 1 +
.../python/mllib/regression_metrics_example.py | 7 +-
examples/src/main/python/mllib/sampled_rdds.py | 1 -
.../main/python/mllib/standard_scaler_example.py | 2 +
.../main/python/mllib/streaming_k_means_example.py | 6 +-
.../mllib/streaming_linear_regression_example.py | 9 +-
.../python/mllib/summary_statistics_example.py | 4 +-
examples/src/main/python/mllib/svd_example.py | 2 +
.../src/main/python/mllib/svm_with_sgd_example.py | 4 +-
examples/src/main/python/mllib/tf_idf_example.py | 4 +-
examples/src/main/python/mllib/word2vec_example.py | 2 +
examples/src/main/python/pi.py | 3 +-
examples/src/main/python/sort.py | 1 -
examples/src/main/python/sql/arrow.py | 11 +-
examples/src/main/python/sql/basic.py | 9 +-
examples/src/main/python/sql/datasource.py | 4 +-
examples/src/main/python/sql/hive.py | 4 +-
examples/src/main/python/sql/jdbc.py | 2 +-
.../sql/streaming/structured_kafka_wordcount.py | 3 +-
.../sql/streaming/structured_network_wordcount.py | 3 +-
.../structured_network_wordcount_session_window.py | 12 +-
.../structured_network_wordcount_windowed.py | 4 +-
.../sql/streaming/structured_sessionization.py | 4 +-
examples/src/main/python/sql/udtf.py | 1 +
examples/src/main/python/status_api_demo.py | 4 +-
.../python/streaming/network_wordjoinsentiments.py | 3 +-
.../streaming/recoverable_network_wordcount.py | 2 +-
.../main/python/streaming/sql_network_wordcount.py | 6 +-
examples/src/main/python/wordcount.py | 1 -
pyproject.toml | 1 +
python/benchmarks/bench_eval_type.py | 6 +-
python/benchmarks/bench_local_data_to_arrow.py | 2 +
python/benchmarks/bench_pipelined_udf.py | 1 -
python/conf_viztracer/daemon_viztracer.py | 3 +-
python/conf_viztracer/worker_viztracer.py | 1 -
python/conf_vscode/sitecustomize.py | 3 +-
python/packaging/classic/setup.py | 11 +-
python/packaging/client/setup.py | 9 +-
python/packaging/connect/setup.py | 9 +-
python/pyspark/__init__.py | 26 ++--
python/pyspark/_typing.pyi | 4 +-
python/pyspark/accumulators.py | 13 +-
python/pyspark/conf.py | 6 +-
python/pyspark/core/broadcast.py | 17 +--
python/pyspark/core/context.py | 38 +++---
python/pyspark/core/files.py | 3 +-
python/pyspark/core/rdd.py | 77 ++++++------
python/pyspark/daemon.py | 18 +--
python/pyspark/errors/__init__.py | 40 +++---
python/pyspark/errors/error_classes.py | 2 +-
python/pyspark/errors/exceptions/__init__.py | 1 +
python/pyspark/errors/exceptions/base.py | 4 +-
python/pyspark/errors/exceptions/captured.py | 74 ++++++++---
python/pyspark/errors/exceptions/connect.py | 60 +++++++--
.../errors/tests/test_connect_errors_conversion.py | 21 ++--
python/pyspark/errors/utils.py | 13 +-
python/pyspark/errors_doc_gen.py | 1 +
python/pyspark/install.py | 1 -
python/pyspark/instrumentation_utils.py | 4 +-
python/pyspark/java_gateway.py | 12 +-
python/pyspark/logger/__init__.py | 2 +-
python/pyspark/logger/logger.py | 7 +-
python/pyspark/logger/tests/test_logger.py | 8 +-
python/pyspark/logger/worker_io.py | 4 +-
python/pyspark/memory_profiler_ext.py | 4 +-
python/pyspark/messages/__init__.py | 2 +-
.../socket/spark_socket_message_receiver.py | 4 +-
python/pyspark/messages/spark_message_receiver.py | 3 +-
python/pyspark/messages/zero_copy_byte_stream.py | 2 +-
python/pyspark/ml/__init__.py | 22 ++--
python/pyspark/ml/_typing.pyi | 8 +-
python/pyspark/ml/base.py | 10 +-
python/pyspark/ml/classification.py | 73 +++++------
python/pyspark/ml/clustering.py | 41 ++++---
python/pyspark/ml/common.py | 15 +--
python/pyspark/ml/connect/__init__.py | 12 +-
python/pyspark/ml/connect/base.py | 10 +-
python/pyspark/ml/connect/classification.py | 32 ++---
python/pyspark/ml/connect/evaluation.py | 6 +-
python/pyspark/ml/connect/feature.py | 12 +-
python/pyspark/ml/connect/functions.py | 5 +-
python/pyspark/ml/connect/io_utils.py | 6 +-
python/pyspark/ml/connect/pipeline.py | 6 +-
python/pyspark/ml/connect/proto.py | 2 +-
python/pyspark/ml/connect/readwrite.py | 43 ++++---
python/pyspark/ml/connect/serialize.py | 10 +-
python/pyspark/ml/connect/summarizer.py | 4 +-
python/pyspark/ml/connect/tuning.py | 13 +-
python/pyspark/ml/connect/util.py | 2 +-
.../pyspark/ml/deepspeed/deepspeed_distributor.py | 6 +-
python/pyspark/ml/evaluation.py | 15 +--
python/pyspark/ml/feature.py | 34 +++---
python/pyspark/ml/fpm.py | 11 +-
python/pyspark/ml/functions.py | 10 +-
python/pyspark/ml/image.py | 5 +-
python/pyspark/ml/linalg/__init__.py | 22 ++--
python/pyspark/ml/param/__init__.py | 10 +-
python/pyspark/ml/pipeline.py | 18 +--
python/pyspark/ml/recommendation.py | 17 +--
python/pyspark/ml/regression.py | 51 ++++----
python/pyspark/ml/stat.py | 6 +-
.../pyspark/ml/tests/connect/test_connect_cache.py | 2 +-
.../tests/connect/test_connect_classification.py | 11 +-
.../ml/tests/connect/test_connect_evaluation.py | 3 +-
.../ml/tests/connect/test_connect_feature.py | 7 +-
.../ml/tests/connect/test_connect_function.py | 4 +-
.../tests/connect/test_connect_model_offloading.py | 16 +--
.../ml/tests/connect/test_connect_pipeline.py | 9 +-
.../ml/tests/connect/test_connect_summarizer.py | 7 +-
.../ml/tests/connect/test_connect_tuning.py | 9 +-
.../connect/test_legacy_mode_classification.py | 11 +-
.../tests/connect/test_legacy_mode_evaluation.py | 10 +-
.../ml/tests/connect/test_legacy_mode_feature.py | 11 +-
.../ml/tests/connect/test_legacy_mode_pipeline.py | 11 +-
.../tests/connect/test_legacy_mode_summarizer.py | 4 +-
.../ml/tests/connect/test_legacy_mode_tuning.py | 19 +--
.../tests/connect/test_parity_torch_data_loader.py | 2 +-
.../tests/connect/test_parity_torch_distributor.py | 12 +-
python/pyspark/ml/tests/test_algorithms.py | 6 +-
python/pyspark/ml/tests/test_base.py | 2 +-
python/pyspark/ml/tests/test_classification.py | 40 +++---
python/pyspark/ml/tests/test_clustering.py | 12 +-
python/pyspark/ml/tests/test_dl_util.py | 4 +-
python/pyspark/ml/tests/test_evaluation.py | 4 +-
python/pyspark/ml/tests/test_feature.py | 62 +++++-----
python/pyspark/ml/tests/test_fpm.py | 2 +-
python/pyspark/ml/tests/test_functions.py | 6 +-
python/pyspark/ml/tests/test_image.py | 2 +-
python/pyspark/ml/tests/test_linalg.py | 6 +-
python/pyspark/ml/tests/test_ovr.py | 2 +-
python/pyspark/ml/tests/test_param.py | 8 +-
python/pyspark/ml/tests/test_persistence.py | 8 +-
python/pyspark/ml/tests/test_pipeline.py | 10 +-
python/pyspark/ml/tests/test_regression.py | 22 ++--
python/pyspark/ml/tests/test_stat.py | 4 +-
python/pyspark/ml/tests/test_training_summary.py | 4 +-
python/pyspark/ml/tests/test_tuning.py | 4 +-
python/pyspark/ml/tests/test_wrapper.py | 4 +-
python/pyspark/ml/tests/tuning/test_cv_io_basic.py | 2 +-
.../pyspark/ml/tests/tuning/test_cv_io_nested.py | 2 +-
.../pyspark/ml/tests/tuning/test_cv_io_pipeline.py | 4 +-
.../pyspark/ml/tests/tuning/test_tvs_io_basic.py | 2 +-
.../pyspark/ml/tests/tuning/test_tvs_io_nested.py | 2 +-
.../ml/tests/tuning/test_tvs_io_pipeline.py | 5 +-
python/pyspark/ml/torch/data.py | 2 +-
python/pyspark/ml/torch/distributor.py | 34 +++---
python/pyspark/ml/torch/log_communication.py | 8 +-
python/pyspark/ml/torch/tests/test_data_loader.py | 2 +-
python/pyspark/ml/torch/tests/test_distributor.py | 12 +-
.../ml/torch/tests/test_log_communication.py | 8 +-
python/pyspark/ml/tree.py | 10 +-
python/pyspark/ml/tuning.py | 25 ++--
python/pyspark/ml/util.py | 37 +++---
python/pyspark/ml/wrapper.py | 27 +++--
python/pyspark/mllib/_typing.pyi | 6 +-
python/pyspark/mllib/classification.py | 18 +--
python/pyspark/mllib/clustering.py | 15 ++-
python/pyspark/mllib/common.py | 8 +-
python/pyspark/mllib/evaluation.py | 6 +-
python/pyspark/mllib/feature.py | 14 +--
python/pyspark/mllib/fpm.py | 10 +-
python/pyspark/mllib/linalg/__init__.py | 24 ++--
python/pyspark/mllib/linalg/distributed.py | 12 +-
python/pyspark/mllib/random.py | 3 +-
python/pyspark/mllib/recommendation.py | 1 +
python/pyspark/mllib/regression.py | 16 +--
python/pyspark/mllib/stat/KernelDensity.py | 2 +-
python/pyspark/mllib/stat/__init__.py | 4 +-
python/pyspark/mllib/stat/_statistics.py | 6 +-
python/pyspark/mllib/stat/test.py | 2 +-
python/pyspark/mllib/tests/test_algorithms.py | 10 +-
python/pyspark/mllib/tests/test_feature.py | 6 +-
python/pyspark/mllib/tests/test_linalg.py | 24 ++--
python/pyspark/mllib/tests/test_stat.py | 4 +-
.../mllib/tests/test_streaming_algorithms.py | 4 +-
python/pyspark/mllib/tests/test_util.py | 5 +-
python/pyspark/mllib/tree.py | 8 +-
python/pyspark/mllib/util.py | 13 +-
python/pyspark/pandas/__init__.py | 6 +-
python/pyspark/pandas/_typing.py | 2 +-
python/pyspark/pandas/accessors.py | 35 +++---
python/pyspark/pandas/base.py | 57 ++++-----
python/pyspark/pandas/categorical.py | 9 +-
python/pyspark/pandas/config.py | 11 +-
python/pyspark/pandas/correlation.py | 5 +-
python/pyspark/pandas/data_type_ops/base.py | 38 +++---
python/pyspark/pandas/data_type_ops/binary_ops.py | 2 +-
python/pyspark/pandas/data_type_ops/boolean_ops.py | 17 +--
.../pandas/data_type_ops/categorical_ops.py | 8 +-
python/pyspark/pandas/data_type_ops/complex_ops.py | 5 +-
python/pyspark/pandas/data_type_ops/date_ops.py | 7 +-
.../pyspark/pandas/data_type_ops/datetime_ops.py | 23 ++--
python/pyspark/pandas/data_type_ops/null_ops.py | 5 +-
python/pyspark/pandas/data_type_ops/num_ops.py | 23 ++--
python/pyspark/pandas/data_type_ops/string_ops.py | 9 +-
.../pyspark/pandas/data_type_ops/timedelta_ops.py | 10 +-
python/pyspark/pandas/datetimes.py | 11 +-
python/pyspark/pandas/exceptions.py | 5 +-
python/pyspark/pandas/extensions.py | 8 +-
python/pyspark/pandas/frame.py | 135 +++++++++++----------
python/pyspark/pandas/generic.py | 42 ++++---
python/pyspark/pandas/groupby.py | 72 +++++------
python/pyspark/pandas/indexes/base.py | 63 +++++-----
python/pyspark/pandas/indexes/category.py | 7 +-
python/pyspark/pandas/indexes/datetimes.py | 9 +-
python/pyspark/pandas/indexes/multi.py | 26 ++--
python/pyspark/pandas/indexes/timedelta.py | 4 +-
python/pyspark/pandas/indexing.py | 29 +++--
python/pyspark/pandas/internal.py | 51 ++++----
python/pyspark/pandas/missing/frame.py | 2 +-
python/pyspark/pandas/missing/indexes.py | 2 +-
python/pyspark/pandas/missing/series.py | 2 +-
python/pyspark/pandas/mlflow.py | 18 +--
python/pyspark/pandas/namespace.py | 101 +++++++--------
python/pyspark/pandas/numpy_compat.py | 13 +-
python/pyspark/pandas/plot/core.py | 11 +-
python/pyspark/pandas/plot/matplotlib.py | 37 +++---
python/pyspark/pandas/plot/plotly.py | 11 +-
python/pyspark/pandas/resample.py | 24 ++--
python/pyspark/pandas/series.py | 112 +++++++++--------
python/pyspark/pandas/spark/accessors.py | 18 +--
python/pyspark/pandas/spark/utils.py | 3 +-
python/pyspark/pandas/sql_formatter.py | 18 +--
python/pyspark/pandas/sql_processor.py | 14 ++-
python/pyspark/pandas/strings.py | 11 +-
python/pyspark/pandas/supported_api_gen.py | 8 +-
python/pyspark/pandas/testing.py | 1 +
.../pandas/tests/computation/test_compute.py | 2 +-
.../pyspark/pandas/tests/computation/test_corr.py | 2 +-
.../pandas/tests/computation/test_describe.py | 2 +-
.../pandas/tests/computation/test_idxmax_idxmin.py | 2 +-
.../pyspark/pandas/tests/computation/test_melt.py | 2 +-
.../connect/data_type_ops/test_parity_as_type.py | 2 +-
.../data_type_ops/test_parity_binary_ops.py | 2 +-
.../data_type_ops/test_parity_boolean_ops.py | 4 +-
.../data_type_ops/test_parity_categorical_ops.py | 2 +-
.../data_type_ops/test_parity_complex_ops.py | 2 +-
.../connect/data_type_ops/test_parity_date_ops.py | 2 +-
.../data_type_ops/test_parity_datetime_ops.py | 2 +-
.../connect/data_type_ops/test_parity_null_ops.py | 2 +-
.../data_type_ops/test_parity_num_arithmetic.py | 2 +-
.../connect/data_type_ops/test_parity_num_mod.py | 2 +-
.../data_type_ops/test_parity_num_mul_div.py | 2 +-
.../connect/data_type_ops/test_parity_num_ops.py | 2 +-
.../test_parity_num_ops_fractional_ext.py | 2 +-
...est_parity_num_ops_fractional_ext_astype_cmp.py | 2 +-
.../test_parity_num_ops_integral_ext.py | 2 +-
.../test_parity_num_ops_integral_ext_astype_cmp.py | 2 +-
.../connect/data_type_ops/test_parity_num_pow.py | 2 +-
.../data_type_ops/test_parity_num_reverse.py | 2 +-
.../data_type_ops/test_parity_string_ops.py | 4 +-
.../data_type_ops/test_parity_timedelta_ops.py | 2 +-
.../connect/data_type_ops/test_parity_udt_ops.py | 2 +-
.../connect/diff_frames_ops/test_parity_basic.py | 2 +-
.../connect/indexes/test_parity_indexing_adv.py | 1 -
.../pandas/tests/data_type_ops/test_as_type.py | 4 +-
.../pandas/tests/data_type_ops/test_base.py | 10 +-
.../pandas/tests/data_type_ops/test_binary_ops.py | 2 +-
.../pandas/tests/data_type_ops/test_boolean_ops.py | 6 +-
.../tests/data_type_ops/test_categorical_ops.py | 4 +-
.../pandas/tests/data_type_ops/test_complex_ops.py | 4 +-
.../pandas/tests/data_type_ops/test_date_ops.py | 2 +-
.../tests/data_type_ops/test_datetime_ops.py | 2 +-
.../pandas/tests/data_type_ops/test_null_ops.py | 2 +-
.../tests/data_type_ops/test_num_arithmetic.py | 4 +-
.../pandas/tests/data_type_ops/test_num_mod.py | 4 +-
.../pandas/tests/data_type_ops/test_num_mul_div.py | 6 +-
.../pandas/tests/data_type_ops/test_num_ops.py | 6 +-
.../data_type_ops/test_num_ops_fractional_ext.py | 2 +-
.../test_num_ops_fractional_ext_astype_cmp.py | 4 +-
.../data_type_ops/test_num_ops_integral_ext.py | 2 +-
.../test_num_ops_integral_ext_astype_cmp.py | 4 +-
.../pandas/tests/data_type_ops/test_num_pow.py | 2 +-
.../pandas/tests/data_type_ops/test_num_reverse.py | 2 +-
.../pandas/tests/data_type_ops/test_string_ops.py | 2 +-
.../tests/data_type_ops/test_timedelta_ops.py | 2 +-
.../pandas/tests/data_type_ops/test_udt_ops.py | 2 +-
.../pandas/tests/diff_frames_ops/test_align.py | 2 +-
.../tests/diff_frames_ops/test_arithmetic.py | 4 +-
.../tests/diff_frames_ops/test_arithmetic_chain.py | 4 +-
.../diff_frames_ops/test_arithmetic_chain_ext.py | 6 +-
.../test_arithmetic_chain_ext_float.py | 6 +-
.../tests/diff_frames_ops/test_arithmetic_ext.py | 6 +-
.../diff_frames_ops/test_arithmetic_ext_float.py | 6 +-
.../tests/diff_frames_ops/test_assign_frame.py | 2 +-
.../tests/diff_frames_ops/test_assign_series.py | 2 +-
.../pandas/tests/diff_frames_ops/test_basic.py | 2 +-
.../tests/diff_frames_ops/test_basic_slow.py | 4 +-
.../pandas/tests/diff_frames_ops/test_bitwise.py | 5 +-
.../tests/diff_frames_ops/test_combine_first.py | 2 +-
.../tests/diff_frames_ops/test_compare_series.py | 2 +-
.../tests/diff_frames_ops/test_concat_inner.py | 2 +-
.../tests/diff_frames_ops/test_concat_outer.py | 4 +-
.../pandas/tests/diff_frames_ops/test_corrwith.py | 4 +-
.../pandas/tests/diff_frames_ops/test_cov.py | 2 +-
.../pandas/tests/diff_frames_ops/test_dot_frame.py | 2 +-
.../tests/diff_frames_ops/test_dot_series.py | 2 +-
.../pandas/tests/diff_frames_ops/test_error.py | 4 +-
.../pandas/tests/diff_frames_ops/test_groupby.py | 2 +-
.../diff_frames_ops/test_groupby_aggregate.py | 2 +-
.../tests/diff_frames_ops/test_groupby_apply.py | 2 +-
.../diff_frames_ops/test_groupby_cumulative.py | 2 +-
.../tests/diff_frames_ops/test_groupby_diff.py | 2 +-
.../tests/diff_frames_ops/test_groupby_diff_len.py | 2 +-
.../diff_frames_ops/test_groupby_expanding.py | 2 +-
.../diff_frames_ops/test_groupby_expanding_adv.py | 4 +-
.../test_groupby_expanding_count.py | 4 +-
.../tests/diff_frames_ops/test_groupby_fillna.py | 2 +-
.../tests/diff_frames_ops/test_groupby_filter.py | 2 +-
.../tests/diff_frames_ops/test_groupby_rolling.py | 2 +-
.../diff_frames_ops/test_groupby_rolling_adv.py | 4 +-
.../diff_frames_ops/test_groupby_rolling_count.py | 2 +-
.../tests/diff_frames_ops/test_groupby_shift.py | 2 +-
.../test_groupby_split_apply_combine.py | 2 +-
.../diff_frames_ops/test_groupby_transform.py | 2 +-
.../pandas/tests/diff_frames_ops/test_index.py | 2 +-
.../pandas/tests/diff_frames_ops/test_series.py | 4 +-
.../tests/diff_frames_ops/test_setitem_frame.py | 2 +-
.../tests/diff_frames_ops/test_setitem_series.py | 2 +-
.../pyspark/pandas/tests/frame/test_constructor.py | 3 +-
python/pyspark/pandas/tests/frame/test_spark.py | 8 +-
.../pyspark/pandas/tests/groupby/test_describe.py | 1 +
.../pyspark/pandas/tests/groupby/test_grouping.py | 5 +-
python/pyspark/pandas/tests/groupby/test_index.py | 2 +-
.../pyspark/pandas/tests/groupby/test_missing.py | 2 +-
.../tests/groupby/test_nlargest_nsmallest.py | 3 +-
python/pyspark/pandas/tests/groupby/test_rank.py | 3 +-
python/pyspark/pandas/tests/groupby/test_size.py | 1 +
.../pandas/tests/groupby/test_split_apply_count.py | 2 +-
.../pandas/tests/groupby/test_split_apply_first.py | 2 +-
.../pandas/tests/groupby/test_split_apply_last.py | 2 +-
.../pandas/tests/groupby/test_split_apply_mean.py | 2 +-
.../tests/groupby/test_split_apply_min_max.py | 2 +-
.../pandas/tests/groupby/test_split_apply_skew.py | 2 +-
.../pandas/tests/groupby/test_split_apply_std.py | 2 +-
.../pandas/tests/groupby/test_split_apply_var.py | 2 +-
.../pyspark/pandas/tests/groupby/test_stat_adv.py | 4 +-
.../pyspark/pandas/tests/groupby/test_stat_func.py | 2 +-
.../pandas/tests/groupby/test_stat_median.py | 2 +-
.../pyspark/pandas/tests/groupby/test_stat_prod.py | 2 +-
.../pandas/tests/groupby/test_value_counts.py | 3 +-
.../pandas/tests/indexes/test_conversion.py | 2 +-
.../pandas/tests/indexes/test_datetime_at.py | 2 +-
.../pandas/tests/indexes/test_datetime_between.py | 2 +-
.../pandas/tests/indexes/test_datetime_ceil.py | 2 +-
.../pandas/tests/indexes/test_datetime_floor.py | 2 +-
.../pandas/tests/indexes/test_datetime_iso.py | 2 +-
.../pandas/tests/indexes/test_datetime_map.py | 2 +-
.../pandas/tests/indexes/test_datetime_property.py | 2 +-
.../pandas/tests/indexes/test_datetime_round.py | 2 +-
.../pyspark/pandas/tests/indexes/test_default.py | 2 +-
.../pyspark/pandas/tests/indexes/test_missing.py | 2 +-
.../pandas/tests/indexes/test_reset_index.py | 2 +-
python/pyspark/pandas/tests/io/test_csv.py | 2 +-
.../pandas/tests/io/test_dataframe_conversion.py | 6 +-
python/pyspark/pandas/tests/io/test_io.py | 2 +-
.../pandas/tests/io/test_series_conversion.py | 2 +-
.../pyspark/pandas/tests/plot/test_frame_plot.py | 6 +-
.../tests/plot/test_frame_plot_matplotlib.py | 6 +-
.../pandas/tests/plot/test_frame_plot_plotly.py | 10 +-
.../pyspark/pandas/tests/plot/test_series_plot.py | 4 +-
.../tests/plot/test_series_plot_matplotlib.py | 4 +-
.../pandas/tests/plot/test_series_plot_plotly.py | 8 +-
.../pyspark/pandas/tests/resample/test_missing.py | 2 +-
python/pyspark/pandas/tests/series/test_arg_ops.py | 2 +-
python/pyspark/pandas/tests/series/test_as_type.py | 4 +-
.../pyspark/pandas/tests/series/test_conversion.py | 2 +-
python/pyspark/pandas/tests/series/test_series.py | 12 +-
.../pandas/tests/series/test_string_ops_adv.py | 5 +-
.../pandas/tests/series/test_string_ops_basic.py | 5 +-
.../pyspark/pandas/tests/test_arrow_interface.py | 9 +-
python/pyspark/pandas/tests/test_config.py | 2 +-
python/pyspark/pandas/tests/test_extension.py | 4 +-
python/pyspark/pandas/tests/test_indexops_spark.py | 2 +-
python/pyspark/pandas/tests/test_internal.py | 4 +-
python/pyspark/pandas/tests/test_namespace.py | 10 +-
python/pyspark/pandas/tests/test_numpy_compat.py | 7 +-
python/pyspark/pandas/tests/test_repr.py | 2 +-
python/pyspark/pandas/tests/test_typedef.py | 33 +++--
python/pyspark/pandas/tests/test_utils.py | 4 +-
.../pyspark/pandas/tests/window/test_ewm_error.py | 2 +-
.../pandas/tests/window/test_expanding_adv.py | 2 +-
.../tests/window/test_groupby_expanding_adv.py | 2 +-
.../tests/window/test_groupby_rolling_adv.py | 2 +-
python/pyspark/pandas/tests/window/test_missing.py | 4 +-
.../pandas/tests/window/test_rolling_adv.py | 2 +-
.../pandas/tests/window/test_rolling_error.py | 2 +-
python/pyspark/pandas/typedef/typehints.py | 14 ++-
python/pyspark/pandas/usage_logging/__init__.py | 12 +-
.../pyspark/pandas/usage_logging/usage_logger.py | 2 +-
python/pyspark/pandas/utils.py | 31 ++---
python/pyspark/pandas/window.py | 23 ++--
python/pyspark/pipelines/__init__.py | 2 +-
.../pipelines/add_pipeline_analysis_context.py | 8 +-
python/pyspark/pipelines/api.py | 12 +-
.../pyspark/pipelines/block_session_mutations.py | 2 +-
python/pyspark/pipelines/cli.py | 14 +--
python/pyspark/pipelines/flow.py | 3 +-
python/pyspark/pipelines/graph_element_registry.py | 7 +-
.../spark_connect_graph_element_registry.py | 18 +--
python/pyspark/pipelines/spark_connect_pipeline.py | 4 +-
.../tests/local_graph_element_registry.py | 2 +-
.../tests/test_add_pipeline_analysis_context.py | 5 +-
.../pyspark/pipelines/tests/test_auto_cdc_flow.py | 4 +-
.../tests/test_block_session_mutations.py | 4 +-
python/pyspark/pipelines/tests/test_cli.py | 10 +-
python/pyspark/pipelines/tests/test_decorators.py | 2 +-
.../pipelines/tests/test_graph_element_registry.py | 4 +-
python/pyspark/pipelines/tests/test_init_cli.py | 6 +-
.../pyspark/pipelines/tests/test_spark_connect.py | 4 +-
python/pyspark/pipelines/type_error_utils.py | 1 +
python/pyspark/profiler.py | 14 +--
python/pyspark/rddsampler.py | 5 +-
python/pyspark/resource/__init__.py | 6 +-
python/pyspark/resource/profile.py | 7 +-
python/pyspark/resource/requests.py | 2 +-
.../resource/tests/test_connect_resources.py | 6 +-
python/pyspark/resource/tests/test_resources.py | 3 +-
python/pyspark/resultiterable.py | 2 +-
python/pyspark/serializers.py | 14 +--
python/pyspark/shell.py | 4 +-
python/pyspark/shuffle.py | 18 +--
python/pyspark/sql/__init__.py | 14 +--
python/pyspark/sql/_typing.pyi | 10 +-
python/pyspark/sql/avro/functions.py | 8 +-
python/pyspark/sql/catalog.py | 12 +-
python/pyspark/sql/classic/column.py | 18 +--
python/pyspark/sql/classic/dataframe.py | 79 ++++++------
python/pyspark/sql/classic/table_arg.py | 1 +
python/pyspark/sql/classic/window.py | 12 +-
python/pyspark/sql/column.py | 13 +-
python/pyspark/sql/conf.py | 7 +-
python/pyspark/sql/connect/_typing.py | 4 +-
python/pyspark/sql/connect/avro/functions.py | 11 +-
python/pyspark/sql/connect/catalog.py | 26 ++--
python/pyspark/sql/connect/client/artifact.py | 13 +-
python/pyspark/sql/connect/client/core.py | 109 ++++++++---------
python/pyspark/sql/connect/client/reattach.py | 13 +-
python/pyspark/sql/connect/client/retries.py | 10 +-
python/pyspark/sql/connect/column.py | 33 +++--
python/pyspark/sql/connect/conf.py | 10 +-
python/pyspark/sql/connect/context.py | 17 +--
python/pyspark/sql/connect/conversion.py | 2 +-
python/pyspark/sql/connect/dataframe.py | 101 +++++++--------
python/pyspark/sql/connect/datasource.py | 4 +-
python/pyspark/sql/connect/expressions.py | 78 ++++++------
python/pyspark/sql/connect/functions/__init__.py | 2 +-
python/pyspark/sql/connect/functions/builtin.py | 60 ++++-----
.../pyspark/sql/connect/functions/partitioning.py | 17 ++-
python/pyspark/sql/connect/group.py | 48 ++++----
python/pyspark/sql/connect/local_server_pool.py | 3 +-
python/pyspark/sql/connect/logging.py | 3 +-
python/pyspark/sql/connect/merge.py | 5 +-
python/pyspark/sql/connect/observation.py | 13 +-
python/pyspark/sql/connect/plan.py | 46 ++++---
python/pyspark/sql/connect/protobuf/functions.py | 11 +-
python/pyspark/sql/connect/readwriter.py | 45 +++----
python/pyspark/sql/connect/resource/profile.py | 5 +-
python/pyspark/sql/connect/session.py | 95 ++++++++-------
python/pyspark/sql/connect/shell/progress.py | 6 +-
python/pyspark/sql/connect/sql_formatter.py | 6 +-
python/pyspark/sql/connect/streaming/query.py | 21 ++--
python/pyspark/sql/connect/streaming/readwriter.py | 35 +++---
.../streaming/worker/foreach_batch_worker.py | 13 +-
.../connect/streaming/worker/listener_worker.py | 20 ++-
python/pyspark/sql/connect/table_arg.py | 13 +-
python/pyspark/sql/connect/tvf.py | 7 +-
python/pyspark/sql/connect/types.py | 52 ++++----
python/pyspark/sql/connect/udf.py | 16 +--
python/pyspark/sql/connect/udtf.py | 14 +--
python/pyspark/sql/connect/utils.py | 4 +-
python/pyspark/sql/connect/window.py | 13 +-
python/pyspark/sql/context.py | 33 ++---
python/pyspark/sql/conversion.py | 41 ++++---
python/pyspark/sql/dataframe.py | 27 +++--
python/pyspark/sql/datasource.py | 5 +-
python/pyspark/sql/datasource_internal.py | 10 +-
python/pyspark/sql/functions/__init__.py | 2 +-
python/pyspark/sql/functions/builtin.py | 83 ++++++++-----
python/pyspark/sql/functions/partitioning.py | 11 +-
python/pyspark/sql/group.py | 9 +-
python/pyspark/sql/interchange.py | 2 +-
python/pyspark/sql/internal.py | 7 +-
python/pyspark/sql/merge.py | 6 +-
python/pyspark/sql/metrics.py | 2 +-
python/pyspark/sql/observation.py | 9 +-
python/pyspark/sql/pandas/_typing/__init__.pyi | 13 +-
python/pyspark/sql/pandas/conversion.py | 43 ++++---
python/pyspark/sql/pandas/functions.py | 9 +-
python/pyspark/sql/pandas/group_ops.py | 15 +--
python/pyspark/sql/pandas/map_ops.py | 12 +-
python/pyspark/sql/pandas/serializers.py | 2 +-
python/pyspark/sql/pandas/typehints.py | 25 ++--
python/pyspark/sql/pandas/types.py | 56 ++++-----
python/pyspark/sql/pandas/utils.py | 2 +-
python/pyspark/sql/plot/core.py | 13 +-
python/pyspark/sql/plot/plotly.py | 9 +-
python/pyspark/sql/profiler.py | 8 +-
python/pyspark/sql/protobuf/functions.py | 8 +-
python/pyspark/sql/readwriter.py | 19 +--
python/pyspark/sql/session.py | 50 ++++----
python/pyspark/sql/sql_formatter.py | 6 +-
python/pyspark/sql/streaming/__init__.py | 4 +-
.../benchmark/benchmark_tws_state_server.py | 21 ++--
.../pyspark/sql/streaming/benchmark/tws_utils.py | 2 +-
python/pyspark/sql/streaming/benchmark/utils.py | 4 +-
python/pyspark/sql/streaming/list_state_client.py | 6 +-
python/pyspark/sql/streaming/listener.py | 14 ++-
python/pyspark/sql/streaming/map_state_client.py | 6 +-
.../streaming/python_streaming_source_runner.py | 20 +--
python/pyspark/sql/streaming/query.py | 11 +-
python/pyspark/sql/streaming/readwriter.py | 25 ++--
python/pyspark/sql/streaming/state.py | 4 +-
python/pyspark/sql/streaming/stateful_processor.py | 12 +-
.../sql/streaming/stateful_processor_api_client.py | 19 ++-
.../sql/streaming/stateful_processor_util.py | 15 +--
.../transform_with_state_driver_worker.py | 16 ++-
python/pyspark/sql/streaming/tws_tester.py | 6 +-
python/pyspark/sql/streaming/value_state_client.py | 4 +-
python/pyspark/sql/tests/arrow/test_arrow.py | 50 ++++----
.../tests/arrow/test_arrow_cogrouped_map_misc.py | 2 +-
.../sql/tests/arrow/test_arrow_grouped_map.py | 7 +-
python/pyspark/sql/tests/arrow/test_arrow_map.py | 8 +-
.../tests/arrow/test_arrow_python_aggregator.py | 9 +-
.../sql/tests/arrow/test_arrow_python_udf.py | 6 +-
.../tests/arrow/test_arrow_python_udf_cached.py | 2 +-
python/pyspark/sql/tests/arrow/test_arrow_udf.py | 17 +--
.../sql/tests/arrow/test_arrow_udf_grouped_agg.py | 35 +++---
.../sql/tests/arrow/test_arrow_udf_scalar.py | 45 ++++---
.../sql/tests/arrow/test_arrow_udf_typehints.py | 16 +--
.../sql/tests/arrow/test_arrow_udf_window.py | 17 +--
python/pyspark/sql/tests/arrow/test_arrow_udtf.py | 11 +-
.../tests/coercion/test_pandas_udf_input_type.py | 18 +--
.../tests/coercion/test_pandas_udf_return_type.py | 16 +--
.../tests/coercion/test_python_udf_input_type.py | 18 +--
.../tests/coercion/test_python_udf_return_type.py | 16 +--
.../connect/arrow/test_parity_arrow_python_udf.py | 2 +-
.../sql/tests/connect/client/test_artifact.py | 10 +-
.../sql/tests/connect/client/test_client.py | 24 ++--
.../connect/client/test_client_call_stack_trace.py | 2 +-
.../tests/connect/client/test_client_retries.py | 14 +--
.../sql/tests/connect/client/test_reattach.py | 2 +-
.../test_parity_pandas_transform_with_state.py | 2 +-
...y_pandas_transform_with_state_state_variable.py | 2 +-
.../streaming/test_parity_transform_with_state.py | 2 +-
...t_parity_transform_with_state_state_variable.py | 2 +-
.../sql/tests/connect/shell/test_progress.py | 6 +-
.../connect/streaming/test_parity_foreach_batch.py | 3 +-
.../connect/streaming/test_parity_listener.py | 4 +-
.../sql/tests/connect/test_connect_basic.py | 43 +++----
.../sql/tests/connect/test_connect_channel.py | 7 +-
.../sql/tests/connect/test_connect_collection.py | 2 +-
.../sql/tests/connect/test_connect_column.py | 51 ++++----
.../sql/tests/connect/test_connect_creation.py | 20 +--
.../connect/test_connect_dataframe_property.py | 12 +-
.../sql/tests/connect/test_connect_error.py | 5 +-
.../sql/tests/connect/test_connect_function.py | 18 +--
.../sql/tests/connect/test_connect_local_server.py | 2 +-
.../connect/test_connect_local_server_pool.py | 2 +-
.../pyspark/sql/tests/connect/test_connect_plan.py | 41 +++----
.../sql/tests/connect/test_connect_readwriter.py | 20 +--
.../sql/tests/connect/test_connect_retry.py | 3 +-
.../sql/tests/connect/test_connect_session.py | 9 +-
.../pyspark/sql/tests/connect/test_connect_stat.py | 6 +-
python/pyspark/sql/tests/connect/test_df_debug.py | 2 +-
.../sql/tests/connect/test_parity_frame_plot.py | 2 +-
.../tests/connect/test_parity_frame_plot_plotly.py | 2 +-
.../tests/connect/test_parity_memory_profiler.py | 2 +-
.../sql/tests/connect/test_parity_readwriter.py | 2 +-
.../sql/tests/connect/test_parity_resources.py | 2 +-
.../pyspark/sql/tests/connect/test_parity_udf.py | 2 +-
.../pyspark/sql/tests/connect/test_parity_udtf.py | 7 +-
.../pyspark/sql/tests/connect/test_parity_utils.py | 2 +-
python/pyspark/sql/tests/df_golden/df_golden.py | 1 -
.../scripts/group_by/bool_and_decimal_error.py | 1 +
.../pyspark/sql/tests/df_golden/test_df_golden.py | 3 +-
.../sql/tests/pandas/bench_arrow_columnar_udf.py | 5 +-
.../sql/tests/pandas/bench_pipelined_udf.py | 1 -
.../helper/helper_pandas_transform_with_state.py | 24 ++--
.../streaming/test_pandas_transform_with_state.py | 63 +++++-----
...st_pandas_transform_with_state_checkpoint_v2.py | 2 +-
...t_pandas_transform_with_state_state_variable.py | 44 +++----
...form_with_state_state_variable_checkpoint_v2.py | 2 +-
.../pandas/streaming/test_transform_with_state.py | 7 +-
.../test_transform_with_state_checkpoint_v2.py | 2 +-
.../test_transform_with_state_state_variable.py | 8 +-
...form_with_state_state_variable_checkpoint_v2.py | 2 +-
.../sql/tests/pandas/streaming/test_tws_tester.py | 7 +-
python/pyspark/sql/tests/pandas/test_converter.py | 7 +-
.../sql/tests/pandas/test_pandas_cogrouped_map.py | 6 +-
.../tests/pandas/test_pandas_cogrouped_map_misc.py | 2 +-
.../sql/tests/pandas/test_pandas_grouped_map.py | 28 ++---
.../pandas/test_pandas_grouped_map_with_state.py | 9 +-
python/pyspark/sql/tests/pandas/test_pandas_map.py | 8 +-
.../sql/tests/pandas/test_pandas_sqlmetrics.py | 1 +
python/pyspark/sql/tests/pandas/test_pandas_udf.py | 19 +--
.../tests/pandas/test_pandas_udf_grouped_agg.py | 18 +--
.../sql/tests/pandas/test_pandas_udf_scalar.py | 38 +++---
.../sql/tests/pandas/test_pandas_udf_typehints.py | 12 +-
...pandas_udf_typehints_with_future_annotations.py | 12 +-
.../sql/tests/pandas/test_pandas_udf_window.py | 12 +-
.../pyspark/sql/tests/pandas/test_pipelined_udf.py | 2 +-
python/pyspark/sql/tests/plot/test_frame_plot.py | 5 +-
.../sql/tests/plot/test_frame_plot_plotly.py | 4 +-
python/pyspark/sql/tests/streaming/kafka_utils.py | 2 +-
.../pyspark/sql/tests/streaming/test_streaming.py | 6 +-
.../streaming/test_streaming_foreach_batch.py | 1 +
.../tests/streaming/test_streaming_kafka_rtm.py | 2 +-
.../sql/tests/streaming/test_streaming_listener.py | 4 +-
.../test_streaming_offline_state_repartition.py | 2 +-
python/pyspark/sql/tests/test_artifact.py | 4 +-
python/pyspark/sql/tests/test_catalog.py | 2 +-
python/pyspark/sql/tests/test_collection.py | 26 ++--
python/pyspark/sql/tests/test_column.py | 12 +-
.../sql/tests/test_connect_compatibility.py | 44 +++----
python/pyspark/sql/tests/test_context.py | 2 +-
python/pyspark/sql/tests/test_conversion.py | 7 +-
python/pyspark/sql/tests/test_creation.py | 35 +++---
python/pyspark/sql/tests/test_dataframe.py | 34 +++---
.../sql/tests/test_dataframe_query_context.py | 2 +-
python/pyspark/sql/tests/test_datasources.py | 8 +-
python/pyspark/sql/tests/test_functions.py | 17 +--
python/pyspark/sql/tests/test_group.py | 4 +-
python/pyspark/sql/tests/test_job_cancellation.py | 2 +-
python/pyspark/sql/tests/test_listener.py | 4 +-
python/pyspark/sql/tests/test_observation.py | 5 +-
python/pyspark/sql/tests/test_python_datasource.py | 8 +-
.../sql/tests/test_python_streaming_datasource.py | 19 +--
python/pyspark/sql/tests/test_readwriter.py | 14 ++-
python/pyspark/sql/tests/test_repartition.py | 10 +-
python/pyspark/sql/tests/test_resources.py | 2 +-
python/pyspark/sql/tests/test_serde.py | 2 +-
python/pyspark/sql/tests/test_session.py | 11 +-
python/pyspark/sql/tests/test_stat.py | 19 +--
python/pyspark/sql/tests/test_types.py | 81 ++++++-------
python/pyspark/sql/tests/test_udf.py | 35 +++---
python/pyspark/sql/tests/test_udf_combinations.py | 4 +-
.../sql/tests/test_udf_in_higher_order_function.py | 30 ++++-
python/pyspark/sql/tests/test_udf_profiler.py | 15 +--
.../sql/tests/test_udf_transpile_hypothesis.py | 4 +-
.../pyspark/sql/tests/test_udf_transpile_parity.py | 1 -
.../pyspark/sql/tests/test_udf_transpile_unit.py | 17 ++-
python/pyspark/sql/tests/test_udtf.py | 31 +++--
python/pyspark/sql/tests/test_unified_udf.py | 4 +-
python/pyspark/sql/tests/test_utils.py | 60 ++++-----
python/pyspark/sql/transpile.py | 22 ++--
python/pyspark/sql/tvf.py | 7 +-
python/pyspark/sql/types.py | 57 ++++-----
python/pyspark/sql/udf.py | 17 +--
python/pyspark/sql/udtf.py | 17 +--
python/pyspark/sql/utils.py | 35 +++---
python/pyspark/sql/variant_utils.py | 5 +-
python/pyspark/sql/window.py | 5 +-
python/pyspark/sql/worker/analyze_udtf.py | 9 +-
python/pyspark/sql/worker/create_data_source.py | 6 +-
.../sql/worker/data_source_pushdown_filters.py | 2 +-
python/pyspark/sql/worker/plan_data_source_read.py | 9 +-
.../sql/worker/python_streaming_sink_runner.py | 4 +-
python/pyspark/sql/worker/utils.py | 22 ++--
.../pyspark/sql/worker/write_into_data_source.py | 16 +--
python/pyspark/streaming/context.py | 7 +-
python/pyspark/streaming/dstream.py | 12 +-
python/pyspark/streaming/kinesis.py | 2 +-
python/pyspark/streaming/tests/test_kinesis.py | 6 +-
python/pyspark/streaming/util.py | 6 +-
python/pyspark/taskcontext.py | 7 +-
python/pyspark/testing/connectutils.py | 22 ++--
python/pyspark/testing/goldenutils.py | 2 +-
python/pyspark/testing/mlutils.py | 4 +-
python/pyspark/testing/pandasutils.py | 6 +-
python/pyspark/testing/sqlutils.py | 6 +-
python/pyspark/testing/streamingutils.py | 2 +-
python/pyspark/testing/tests/test_changed_files.py | 2 +-
python/pyspark/testing/utils.py | 26 ++--
python/pyspark/tests/test_broadcast.py | 6 +-
python/pyspark/tests/test_conf.py | 2 +-
python/pyspark/tests/test_context.py | 4 +-
python/pyspark/tests/test_daemon.py | 4 +-
python/pyspark/tests/test_install_spark.py | 8 +-
python/pyspark/tests/test_memory_profiler.py | 2 +-
python/pyspark/tests/test_pin_thread.py | 4 +-
python/pyspark/tests/test_profiler.py | 6 +-
python/pyspark/tests/test_rdd.py | 10 +-
python/pyspark/tests/test_rddsampler.py | 2 +-
python/pyspark/tests/test_serializers.py | 21 ++--
python/pyspark/tests/test_shuffle.py | 8 +-
python/pyspark/tests/test_stage_sched.py | 6 +-
python/pyspark/tests/test_taskcontext.py | 4 +-
python/pyspark/tests/test_util.py | 5 +-
python/pyspark/tests/test_worker.py | 2 +-
.../upstream/pyarrow/test_pyarrow_array_cast.py | 10 +-
.../test_pyarrow_array_from_pandas_default.py | 10 +-
.../test_pyarrow_array_from_pandas_non_default.py | 8 +-
.../pyarrow/test_pyarrow_array_type_inference.py | 3 +
.../test_pyarrow_arrow_to_pandas_default.py | 10 +-
.../test_pyarrow_arrow_to_pandas_non_default.py | 8 +-
.../pyarrow/test_pyarrow_dataframe_from_pandas.py | 6 +-
.../pyarrow/test_pyarrow_ignore_timezone.py | 6 +-
.../upstream/pyarrow/test_pyarrow_table_cast.py | 6 +-
.../pyarrow/test_pyarrow_table_to_pandas.py | 6 +-
.../upstream/pyarrow/test_pyarrow_type_coercion.py | 8 +-
python/pyspark/traceback_utils.py | 2 +-
python/pyspark/util.py | 94 +++++++-------
python/pyspark/worker.py | 76 ++++++------
python/pyspark/worker_message.py | 8 +-
python/pyspark/worker_util.py | 15 ++-
python/run-tests.py | 12 +-
python/test_support/pytorch_training_test_file.py | 5 +-
.../integration-tests/tests/autoscale.py | 1 -
.../integration-tests/tests/decommissioning.py | 1 -
.../tests/decommissioning_cleanup.py | 1 -
.../kubernetes/integration-tests/tests/pyfiles.py | 1 -
.../integration-tests/tests/pyfiles_connect.py | 1 -
.../tests/python_executable_check.py | 1 -
.../integration-tests/tests/worker_memory_check.py | 1 -
sql/core/src/test/resources/script.py | 1 +
sql/gen-sql-api-docs.py | 1 -
sql/gen-sql-config-docs.py | 3 -
sql/gen-sql-functions-docs.py | 2 -
sql/hive/src/test/resources/data/scripts/cat.py | 2 +-
.../test/resources/data/scripts/input20_script.py | 3 +-
838 files changed, 4455 insertions(+), 3938 deletions(-)
diff --git a/connector/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py b/connector/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
index 53a6b69dc93a..73b942ece024 100644
--- a/connector/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
+++ b/connector/kinesis-asl/src/main/python/examples/streaming/kinesis_wordcount_asl.py
@@ -59,7 +59,7 @@ import sys
from pyspark import SparkContext
from pyspark.streaming import StreamingContext
-from pyspark.streaming.kinesis import KinesisUtils, InitialPositionInStream
+from pyspark.streaming.kinesis import InitialPositionInStream, KinesisUtils
if __name__ == "__main__":
if len(sys.argv) != 5:
diff --git a/dev/check-protos.py b/dev/check-protos.py
index 4ddd1f105882..42ad3e005ed1 100755
--- a/dev/check-protos.py
+++ b/dev/check-protos.py
@@ -20,11 +20,11 @@
# Utility for checking whether generated codes in PySpark are out of sync.
# usage: ./dev/check-protos.py
+import filecmp
import os
+import subprocess
import sys
-import filecmp
import tempfile
-import subprocess
# Location of your Spark git development area
SPARK_HOME = os.environ.get("SPARK_HOME", os.getcwd())
diff --git a/dev/create-release/generate-contributors.py b/dev/create-release/generate-contributors.py
index e32d81838e83..69ae19b859be 100755
--- a/dev/create-release/generate-contributors.py
+++ b/dev/create-release/generate-contributors.py
@@ -22,13 +22,12 @@ import os
import sys
from github import Github
-
from releaseutils import (
- tag_exists,
- get_commits,
- yesOrNoPrompt,
contributors_file_name,
+ get_commits,
get_github_name,
+ tag_exists,
+ yesOrNoPrompt,
)
# You must set the following before use!
diff --git a/dev/create-release/generate-llms-txt.py b/dev/create-release/generate-llms-txt.py
index 6175a2e3f398..593c48924cf3 100755
--- a/dev/create-release/generate-llms-txt.py
+++ b/dev/create-release/generate-llms-txt.py
@@ -18,8 +18,8 @@
#
# This script generates llms.txt file for Apache Spark documentation
-import sys
import argparse
+import sys
from pathlib import Path
diff --git a/dev/create-release/releaseutils.py b/dev/create-release/releaseutils.py
index 728e58426b3d..383a574e47aa 100755
--- a/dev/create-release/releaseutils.py
+++ b/dev/create-release/releaseutils.py
@@ -20,11 +20,13 @@
import re
import sys
-from subprocess import Popen, PIPE
+from subprocess import PIPE, Popen
try:
- from github import Github # noqa: F401
- from github import GithubException
+ from github import (
+ Github, # noqa: F401
+ GithubException,
+ )
except ImportError:
print("This tool requires the PyGithub library")
print("Install using 'pip install PyGithub'")
diff --git a/dev/is-changed.py b/dev/is-changed.py
index 92f61c8f07cd..2e34a268f742 100755
--- a/dev/is-changed.py
+++ b/dev/is-changed.py
@@ -17,17 +17,18 @@
# limitations under the License.
#
-import warnings
-import traceback
import os
import sys
+import traceback
+import warnings
from argparse import ArgumentParser
+
+import sparktestsupport.modules as modules
from sparktestsupport.utils import (
determine_modules_for_files,
determine_modules_to_test,
identify_changed_files_from_git_commits,
)
-import sparktestsupport.modules as modules
def parse_opts():
diff --git a/dev/merge_spark_pr.py b/dev/merge_spark_pr.py
index 8d50366550f5..bc793d91e8ff 100755
--- a/dev/merge_spark_pr.py
+++ b/dev/merge_spark_pr.py
@@ -31,9 +31,8 @@ import re
import subprocess
import sys
import traceback
-from urllib.request import urlopen
-from urllib.request import Request
from urllib.error import HTTPError
+from urllib.request import Request, urlopen
try:
import jira.client
diff --git a/dev/pip-sanity-check.py b/dev/pip-sanity-check.py
index fdb1107f930f..27443b4602ef 100644
--- a/dev/pip-sanity-check.py
+++ b/dev/pip-sanity-check.py
@@ -15,9 +15,10 @@
# limitations under the License.
#
-from pyspark.sql import SparkSession
import sys
+from pyspark.sql import SparkSession
+
if __name__ == "__main__":
spark = SparkSession.builder.appName("PipSanityCheck").getOrCreate()
sc = spark.sparkContext
diff --git a/dev/reformat-python b/dev/reformat-python
index e7686f0b85ee..102c74a92992 100755
--- a/dev/reformat-python
+++ b/dev/reformat-python
@@ -30,3 +30,4 @@ if [ $? -ne 0 ]; then
fi
$RUFF_BUILD format python/pyspark dev python/packaging python/benchmarks
+$RUFF_BUILD check --select I --fix
diff --git a/dev/run-tests.py b/dev/run-tests.py
index 1c323ed1b545..35b3a7058927 100755
--- a/dev/run-tests.py
+++ b/dev/run-tests.py
@@ -18,23 +18,23 @@
#
import itertools
-from argparse import ArgumentParser
import os
import re
-import sys
import subprocess
+import sys
import tempfile
+from argparse import ArgumentParser
from contextlib import contextmanager
+import sparktestsupport.modules as modules
from sparktestsupport import SPARK_HOME, USER_HOME
-from sparktestsupport.shellutils import exit_from_command_with_retcode, run_cmd, rm_r, which
+from sparktestsupport.shellutils import exit_from_command_with_retcode, rm_r, run_cmd, which
from sparktestsupport.utils import (
determine_dangling_python_tests,
determine_modules_for_files,
determine_modules_to_test,
identify_changed_files_from_git_commits,
)
-import sparktestsupport.modules as modules
def setup_test_environ(environ):
@@ -665,6 +665,7 @@ def main():
def _test():
import doctest
+
import sparktestsupport.modules
import sparktestsupport.utils
diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index f80cde17b93a..5d3efb657b9c 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -15,11 +15,11 @@
# limitations under the License.
#
-from functools import total_ordering
import itertools
import os
import re
import sys
+from functools import total_ordering
from pathlib import Path, PurePath
all_modules = []
diff --git a/dev/sparktestsupport/utils.py b/dev/sparktestsupport/utils.py
index 39bdeed5e934..b177952eb31d 100755
--- a/dev/sparktestsupport/utils.py
+++ b/dev/sparktestsupport/utils.py
@@ -18,8 +18,9 @@
#
import os
-import sys
import subprocess
+import sys
+
from sparktestsupport import modules
from sparktestsupport.shellutils import run_cmd
from sparktestsupport.toposort import toposort_flatten
diff --git a/dev/structured_logging_style.py b/dev/structured_logging_style.py
index b0d1b2a76e4a..9689a6455bcc 100755
--- a/dev/structured_logging_style.py
+++ b/dev/structured_logging_style.py
@@ -17,10 +17,10 @@
# limitations under the License.
#
+import glob
import os
-import sys
import re
-import glob
+import sys
def main():
diff --git a/examples/src/main/python/als.py b/examples/src/main/python/als.py
index 5bd1807cce1b..37d3ed22c6bd 100755
--- a/examples/src/main/python/als.py
+++ b/examples/src/main/python/als.py
@@ -24,8 +24,8 @@ This example requires numpy (http://www.numpy.org/)
import sys
import numpy as np
-from numpy.random import rand
from numpy import matrix
+from numpy.random import rand
from pyspark.sql import SparkSession
LAMBDA = 0.01 # regularization
diff --git a/examples/src/main/python/avro_inputformat.py b/examples/src/main/python/avro_inputformat.py
index 97475dde14ec..19c02f899923 100644
--- a/examples/src/main/python/avro_inputformat.py
+++ b/examples/src/main/python/avro_inputformat.py
@@ -44,9 +44,9 @@ $ ./bin/spark-submit --driver-class-path /path/to/example/jar \
{u'favorite_color': u'red', u'name': u'Ben'}
"""
import sys
+from functools import reduce
from typing import Any, Tuple
-from functools import reduce
from pyspark import RDD
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/logistic_regression.py b/examples/src/main/python/logistic_regression.py
index 9645af619b1e..788729bf8150 100755
--- a/examples/src/main/python/logistic_regression.py
+++ b/examples/src/main/python/logistic_regression.py
@@ -25,11 +25,9 @@ ML, as shown in examples/src/main/python/ml/logistic_regression_with_elastic_net
import sys
from typing import Iterable, List
-
import numpy as np
from pyspark.sql import SparkSession
-
D = 10 # Number of dimensions
diff --git a/examples/src/main/python/ml/aft_survival_regression.py b/examples/src/main/python/ml/aft_survival_regression.py
index 2040a7876c7f..7f4092784892 100644
--- a/examples/src/main/python/ml/aft_survival_regression.py
+++ b/examples/src/main/python/ml/aft_survival_regression.py
@@ -21,8 +21,9 @@ Run with:
bin/spark-submit examples/src/main/python/ml/aft_survival_regression.py
"""
# $example on$
-from pyspark.ml.regression import AFTSurvivalRegression
from pyspark.ml.linalg import Vectors
+from pyspark.ml.regression import AFTSurvivalRegression
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/als_example.py b/examples/src/main/python/ml/als_example.py
index b39263978402..7eebc225c5ed 100644
--- a/examples/src/main/python/ml/als_example.py
+++ b/examples/src/main/python/ml/als_example.py
@@ -15,12 +15,11 @@
# limitations under the License.
#
-from pyspark.sql import SparkSession
-
# $example on$
from pyspark.ml.evaluation import RegressionEvaluator
from pyspark.ml.recommendation import ALS
-from pyspark.sql import Row
+from pyspark.sql import Row, SparkSession
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/ml/binarizer_example.py b/examples/src/main/python/ml/binarizer_example.py
index 5d5ae4122e1d..265266b40de0 100644
--- a/examples/src/main/python/ml/binarizer_example.py
+++ b/examples/src/main/python/ml/binarizer_example.py
@@ -15,9 +15,10 @@
# limitations under the License.
#
-from pyspark.sql import SparkSession
# $example on$
from pyspark.ml.feature import Binarizer
+from pyspark.sql import SparkSession
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/ml/bisecting_k_means_example.py b/examples/src/main/python/ml/bisecting_k_means_example.py
index 513f80a09ef0..fc4fb84957a6 100644
--- a/examples/src/main/python/ml/bisecting_k_means_example.py
+++ b/examples/src/main/python/ml/bisecting_k_means_example.py
@@ -23,6 +23,7 @@ Run with:
# $example on$
from pyspark.ml.clustering import BisectingKMeans
from pyspark.ml.evaluation import ClusteringEvaluator
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/bucketed_random_projection_lsh_example.py b/examples/src/main/python/ml/bucketed_random_projection_lsh_example.py
index f5836091f35b..9d156e8720cd 100644
--- a/examples/src/main/python/ml/bucketed_random_projection_lsh_example.py
+++ b/examples/src/main/python/ml/bucketed_random_projection_lsh_example.py
@@ -23,9 +23,10 @@ Run with:
# $example on$
from pyspark.ml.feature import BucketedRandomProjectionLSH
from pyspark.ml.linalg import Vectors
-from pyspark.sql.functions import col
+
# $example off$
from pyspark.sql import SparkSession
+from pyspark.sql.functions import col
if __name__ == "__main__":
spark = SparkSession \
diff --git a/examples/src/main/python/ml/bucketizer_example.py b/examples/src/main/python/ml/bucketizer_example.py
index bad5d787e134..71d867046a37 100644
--- a/examples/src/main/python/ml/bucketizer_example.py
+++ b/examples/src/main/python/ml/bucketizer_example.py
@@ -15,9 +15,10 @@
# limitations under the License.
#
-from pyspark.sql import SparkSession
# $example on$
from pyspark.ml.feature import Bucketizer
+from pyspark.sql import SparkSession
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/ml/chi_square_test_example.py b/examples/src/main/python/ml/chi_square_test_example.py
index 0360742faf6f..5939ef27c120 100644
--- a/examples/src/main/python/ml/chi_square_test_example.py
+++ b/examples/src/main/python/ml/chi_square_test_example.py
@@ -20,10 +20,11 @@ An example for Chi-square hypothesis testing.
Run with:
bin/spark-submit examples/src/main/python/ml/chi_square_test_example.py
"""
-from pyspark.sql import SparkSession
# $example on$
from pyspark.ml.linalg import Vectors
from pyspark.ml.stat import ChiSquareTest
+from pyspark.sql import SparkSession
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/ml/chisq_selector_example.py b/examples/src/main/python/ml/chisq_selector_example.py
index c83a8c1bc7b2..4933c1a403fa 100644
--- a/examples/src/main/python/ml/chisq_selector_example.py
+++ b/examples/src/main/python/ml/chisq_selector_example.py
@@ -15,10 +15,11 @@
# limitations under the License.
#
-from pyspark.sql import SparkSession
# $example on$
from pyspark.ml.feature import ChiSqSelector
from pyspark.ml.linalg import Vectors
+from pyspark.sql import SparkSession
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/ml/correlation_example.py b/examples/src/main/python/ml/correlation_example.py
index b15535a59882..1d56ec0e9128 100644
--- a/examples/src/main/python/ml/correlation_example.py
+++ b/examples/src/main/python/ml/correlation_example.py
@@ -23,6 +23,7 @@ Run with:
# $example on$
from pyspark.ml.linalg import Vectors
from pyspark.ml.stat import Correlation
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/count_vectorizer_example.py b/examples/src/main/python/ml/count_vectorizer_example.py
index b3ddfb128c3d..83f4233807ed 100644
--- a/examples/src/main/python/ml/count_vectorizer_example.py
+++ b/examples/src/main/python/ml/count_vectorizer_example.py
@@ -15,9 +15,10 @@
# limitations under the License.
#
-from pyspark.sql import SparkSession
# $example on$
from pyspark.ml.feature import CountVectorizer
+from pyspark.sql import SparkSession
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/ml/cross_validator.py b/examples/src/main/python/ml/cross_validator.py
index 0ad086548695..a3bc2b55be9b 100644
--- a/examples/src/main/python/ml/cross_validator.py
+++ b/examples/src/main/python/ml/cross_validator.py
@@ -28,6 +28,7 @@ from pyspark.ml.classification import LogisticRegression
from pyspark.ml.evaluation import BinaryClassificationEvaluator
from pyspark.ml.feature import HashingTF, Tokenizer
from pyspark.ml.tuning import CrossValidator, ParamGridBuilder
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/dataframe_example.py b/examples/src/main/python/ml/dataframe_example.py
index d2bf93744113..16e4b3a5523e 100644
--- a/examples/src/main/python/ml/dataframe_example.py
+++ b/examples/src/main/python/ml/dataframe_example.py
@@ -20,13 +20,13 @@ An example of how to use DataFrame for ML. Run with::
bin/spark-submit examples/src/main/python/ml/dataframe_example.py <input_path>
"""
import os
+import shutil
import sys
import tempfile
-import shutil
-from pyspark.sql import SparkSession
from pyspark.mllib.stat import Statistics
from pyspark.mllib.util import MLUtils
+from pyspark.sql import SparkSession
if __name__ == "__main__":
if len(sys.argv) > 2:
diff --git a/examples/src/main/python/ml/dct_example.py b/examples/src/main/python/ml/dct_example.py
index 37da4f5e8f1c..e8dba4182374 100644
--- a/examples/src/main/python/ml/dct_example.py
+++ b/examples/src/main/python/ml/dct_example.py
@@ -18,6 +18,7 @@
# $example on$
from pyspark.ml.feature import DCT
from pyspark.ml.linalg import Vectors
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/decision_tree_classification_example.py b/examples/src/main/python/ml/decision_tree_classification_example.py
index eb7177b84535..82a1bf9ac0f6 100644
--- a/examples/src/main/python/ml/decision_tree_classification_example.py
+++ b/examples/src/main/python/ml/decision_tree_classification_example.py
@@ -21,8 +21,9 @@ Decision Tree Classification Example.
# $example on$
from pyspark.ml import Pipeline
from pyspark.ml.classification import DecisionTreeClassifier
-from pyspark.ml.feature import StringIndexer, VectorIndexer
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
+from pyspark.ml.feature import StringIndexer, VectorIndexer
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/decision_tree_regression_example.py b/examples/src/main/python/ml/decision_tree_regression_example.py
index 1ed1636a3d96..9eafcc9ef0ec 100644
--- a/examples/src/main/python/ml/decision_tree_regression_example.py
+++ b/examples/src/main/python/ml/decision_tree_regression_example.py
@@ -20,9 +20,10 @@ Decision Tree Regression Example.
"""
# $example on$
from pyspark.ml import Pipeline
-from pyspark.ml.regression import DecisionTreeRegressor
-from pyspark.ml.feature import VectorIndexer
from pyspark.ml.evaluation import RegressionEvaluator
+from pyspark.ml.feature import VectorIndexer
+from pyspark.ml.regression import DecisionTreeRegressor
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/elementwise_product_example.py b/examples/src/main/python/ml/elementwise_product_example.py
index 71eec8d43299..e972db907a3a 100644
--- a/examples/src/main/python/ml/elementwise_product_example.py
+++ b/examples/src/main/python/ml/elementwise_product_example.py
@@ -18,6 +18,7 @@
# $example on$
from pyspark.ml.feature import ElementwiseProduct
from pyspark.ml.linalg import Vectors
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/estimator_transformer_param_example.py b/examples/src/main/python/ml/estimator_transformer_param_example.py
index b34861a97fa7..e981415f0dd4 100644
--- a/examples/src/main/python/ml/estimator_transformer_param_example.py
+++ b/examples/src/main/python/ml/estimator_transformer_param_example.py
@@ -19,8 +19,9 @@
Estimator Transformer Param Example.
"""
# $example on$
-from pyspark.ml.linalg import Vectors
from pyspark.ml.classification import LogisticRegression
+from pyspark.ml.linalg import Vectors
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/feature_hasher_example.py b/examples/src/main/python/ml/feature_hasher_example.py
index 4fe573d19dfb..64e8a047b017 100644
--- a/examples/src/main/python/ml/feature_hasher_example.py
+++ b/examples/src/main/python/ml/feature_hasher_example.py
@@ -15,9 +15,10 @@
# limitations under the License.
#
-from pyspark.sql import SparkSession
# $example on$
from pyspark.ml.feature import FeatureHasher
+from pyspark.sql import SparkSession
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/ml/fm_classifier_example.py b/examples/src/main/python/ml/fm_classifier_example.py
index da49e5fc2baa..d3fba01b85f5 100644
--- a/examples/src/main/python/ml/fm_classifier_example.py
+++ b/examples/src/main/python/ml/fm_classifier_example.py
@@ -21,8 +21,9 @@ FMClassifier Example.
# $example on$
from pyspark.ml import Pipeline
from pyspark.ml.classification import FMClassifier
-from pyspark.ml.feature import MinMaxScaler, StringIndexer
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
+from pyspark.ml.feature import MinMaxScaler, StringIndexer
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/fm_regressor_example.py b/examples/src/main/python/ml/fm_regressor_example.py
index 47544b632420..02527a9a34f6 100644
--- a/examples/src/main/python/ml/fm_regressor_example.py
+++ b/examples/src/main/python/ml/fm_regressor_example.py
@@ -20,9 +20,10 @@ FMRegressor Example.
"""
# $example on$
from pyspark.ml import Pipeline
-from pyspark.ml.regression import FMRegressor
-from pyspark.ml.feature import MinMaxScaler
from pyspark.ml.evaluation import RegressionEvaluator
+from pyspark.ml.feature import MinMaxScaler
+from pyspark.ml.regression import FMRegressor
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/fpgrowth_example.py b/examples/src/main/python/ml/fpgrowth_example.py
index 39092e616d42..d2ab9bdd7c96 100644
--- a/examples/src/main/python/ml/fpgrowth_example.py
+++ b/examples/src/main/python/ml/fpgrowth_example.py
@@ -22,6 +22,7 @@ Run with:
"""
# $example on$
from pyspark.ml.fpm import FPGrowth
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/gaussian_mixture_example.py b/examples/src/main/python/ml/gaussian_mixture_example.py
index 1441faa79298..a8901f43f0a7 100644
--- a/examples/src/main/python/ml/gaussian_mixture_example.py
+++ b/examples/src/main/python/ml/gaussian_mixture_example.py
@@ -22,6 +22,7 @@ Run with:
"""
# $example on$
from pyspark.ml.clustering import GaussianMixture
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/generalized_linear_regression_example.py b/examples/src/main/python/ml/generalized_linear_regression_example.py
index 06a8a5a2e942..df3801607977 100644
--- a/examples/src/main/python/ml/generalized_linear_regression_example.py
+++ b/examples/src/main/python/ml/generalized_linear_regression_example.py
@@ -20,9 +20,10 @@ An example demonstrating generalized linear regression.
Run with:
bin/spark-submit examples/src/main/python/ml/generalized_linear_regression_example.py
"""
-from pyspark.sql import SparkSession
# $example on$
from pyspark.ml.regression import GeneralizedLinearRegression
+from pyspark.sql import SparkSession
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/ml/gradient_boosted_tree_classifier_example.py b/examples/src/main/python/ml/gradient_boosted_tree_classifier_example.py
index a7efa2170a06..803ae70e163f 100644
--- a/examples/src/main/python/ml/gradient_boosted_tree_classifier_example.py
+++ b/examples/src/main/python/ml/gradient_boosted_tree_classifier_example.py
@@ -21,8 +21,9 @@ Gradient Boosted Tree Classifier Example.
# $example on$
from pyspark.ml import Pipeline
from pyspark.ml.classification import GBTClassifier
-from pyspark.ml.feature import StringIndexer, VectorIndexer
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
+from pyspark.ml.feature import StringIndexer, VectorIndexer
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/gradient_boosted_tree_regressor_example.py b/examples/src/main/python/ml/gradient_boosted_tree_regressor_example.py
index 5e09b96c1ea3..821ae6c6f8d8 100644
--- a/examples/src/main/python/ml/gradient_boosted_tree_regressor_example.py
+++ b/examples/src/main/python/ml/gradient_boosted_tree_regressor_example.py
@@ -20,9 +20,10 @@ Gradient Boosted Tree Regressor Example.
"""
# $example on$
from pyspark.ml import Pipeline
-from pyspark.ml.regression import GBTRegressor
-from pyspark.ml.feature import VectorIndexer
from pyspark.ml.evaluation import RegressionEvaluator
+from pyspark.ml.feature import VectorIndexer
+from pyspark.ml.regression import GBTRegressor
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/imputer_example.py b/examples/src/main/python/ml/imputer_example.py
index 9ba014776361..cec250a17277 100644
--- a/examples/src/main/python/ml/imputer_example.py
+++ b/examples/src/main/python/ml/imputer_example.py
@@ -22,6 +22,7 @@ Run with:
"""
# $example on$
from pyspark.ml.feature import Imputer
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/index_to_string_example.py b/examples/src/main/python/ml/index_to_string_example.py
index 98bdb89ce303..16e5a6510568 100644
--- a/examples/src/main/python/ml/index_to_string_example.py
+++ b/examples/src/main/python/ml/index_to_string_example.py
@@ -17,6 +17,7 @@
# $example on$
from pyspark.ml.feature import IndexToString, StringIndexer
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/interaction_example.py b/examples/src/main/python/ml/interaction_example.py
index ac365179b0c2..bc8f4441fcb7 100644
--- a/examples/src/main/python/ml/interaction_example.py
+++ b/examples/src/main/python/ml/interaction_example.py
@@ -17,6 +17,7 @@
# $example on$
from pyspark.ml.feature import Interaction, VectorAssembler
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/isotonic_regression_example.py b/examples/src/main/python/ml/isotonic_regression_example.py
index d7b893894fc7..76f462389b5d 100644
--- a/examples/src/main/python/ml/isotonic_regression_example.py
+++ b/examples/src/main/python/ml/isotonic_regression_example.py
@@ -23,6 +23,7 @@ Run with:
"""
# $example on$
from pyspark.ml.regression import IsotonicRegression
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/kmeans_example.py b/examples/src/main/python/ml/kmeans_example.py
index 47223fd953d1..939e101179e8 100644
--- a/examples/src/main/python/ml/kmeans_example.py
+++ b/examples/src/main/python/ml/kmeans_example.py
@@ -25,8 +25,8 @@ This example requires NumPy (http://www.numpy.org/).
# $example on$
from pyspark.ml.clustering import KMeans
from pyspark.ml.evaluation import ClusteringEvaluator
-# $example off$
+# $example off$
from pyspark.sql import SparkSession
if __name__ == "__main__":
diff --git a/examples/src/main/python/ml/lda_example.py b/examples/src/main/python/ml/lda_example.py
index a47dfa383c89..4999314f032e 100644
--- a/examples/src/main/python/ml/lda_example.py
+++ b/examples/src/main/python/ml/lda_example.py
@@ -22,6 +22,7 @@ Run with:
"""
# $example on$
from pyspark.ml.clustering import LDA
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/linear_regression_with_elastic_net.py b/examples/src/main/python/ml/linear_regression_with_elastic_net.py
index 864fc76cff13..8282614ec4c4 100644
--- a/examples/src/main/python/ml/linear_regression_with_elastic_net.py
+++ b/examples/src/main/python/ml/linear_regression_with_elastic_net.py
@@ -17,6 +17,7 @@
# $example on$
from pyspark.ml.regression import LinearRegression
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/linearsvc.py b/examples/src/main/python/ml/linearsvc.py
index 61d726cf3f1a..f04106a3214b 100644
--- a/examples/src/main/python/ml/linearsvc.py
+++ b/examples/src/main/python/ml/linearsvc.py
@@ -17,6 +17,7 @@
# $example on$
from pyspark.ml.classification import LinearSVC
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/logistic_regression_summary_example.py b/examples/src/main/python/ml/logistic_regression_summary_example.py
index 6d045108da0a..10b24bd848e7 100644
--- a/examples/src/main/python/ml/logistic_regression_summary_example.py
+++ b/examples/src/main/python/ml/logistic_regression_summary_example.py
@@ -22,6 +22,7 @@ Run with:
"""
# $example on$
from pyspark.ml.classification import LogisticRegression
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/logistic_regression_with_elastic_net.py b/examples/src/main/python/ml/logistic_regression_with_elastic_net.py
index 916fdade2762..63c4a9760e93 100644
--- a/examples/src/main/python/ml/logistic_regression_with_elastic_net.py
+++ b/examples/src/main/python/ml/logistic_regression_with_elastic_net.py
@@ -17,6 +17,7 @@
# $example on$
from pyspark.ml.classification import LogisticRegression
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/max_abs_scaler_example.py b/examples/src/main/python/ml/max_abs_scaler_example.py
index d7ff3561ce42..76e6afff1f9c 100644
--- a/examples/src/main/python/ml/max_abs_scaler_example.py
+++ b/examples/src/main/python/ml/max_abs_scaler_example.py
@@ -18,6 +18,7 @@
# $example on$
from pyspark.ml.feature import MaxAbsScaler
from pyspark.ml.linalg import Vectors
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/min_hash_lsh_example.py b/examples/src/main/python/ml/min_hash_lsh_example.py
index 683f97a055ed..bbaa69bafaea 100644
--- a/examples/src/main/python/ml/min_hash_lsh_example.py
+++ b/examples/src/main/python/ml/min_hash_lsh_example.py
@@ -23,9 +23,10 @@ Run with:
# $example on$
from pyspark.ml.feature import MinHashLSH
from pyspark.ml.linalg import Vectors
-from pyspark.sql.functions import col
+
# $example off$
from pyspark.sql import SparkSession
+from pyspark.sql.functions import col
if __name__ == "__main__":
spark = SparkSession \
diff --git a/examples/src/main/python/ml/min_max_scaler_example.py b/examples/src/main/python/ml/min_max_scaler_example.py
index cd7424369989..5b3ae052b16b 100644
--- a/examples/src/main/python/ml/min_max_scaler_example.py
+++ b/examples/src/main/python/ml/min_max_scaler_example.py
@@ -18,6 +18,7 @@
# $example on$
from pyspark.ml.feature import MinMaxScaler
from pyspark.ml.linalg import Vectors
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/multiclass_logistic_regression_with_elastic_net.py b/examples/src/main/python/ml/multiclass_logistic_regression_with_elastic_net.py
index 3bb4a7286410..d142b1946c11 100644
--- a/examples/src/main/python/ml/multiclass_logistic_regression_with_elastic_net.py
+++ b/examples/src/main/python/ml/multiclass_logistic_regression_with_elastic_net.py
@@ -17,6 +17,7 @@
# $example on$
from pyspark.ml.classification import LogisticRegression
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/multilayer_perceptron_classification.py b/examples/src/main/python/ml/multilayer_perceptron_classification.py
index 74f532193573..e5c337e14c20 100644
--- a/examples/src/main/python/ml/multilayer_perceptron_classification.py
+++ b/examples/src/main/python/ml/multilayer_perceptron_classification.py
@@ -18,6 +18,7 @@
# $example on$
from pyspark.ml.classification import MultilayerPerceptronClassifier
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/n_gram_example.py b/examples/src/main/python/ml/n_gram_example.py
index 8c8031b93945..2872c29b9c2a 100644
--- a/examples/src/main/python/ml/n_gram_example.py
+++ b/examples/src/main/python/ml/n_gram_example.py
@@ -17,6 +17,7 @@
# $example on$
from pyspark.ml.feature import NGram
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/naive_bayes_example.py b/examples/src/main/python/ml/naive_bayes_example.py
index 8d1777c6f9e3..a098962e1b4e 100644
--- a/examples/src/main/python/ml/naive_bayes_example.py
+++ b/examples/src/main/python/ml/naive_bayes_example.py
@@ -18,6 +18,7 @@
# $example on$
from pyspark.ml.classification import NaiveBayes
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/normalizer_example.py b/examples/src/main/python/ml/normalizer_example.py
index 2aa012961a2e..fd17a3a7c5d3 100644
--- a/examples/src/main/python/ml/normalizer_example.py
+++ b/examples/src/main/python/ml/normalizer_example.py
@@ -18,6 +18,7 @@
# $example on$
from pyspark.ml.feature import Normalizer
from pyspark.ml.linalg import Vectors
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/one_vs_rest_example.py b/examples/src/main/python/ml/one_vs_rest_example.py
index 4cae1a99808e..e8d8fc5e59c4 100644
--- a/examples/src/main/python/ml/one_vs_rest_example.py
+++ b/examples/src/main/python/ml/one_vs_rest_example.py
@@ -24,6 +24,7 @@ Run with:
# $example on$
from pyspark.ml.classification import LogisticRegression, OneVsRest
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/onehot_encoder_example.py b/examples/src/main/python/ml/onehot_encoder_example.py
index 6deb84ed785c..f52ecade792c 100644
--- a/examples/src/main/python/ml/onehot_encoder_example.py
+++ b/examples/src/main/python/ml/onehot_encoder_example.py
@@ -17,6 +17,7 @@
# $example on$
from pyspark.ml.feature import OneHotEncoder
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/pca_example.py b/examples/src/main/python/ml/pca_example.py
index 03fb709c8e91..32b2263efc45 100644
--- a/examples/src/main/python/ml/pca_example.py
+++ b/examples/src/main/python/ml/pca_example.py
@@ -18,6 +18,7 @@
# $example on$
from pyspark.ml.feature import PCA
from pyspark.ml.linalg import Vectors
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/pipeline_example.py b/examples/src/main/python/ml/pipeline_example.py
index 5fff3d61d5c4..b8cf59af285b 100644
--- a/examples/src/main/python/ml/pipeline_example.py
+++ b/examples/src/main/python/ml/pipeline_example.py
@@ -23,6 +23,7 @@ Pipeline Example.
from pyspark.ml import Pipeline
from pyspark.ml.classification import LogisticRegression
from pyspark.ml.feature import HashingTF, Tokenizer
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/polynomial_expansion_example.py b/examples/src/main/python/ml/polynomial_expansion_example.py
index 75f436e768dc..09e89924d1c4 100644
--- a/examples/src/main/python/ml/polynomial_expansion_example.py
+++ b/examples/src/main/python/ml/polynomial_expansion_example.py
@@ -18,6 +18,7 @@
# $example on$
from pyspark.ml.feature import PolynomialExpansion
from pyspark.ml.linalg import Vectors
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/power_iteration_clustering_example.py b/examples/src/main/python/ml/power_iteration_clustering_example.py
index c983c4ad2b0d..3a2bbc856d8a 100644
--- a/examples/src/main/python/ml/power_iteration_clustering_example.py
+++ b/examples/src/main/python/ml/power_iteration_clustering_example.py
@@ -22,6 +22,7 @@ Run with:
"""
# $example on$
from pyspark.ml.clustering import PowerIterationClustering
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/prefixspan_example.py b/examples/src/main/python/ml/prefixspan_example.py
index 88d1d4197341..25cf0880a10e 100644
--- a/examples/src/main/python/ml/prefixspan_example.py
+++ b/examples/src/main/python/ml/prefixspan_example.py
@@ -22,6 +22,7 @@ Run with:
"""
# $example on$
from pyspark.ml.fpm import PrefixSpan
+
# $example off$
from pyspark.sql import Row, SparkSession
diff --git a/examples/src/main/python/ml/quantile_discretizer_example.py b/examples/src/main/python/ml/quantile_discretizer_example.py
index 82be3936d259..fa7cc74ae1e6 100644
--- a/examples/src/main/python/ml/quantile_discretizer_example.py
+++ b/examples/src/main/python/ml/quantile_discretizer_example.py
@@ -17,6 +17,7 @@
# $example on$
from pyspark.ml.feature import QuantileDiscretizer
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/random_forest_classifier_example.py b/examples/src/main/python/ml/random_forest_classifier_example.py
index 8983d1f2e979..a07ab40a2e7e 100644
--- a/examples/src/main/python/ml/random_forest_classifier_example.py
+++ b/examples/src/main/python/ml/random_forest_classifier_example.py
@@ -21,8 +21,9 @@ Random Forest Classifier Example.
# $example on$
from pyspark.ml import Pipeline
from pyspark.ml.classification import RandomForestClassifier
-from pyspark.ml.feature import IndexToString, StringIndexer, VectorIndexer
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
+from pyspark.ml.feature import IndexToString, StringIndexer, VectorIndexer
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/random_forest_regressor_example.py b/examples/src/main/python/ml/random_forest_regressor_example.py
index b9306ddf2f82..98cf22e49b9e 100644
--- a/examples/src/main/python/ml/random_forest_regressor_example.py
+++ b/examples/src/main/python/ml/random_forest_regressor_example.py
@@ -20,9 +20,10 @@ Random Forest Regressor Example.
"""
# $example on$
from pyspark.ml import Pipeline
-from pyspark.ml.regression import RandomForestRegressor
-from pyspark.ml.feature import VectorIndexer
from pyspark.ml.evaluation import RegressionEvaluator
+from pyspark.ml.feature import VectorIndexer
+from pyspark.ml.regression import RandomForestRegressor
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/rformula_example.py b/examples/src/main/python/ml/rformula_example.py
index 25bb6dac56e8..795e3202025b 100644
--- a/examples/src/main/python/ml/rformula_example.py
+++ b/examples/src/main/python/ml/rformula_example.py
@@ -17,6 +17,7 @@
# $example on$
from pyspark.ml.feature import RFormula
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/robust_scaler_example.py b/examples/src/main/python/ml/robust_scaler_example.py
index 9f7c6d6507c7..6972e6c1aeac 100644
--- a/examples/src/main/python/ml/robust_scaler_example.py
+++ b/examples/src/main/python/ml/robust_scaler_example.py
@@ -17,6 +17,7 @@
# $example on$
from pyspark.ml.feature import RobustScaler
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/sql_transformer.py b/examples/src/main/python/ml/sql_transformer.py
index c8ac5c46aa5e..2de00320611d 100644
--- a/examples/src/main/python/ml/sql_transformer.py
+++ b/examples/src/main/python/ml/sql_transformer.py
@@ -17,6 +17,7 @@
# $example on$
from pyspark.ml.feature import SQLTransformer
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/standard_scaler_example.py b/examples/src/main/python/ml/standard_scaler_example.py
index 9021c10075d8..0194d768f6f8 100644
--- a/examples/src/main/python/ml/standard_scaler_example.py
+++ b/examples/src/main/python/ml/standard_scaler_example.py
@@ -17,6 +17,7 @@
# $example on$
from pyspark.ml.feature import StandardScaler
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/stopwords_remover_example.py b/examples/src/main/python/ml/stopwords_remover_example.py
index 832a7c7d0ad8..e30ddfc18eba 100644
--- a/examples/src/main/python/ml/stopwords_remover_example.py
+++ b/examples/src/main/python/ml/stopwords_remover_example.py
@@ -17,6 +17,7 @@
# $example on$
from pyspark.ml.feature import StopWordsRemover
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/string_indexer_example.py b/examples/src/main/python/ml/string_indexer_example.py
index f2ac63eabd71..a0a4f303152d 100644
--- a/examples/src/main/python/ml/string_indexer_example.py
+++ b/examples/src/main/python/ml/string_indexer_example.py
@@ -17,6 +17,7 @@
# $example on$
from pyspark.ml.feature import StringIndexer
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/summarizer_example.py b/examples/src/main/python/ml/summarizer_example.py
index 498274645013..5414a9bd744c 100644
--- a/examples/src/main/python/ml/summarizer_example.py
+++ b/examples/src/main/python/ml/summarizer_example.py
@@ -20,11 +20,12 @@ An example for summarizer.
Run with:
bin/spark-submit examples/src/main/python/ml/summarizer_example.py
"""
-from pyspark.sql import SparkSession
+from pyspark.ml.linalg import Vectors
+
# $example on$
from pyspark.ml.stat import Summarizer
-from pyspark.sql import Row
-from pyspark.ml.linalg import Vectors
+from pyspark.sql import Row, SparkSession
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/ml/tf_idf_example.py b/examples/src/main/python/ml/tf_idf_example.py
index b4bb0dfa3183..43fd2e991d6d 100644
--- a/examples/src/main/python/ml/tf_idf_example.py
+++ b/examples/src/main/python/ml/tf_idf_example.py
@@ -16,7 +16,8 @@
#
# $example on$
-from pyspark.ml.feature import HashingTF, IDF, Tokenizer
+from pyspark.ml.feature import IDF, HashingTF, Tokenizer
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/tokenizer_example.py b/examples/src/main/python/ml/tokenizer_example.py
index c6b5fac22731..b04b2ee5ed8a 100644
--- a/examples/src/main/python/ml/tokenizer_example.py
+++ b/examples/src/main/python/ml/tokenizer_example.py
@@ -16,11 +16,12 @@
#
# $example on$
-from pyspark.ml.feature import Tokenizer, RegexTokenizer
-from pyspark.sql.functions import col, udf
-from pyspark.sql.types import IntegerType
+from pyspark.ml.feature import RegexTokenizer, Tokenizer
+
# $example off$
from pyspark.sql import SparkSession
+from pyspark.sql.functions import col, udf
+from pyspark.sql.types import IntegerType
if __name__ == "__main__":
spark = SparkSession\
diff --git a/examples/src/main/python/ml/train_validation_split.py b/examples/src/main/python/ml/train_validation_split.py
index 5e3dc7b3ec2f..21e110419cfa 100644
--- a/examples/src/main/python/ml/train_validation_split.py
+++ b/examples/src/main/python/ml/train_validation_split.py
@@ -26,6 +26,7 @@ Run with:
from pyspark.ml.evaluation import RegressionEvaluator
from pyspark.ml.regression import LinearRegression
from pyspark.ml.tuning import ParamGridBuilder, TrainValidationSplit
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/univariate_feature_selector_example.py b/examples/src/main/python/ml/univariate_feature_selector_example.py
index 6dc293e49643..3b5f2db7f9c0 100644
--- a/examples/src/main/python/ml/univariate_feature_selector_example.py
+++ b/examples/src/main/python/ml/univariate_feature_selector_example.py
@@ -20,10 +20,11 @@ An example for UnivariateFeatureSelector.
Run with:
bin/spark-submit examples/src/main/python/ml/univariate_feature_selector_example.py
"""
-from pyspark.sql import SparkSession
# $example on$
from pyspark.ml.feature import UnivariateFeatureSelector
from pyspark.ml.linalg import Vectors
+from pyspark.sql import SparkSession
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/ml/variance_threshold_selector_example.py b/examples/src/main/python/ml/variance_threshold_selector_example.py
index 0a996e0e2826..8bbcbffa2271 100644
--- a/examples/src/main/python/ml/variance_threshold_selector_example.py
+++ b/examples/src/main/python/ml/variance_threshold_selector_example.py
@@ -20,10 +20,11 @@ An example for VarianceThresholdSelector.
Run with:
bin/spark-submit examples/src/main/python/ml/variance_threshold_selector_example.py
"""
-from pyspark.sql import SparkSession
# $example on$
from pyspark.ml.feature import VarianceThresholdSelector
from pyspark.ml.linalg import Vectors
+from pyspark.sql import SparkSession
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/ml/vector_assembler_example.py b/examples/src/main/python/ml/vector_assembler_example.py
index 0ce31cf0eabc..8de9cad4114a 100644
--- a/examples/src/main/python/ml/vector_assembler_example.py
+++ b/examples/src/main/python/ml/vector_assembler_example.py
@@ -16,8 +16,9 @@
#
# $example on$
-from pyspark.ml.linalg import Vectors
from pyspark.ml.feature import VectorAssembler
+from pyspark.ml.linalg import Vectors
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/vector_indexer_example.py b/examples/src/main/python/ml/vector_indexer_example.py
index 51a4191606fb..bf99ec299d19 100644
--- a/examples/src/main/python/ml/vector_indexer_example.py
+++ b/examples/src/main/python/ml/vector_indexer_example.py
@@ -17,6 +17,7 @@
# $example on$
from pyspark.ml.feature import VectorIndexer
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/vector_size_hint_example.py b/examples/src/main/python/ml/vector_size_hint_example.py
index 355d85aee872..f376f0644192 100644
--- a/examples/src/main/python/ml/vector_size_hint_example.py
+++ b/examples/src/main/python/ml/vector_size_hint_example.py
@@ -16,8 +16,9 @@
#
# $example on$
+from pyspark.ml.feature import VectorAssembler, VectorSizeHint
from pyspark.ml.linalg import Vectors
-from pyspark.ml.feature import (VectorSizeHint, VectorAssembler)
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/ml/vector_slicer_example.py b/examples/src/main/python/ml/vector_slicer_example.py
index 86e089d152c5..368ba54bdb5d 100644
--- a/examples/src/main/python/ml/vector_slicer_example.py
+++ b/examples/src/main/python/ml/vector_slicer_example.py
@@ -18,9 +18,10 @@
# $example on$
from pyspark.ml.feature import VectorSlicer
from pyspark.ml.linalg import Vectors
-from pyspark.sql.types import Row
+
# $example off$
from pyspark.sql import SparkSession
+from pyspark.sql.types import Row
if __name__ == "__main__":
spark = SparkSession\
diff --git a/examples/src/main/python/ml/word2vec_example.py b/examples/src/main/python/ml/word2vec_example.py
index 0eabeda3dce4..a2fc5a5f02db 100644
--- a/examples/src/main/python/ml/word2vec_example.py
+++ b/examples/src/main/python/ml/word2vec_example.py
@@ -17,6 +17,7 @@
# $example on$
from pyspark.ml.feature import Word2Vec
+
# $example off$
from pyspark.sql import SparkSession
diff --git a/examples/src/main/python/mllib/binary_classification_metrics_example.py b/examples/src/main/python/mllib/binary_classification_metrics_example.py
index 741746e6e35a..941f8f64bc58 100644
--- a/examples/src/main/python/mllib/binary_classification_metrics_example.py
+++ b/examples/src/main/python/mllib/binary_classification_metrics_example.py
@@ -18,10 +18,12 @@
Binary Classification Metrics Example.
"""
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.classification import LogisticRegressionWithLBFGS
from pyspark.mllib.evaluation import BinaryClassificationMetrics
from pyspark.mllib.util import MLUtils
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/bisecting_k_means_example.py b/examples/src/main/python/mllib/bisecting_k_means_example.py
index d7b6ad9d424a..0cd500fbe95f 100644
--- a/examples/src/main/python/mllib/bisecting_k_means_example.py
+++ b/examples/src/main/python/mllib/bisecting_k_means_example.py
@@ -17,11 +17,13 @@
# $example on$
from numpy import array
-# $example off$
+# $example off$
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.clustering import BisectingKMeans
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/correlations.py b/examples/src/main/python/mllib/correlations.py
index 27d07b22a564..14c5449a8ae9 100755
--- a/examples/src/main/python/mllib/correlations.py
+++ b/examples/src/main/python/mllib/correlations.py
@@ -25,7 +25,6 @@ from pyspark.mllib.regression import LabeledPoint
from pyspark.mllib.stat import Statistics
from pyspark.mllib.util import MLUtils
-
if __name__ == "__main__":
if len(sys.argv) not in [1, 2]:
print("Usage: correlations (<file>)", file=sys.stderr)
diff --git a/examples/src/main/python/mllib/correlations_example.py b/examples/src/main/python/mllib/correlations_example.py
index bb71b968687c..52c108f4608f 100644
--- a/examples/src/main/python/mllib/correlations_example.py
+++ b/examples/src/main/python/mllib/correlations_example.py
@@ -16,10 +16,11 @@
#
import numpy as np
-
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.stat import Statistics
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/decision_tree_classification_example.py b/examples/src/main/python/mllib/decision_tree_classification_example.py
index 009e393226c0..b29dea1b96e8 100644
--- a/examples/src/main/python/mllib/decision_tree_classification_example.py
+++ b/examples/src/main/python/mllib/decision_tree_classification_example.py
@@ -19,9 +19,11 @@
Decision Tree Classification Example.
"""
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.tree import DecisionTree, DecisionTreeModel
from pyspark.mllib.util import MLUtils
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/decision_tree_regression_example.py b/examples/src/main/python/mllib/decision_tree_regression_example.py
index 71dfbf079017..8cf0388fe822 100644
--- a/examples/src/main/python/mllib/decision_tree_regression_example.py
+++ b/examples/src/main/python/mllib/decision_tree_regression_example.py
@@ -19,9 +19,11 @@
Decision Tree Regression Example.
"""
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.tree import DecisionTree, DecisionTreeModel
from pyspark.mllib.util import MLUtils
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/elementwise_product_example.py b/examples/src/main/python/mllib/elementwise_product_example.py
index 15e6a43f736c..48918a5431ae 100644
--- a/examples/src/main/python/mllib/elementwise_product_example.py
+++ b/examples/src/main/python/mllib/elementwise_product_example.py
@@ -16,9 +16,11 @@
#
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.feature import ElementwiseProduct
from pyspark.mllib.linalg import Vectors
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/fpgrowth_example.py b/examples/src/main/python/mllib/fpgrowth_example.py
index 715f5268206c..d1c9016f81d1 100644
--- a/examples/src/main/python/mllib/fpgrowth_example.py
+++ b/examples/src/main/python/mllib/fpgrowth_example.py
@@ -16,9 +16,9 @@
#
# $example on$
-from pyspark.mllib.fpm import FPGrowth
# $example off$
from pyspark import SparkContext
+from pyspark.mllib.fpm import FPGrowth
if __name__ == "__main__":
sc = SparkContext(appName="FPGrowth")
diff --git a/examples/src/main/python/mllib/gaussian_mixture_example.py b/examples/src/main/python/mllib/gaussian_mixture_example.py
index 3b19478f457e..5c0f65f60d3e 100644
--- a/examples/src/main/python/mllib/gaussian_mixture_example.py
+++ b/examples/src/main/python/mllib/gaussian_mixture_example.py
@@ -17,11 +17,13 @@
# $example on$
from numpy import array
-# $example off$
+# $example off$
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.clustering import GaussianMixture, GaussianMixtureModel
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/gaussian_mixture_model.py b/examples/src/main/python/mllib/gaussian_mixture_model.py
index 96ce6b6f6ab2..df910bb064f5 100644
--- a/examples/src/main/python/mllib/gaussian_mixture_model.py
+++ b/examples/src/main/python/mllib/gaussian_mixture_model.py
@@ -19,10 +19,10 @@
A Gaussian Mixture Model clustering program using MLlib.
"""
-import random
import argparse
-import numpy as np
+import random
+import numpy as np
from pyspark import SparkConf, SparkContext
from pyspark.mllib.clustering import GaussianMixture
diff --git a/examples/src/main/python/mllib/gradient_boosting_classification_example.py b/examples/src/main/python/mllib/gradient_boosting_classification_example.py
index eb12f206196f..1a7cb0f8a932 100644
--- a/examples/src/main/python/mllib/gradient_boosting_classification_example.py
+++ b/examples/src/main/python/mllib/gradient_boosting_classification_example.py
@@ -19,9 +19,11 @@
Gradient Boosted Trees Classification Example.
"""
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.tree import GradientBoostedTrees, GradientBoostedTreesModel
from pyspark.mllib.util import MLUtils
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/gradient_boosting_regression_example.py b/examples/src/main/python/mllib/gradient_boosting_regression_example.py
index eb59a992df53..62b82113db95 100644
--- a/examples/src/main/python/mllib/gradient_boosting_regression_example.py
+++ b/examples/src/main/python/mllib/gradient_boosting_regression_example.py
@@ -19,9 +19,11 @@
Gradient Boosted Trees Regression Example.
"""
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.tree import GradientBoostedTrees, GradientBoostedTreesModel
from pyspark.mllib.util import MLUtils
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/hypothesis_testing_example.py b/examples/src/main/python/mllib/hypothesis_testing_example.py
index 321be8b76f1b..6a92d3d95ebd 100644
--- a/examples/src/main/python/mllib/hypothesis_testing_example.py
+++ b/examples/src/main/python/mllib/hypothesis_testing_example.py
@@ -16,10 +16,12 @@
#
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.linalg import Matrices, Vectors
from pyspark.mllib.regression import LabeledPoint
from pyspark.mllib.stat import Statistics
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/hypothesis_testing_kolmogorov_smirnov_test_example.py b/examples/src/main/python/mllib/hypothesis_testing_kolmogorov_smirnov_test_example.py
index 12a186900e35..0f686086d66c 100644
--- a/examples/src/main/python/mllib/hypothesis_testing_kolmogorov_smirnov_test_example.py
+++ b/examples/src/main/python/mllib/hypothesis_testing_kolmogorov_smirnov_test_example.py
@@ -16,8 +16,10 @@
#
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.stat import Statistics
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/isotonic_regression_example.py b/examples/src/main/python/mllib/isotonic_regression_example.py
index a5a0cfeae9d7..10035b5b7619 100644
--- a/examples/src/main/python/mllib/isotonic_regression_example.py
+++ b/examples/src/main/python/mllib/isotonic_regression_example.py
@@ -18,11 +18,13 @@
"""
Isotonic Regression Example.
"""
-from pyspark import SparkContext
# $example on$
import math
+
+from pyspark import SparkContext
from pyspark.mllib.regression import IsotonicRegression, IsotonicRegressionModel
from pyspark.mllib.util import MLUtils
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/k_means_example.py b/examples/src/main/python/mllib/k_means_example.py
index ead1e56de55c..c2901c154f68 100644
--- a/examples/src/main/python/mllib/k_means_example.py
+++ b/examples/src/main/python/mllib/k_means_example.py
@@ -16,13 +16,16 @@
#
# $example on$
-from numpy import array
from math import sqrt
-# $example off$
+from numpy import array
+
+# $example off$
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.clustering import KMeans, KMeansModel
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/kernel_density_estimation_example.py b/examples/src/main/python/mllib/kernel_density_estimation_example.py
index 22d191716057..c3dc2d027eea 100644
--- a/examples/src/main/python/mllib/kernel_density_estimation_example.py
+++ b/examples/src/main/python/mllib/kernel_density_estimation_example.py
@@ -16,8 +16,10 @@
#
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.stat import KernelDensity
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/latent_dirichlet_allocation_example.py b/examples/src/main/python/mllib/latent_dirichlet_allocation_example.py
index f82a28aadc5a..81b039e56895 100644
--- a/examples/src/main/python/mllib/latent_dirichlet_allocation_example.py
+++ b/examples/src/main/python/mllib/latent_dirichlet_allocation_example.py
@@ -16,9 +16,11 @@
#
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.clustering import LDA, LDAModel
from pyspark.mllib.linalg import Vectors
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/linear_regression_with_sgd_example.py b/examples/src/main/python/mllib/linear_regression_with_sgd_example.py
index cb6739633231..1813a838d635 100644
--- a/examples/src/main/python/mllib/linear_regression_with_sgd_example.py
+++ b/examples/src/main/python/mllib/linear_regression_with_sgd_example.py
@@ -19,8 +19,10 @@
Linear Regression With SGD Example.
"""
from pyspark import SparkContext
+
# $example on$
-from pyspark.mllib.regression import LabeledPoint, LinearRegressionWithSGD, LinearRegressionModel
+from pyspark.mllib.regression import LabeledPoint, LinearRegressionModel, LinearRegressionWithSGD
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/logistic_regression.py b/examples/src/main/python/mllib/logistic_regression.py
index 7b90615a5342..8ae5b0ab8b7d 100755
--- a/examples/src/main/python/mllib/logistic_regression.py
+++ b/examples/src/main/python/mllib/logistic_regression.py
@@ -23,8 +23,8 @@ This example requires NumPy (http://www.numpy.org/).
import sys
from pyspark import SparkContext
-from pyspark.mllib.regression import LabeledPoint
from pyspark.mllib.classification import LogisticRegressionWithSGD
+from pyspark.mllib.regression import LabeledPoint
def parsePoint(line):
diff --git a/examples/src/main/python/mllib/logistic_regression_with_lbfgs_example.py b/examples/src/main/python/mllib/logistic_regression_with_lbfgs_example.py
index ac5ab1d1b5d9..014fbfca1a88 100644
--- a/examples/src/main/python/mllib/logistic_regression_with_lbfgs_example.py
+++ b/examples/src/main/python/mllib/logistic_regression_with_lbfgs_example.py
@@ -19,9 +19,11 @@
Logistic Regression With LBFGS Example.
"""
from pyspark import SparkContext
+
# $example on$
-from pyspark.mllib.classification import LogisticRegressionWithLBFGS, LogisticRegressionModel
+from pyspark.mllib.classification import LogisticRegressionModel, LogisticRegressionWithLBFGS
from pyspark.mllib.regression import LabeledPoint
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/multi_class_metrics_example.py b/examples/src/main/python/mllib/multi_class_metrics_example.py
index 03a564e75be9..e429e8795b36 100644
--- a/examples/src/main/python/mllib/multi_class_metrics_example.py
+++ b/examples/src/main/python/mllib/multi_class_metrics_example.py
@@ -16,12 +16,11 @@
#
# $example on$
-from pyspark.mllib.classification import LogisticRegressionWithLBFGS
-from pyspark.mllib.util import MLUtils
-from pyspark.mllib.evaluation import MulticlassMetrics
# $example off$
-
from pyspark import SparkContext
+from pyspark.mllib.classification import LogisticRegressionWithLBFGS
+from pyspark.mllib.evaluation import MulticlassMetrics
+from pyspark.mllib.util import MLUtils
if __name__ == "__main__":
sc = SparkContext(appName="MultiClassMetricsExample")
diff --git a/examples/src/main/python/mllib/multi_label_metrics_example.py b/examples/src/main/python/mllib/multi_label_metrics_example.py
index 960ade659737..be689b9565d2 100644
--- a/examples/src/main/python/mllib/multi_label_metrics_example.py
+++ b/examples/src/main/python/mllib/multi_label_metrics_example.py
@@ -16,9 +16,9 @@
#
# $example on$
-from pyspark.mllib.evaluation import MultilabelMetrics
# $example off$
from pyspark import SparkContext
+from pyspark.mllib.evaluation import MultilabelMetrics
if __name__ == "__main__":
sc = SparkContext(appName="MultiLabelMetricsExample")
diff --git a/examples/src/main/python/mllib/naive_bayes_example.py b/examples/src/main/python/mllib/naive_bayes_example.py
index 74d18233d533..f76ab891a79a 100644
--- a/examples/src/main/python/mllib/naive_bayes_example.py
+++ b/examples/src/main/python/mllib/naive_bayes_example.py
@@ -25,11 +25,11 @@ Usage:
import shutil
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.classification import NaiveBayes, NaiveBayesModel
from pyspark.mllib.util import MLUtils
-
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/normalizer_example.py b/examples/src/main/python/mllib/normalizer_example.py
index d46110d9a030..df0d9faf9aa5 100644
--- a/examples/src/main/python/mllib/normalizer_example.py
+++ b/examples/src/main/python/mllib/normalizer_example.py
@@ -16,9 +16,11 @@
#
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.feature import Normalizer
from pyspark.mllib.util import MLUtils
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/pca_rowmatrix_example.py b/examples/src/main/python/mllib/pca_rowmatrix_example.py
index 49b9b1bbe08e..99650cc631c7 100644
--- a/examples/src/main/python/mllib/pca_rowmatrix_example.py
+++ b/examples/src/main/python/mllib/pca_rowmatrix_example.py
@@ -16,9 +16,11 @@
#
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.linalg import Vectors
from pyspark.mllib.linalg.distributed import RowMatrix
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/power_iteration_clustering_example.py b/examples/src/main/python/mllib/power_iteration_clustering_example.py
index 60eedef5fab3..3901cce1d2d6 100644
--- a/examples/src/main/python/mllib/power_iteration_clustering_example.py
+++ b/examples/src/main/python/mllib/power_iteration_clustering_example.py
@@ -16,8 +16,10 @@
#
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.clustering import PowerIterationClustering, PowerIterationClusteringModel
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/random_forest_classification_example.py b/examples/src/main/python/mllib/random_forest_classification_example.py
index a929c10d5a57..3880c77f65e2 100644
--- a/examples/src/main/python/mllib/random_forest_classification_example.py
+++ b/examples/src/main/python/mllib/random_forest_classification_example.py
@@ -19,9 +19,11 @@
Random Forest Classification Example.
"""
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.tree import RandomForest, RandomForestModel
from pyspark.mllib.util import MLUtils
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/random_forest_regression_example.py b/examples/src/main/python/mllib/random_forest_regression_example.py
index 4e0593776821..a45bd63c1294 100644
--- a/examples/src/main/python/mllib/random_forest_regression_example.py
+++ b/examples/src/main/python/mllib/random_forest_regression_example.py
@@ -19,9 +19,11 @@
Random Forest Regression Example.
"""
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.tree import RandomForest, RandomForestModel
from pyspark.mllib.util import MLUtils
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/random_rdd_generation.py b/examples/src/main/python/mllib/random_rdd_generation.py
index 49afcfe9391a..2141d7a2f400 100755
--- a/examples/src/main/python/mllib/random_rdd_generation.py
+++ b/examples/src/main/python/mllib/random_rdd_generation.py
@@ -23,7 +23,6 @@ import sys
from pyspark import SparkContext
from pyspark.mllib.random import RandomRDDs
-
if __name__ == "__main__":
if len(sys.argv) not in [1, 2]:
print("Usage: random_rdd_generation", file=sys.stderr)
diff --git a/examples/src/main/python/mllib/ranking_metrics_example.py b/examples/src/main/python/mllib/ranking_metrics_example.py
index 0913bb34cf9d..2e5738d49d00 100644
--- a/examples/src/main/python/mllib/ranking_metrics_example.py
+++ b/examples/src/main/python/mllib/ranking_metrics_example.py
@@ -16,10 +16,10 @@
#
# $example on$
-from pyspark.mllib.recommendation import ALS, Rating
-from pyspark.mllib.evaluation import RegressionMetrics
# $example off$
from pyspark import SparkContext
+from pyspark.mllib.evaluation import RegressionMetrics
+from pyspark.mllib.recommendation import ALS, Rating
if __name__ == "__main__":
sc = SparkContext(appName="Ranking Metrics Example")
diff --git a/examples/src/main/python/mllib/recommendation_example.py b/examples/src/main/python/mllib/recommendation_example.py
index 719f3f904b24..b8204caa7b9a 100644
--- a/examples/src/main/python/mllib/recommendation_example.py
+++ b/examples/src/main/python/mllib/recommendation_example.py
@@ -22,6 +22,7 @@ from pyspark import SparkContext
# $example on$
from pyspark.mllib.recommendation import ALS, MatrixFactorizationModel, Rating
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/regression_metrics_example.py b/examples/src/main/python/mllib/regression_metrics_example.py
index a3a83aafd7a1..4ac0b4ed258d 100644
--- a/examples/src/main/python/mllib/regression_metrics_example.py
+++ b/examples/src/main/python/mllib/regression_metrics_example.py
@@ -15,12 +15,11 @@
# limitations under the License.
#
# $example on$
-from pyspark.mllib.regression import LabeledPoint, LinearRegressionWithSGD
-from pyspark.mllib.evaluation import RegressionMetrics
-from pyspark.mllib.linalg import DenseVector
# $example off$
-
from pyspark import SparkContext
+from pyspark.mllib.evaluation import RegressionMetrics
+from pyspark.mllib.linalg import DenseVector
+from pyspark.mllib.regression import LabeledPoint, LinearRegressionWithSGD
if __name__ == "__main__":
sc = SparkContext(appName="Regression Metrics Example")
diff --git a/examples/src/main/python/mllib/sampled_rdds.py b/examples/src/main/python/mllib/sampled_rdds.py
index 9095c2b2d70d..abf804fbaec7 100755
--- a/examples/src/main/python/mllib/sampled_rdds.py
+++ b/examples/src/main/python/mllib/sampled_rdds.py
@@ -23,7 +23,6 @@ import sys
from pyspark import SparkContext
from pyspark.mllib.util import MLUtils
-
if __name__ == "__main__":
if len(sys.argv) not in [1, 2]:
print("Usage: sampled_rdds <libsvm data file>", file=sys.stderr)
diff --git a/examples/src/main/python/mllib/standard_scaler_example.py b/examples/src/main/python/mllib/standard_scaler_example.py
index c8fd64dfbbf4..9de3b6796841 100644
--- a/examples/src/main/python/mllib/standard_scaler_example.py
+++ b/examples/src/main/python/mllib/standard_scaler_example.py
@@ -16,10 +16,12 @@
#
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.feature import StandardScaler
from pyspark.mllib.linalg import Vectors
from pyspark.mllib.util import MLUtils
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/streaming_k_means_example.py b/examples/src/main/python/mllib/streaming_k_means_example.py
index 4904a9ebcf54..0b1df65277f0 100644
--- a/examples/src/main/python/mllib/streaming_k_means_example.py
+++ b/examples/src/main/python/mllib/streaming_k_means_example.py
@@ -16,11 +16,13 @@
#
from pyspark import SparkContext
-from pyspark.streaming import StreamingContext
+from pyspark.mllib.clustering import StreamingKMeans
+
# $example on$
from pyspark.mllib.linalg import Vectors
from pyspark.mllib.regression import LabeledPoint
-from pyspark.mllib.clustering import StreamingKMeans
+from pyspark.streaming import StreamingContext
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/streaming_linear_regression_example.py b/examples/src/main/python/mllib/streaming_linear_regression_example.py
index 1d52e00fbfb5..fe1b79414acb 100644
--- a/examples/src/main/python/mllib/streaming_linear_regression_example.py
+++ b/examples/src/main/python/mllib/streaming_linear_regression_example.py
@@ -20,14 +20,15 @@ Streaming Linear Regression Example.
"""
# $example on$
import sys
-# $example off$
+# $example off$
from pyspark import SparkContext
-from pyspark.streaming import StreamingContext
+
# $example on$
from pyspark.mllib.linalg import Vectors
-from pyspark.mllib.regression import LabeledPoint
-from pyspark.mllib.regression import StreamingLinearRegressionWithSGD
+from pyspark.mllib.regression import LabeledPoint, StreamingLinearRegressionWithSGD
+from pyspark.streaming import StreamingContext
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/summary_statistics_example.py b/examples/src/main/python/mllib/summary_statistics_example.py
index d86e84114550..fad7cd4c5a6f 100644
--- a/examples/src/main/python/mllib/summary_statistics_example.py
+++ b/examples/src/main/python/mllib/summary_statistics_example.py
@@ -15,11 +15,11 @@
# limitations under the License.
#
-from pyspark import SparkContext
# $example on$
import numpy as np
-
+from pyspark import SparkContext
from pyspark.mllib.stat import Statistics
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/svd_example.py b/examples/src/main/python/mllib/svd_example.py
index 5b220fdb3fd6..5469bb6710ee 100644
--- a/examples/src/main/python/mllib/svd_example.py
+++ b/examples/src/main/python/mllib/svd_example.py
@@ -16,9 +16,11 @@
#
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.linalg import Vectors
from pyspark.mllib.linalg.distributed import RowMatrix
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/svm_with_sgd_example.py b/examples/src/main/python/mllib/svm_with_sgd_example.py
index 24b8f431e059..456350bae7e9 100644
--- a/examples/src/main/python/mllib/svm_with_sgd_example.py
+++ b/examples/src/main/python/mllib/svm_with_sgd_example.py
@@ -16,9 +16,11 @@
#
from pyspark import SparkContext
+
# $example on$
-from pyspark.mllib.classification import SVMWithSGD, SVMModel
+from pyspark.mllib.classification import SVMModel, SVMWithSGD
from pyspark.mllib.regression import LabeledPoint
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/tf_idf_example.py b/examples/src/main/python/mllib/tf_idf_example.py
index 4449066f5b0a..ca46d62dacc9 100644
--- a/examples/src/main/python/mllib/tf_idf_example.py
+++ b/examples/src/main/python/mllib/tf_idf_example.py
@@ -16,8 +16,10 @@
#
from pyspark import SparkContext
+
# $example on$
-from pyspark.mllib.feature import HashingTF, IDF
+from pyspark.mllib.feature import IDF, HashingTF
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/mllib/word2vec_example.py b/examples/src/main/python/mllib/word2vec_example.py
index d37a6e7137b8..175eecb92a10 100644
--- a/examples/src/main/python/mllib/word2vec_example.py
+++ b/examples/src/main/python/mllib/word2vec_example.py
@@ -16,8 +16,10 @@
#
from pyspark import SparkContext
+
# $example on$
from pyspark.mllib.feature import Word2Vec
+
# $example off$
if __name__ == "__main__":
diff --git a/examples/src/main/python/pi.py b/examples/src/main/python/pi.py
index e61740ad5883..5425e5d9142a 100755
--- a/examples/src/main/python/pi.py
+++ b/examples/src/main/python/pi.py
@@ -16,12 +16,11 @@
#
import sys
-from random import random
from operator import add
+from random import random
from pyspark.sql import SparkSession
-
if __name__ == "__main__":
"""
Usage: pi [partitions]
diff --git a/examples/src/main/python/sort.py b/examples/src/main/python/sort.py
index 9ef2d5dbaff4..139a9d5dc28a 100755
--- a/examples/src/main/python/sort.py
+++ b/examples/src/main/python/sort.py
@@ -21,7 +21,6 @@ from typing import Tuple
from pyspark import RDD
from pyspark.sql import SparkSession
-
if __name__ == "__main__":
if len(sys.argv) != 2:
print("Usage: sort <file>", file=sys.stderr)
diff --git a/examples/src/main/python/sql/arrow.py b/examples/src/main/python/sql/arrow.py
index af51fb553205..0780e6c2bd87 100644
--- a/examples/src/main/python/sql/arrow.py
+++ b/examples/src/main/python/sql/arrow.py
@@ -24,9 +24,9 @@ Run with:
# NOTE that this file is imported in tutorials in PySpark documentation.
# The code blocks are referred to via `$example on/off$` markers, included in the
# docs with the `literalinclude` directive's `:start-after:`/`:end-before:` options.
-import pandas as pd
from typing import Iterator
+import pandas as pd
from pyspark.sql import SparkSession
from pyspark.sql.pandas.utils import require_minimum_pandas_version, require_minimum_pyarrow_version
@@ -36,8 +36,8 @@ require_minimum_pyarrow_version()
def dataframe_to_from_arrow_table_example(spark: SparkSession) -> None:
# $example on:dataframe_to_from_arrow_table$
- import pyarrow as pa
import numpy as np
+ import pyarrow as pa
# Create a PyArrow Table
table = pa.table([pa.array(np.random.rand(100)) for i in range(3)], names=["a", "b", "c"])
@@ -79,7 +79,6 @@ def dataframe_with_arrow_example(spark: SparkSession) -> None:
def ser_to_frame_pandas_udf_example(spark: SparkSession) -> None:
# $example on:ser_to_frame_pandas_udf$
import pandas as pd
-
from pyspark.sql.functions import pandas_udf
@pandas_udf("col1 string, col2 long") # type: ignore[call-overload]
@@ -109,7 +108,6 @@ def ser_to_frame_pandas_udf_example(spark: SparkSession) -> None:
def ser_to_ser_pandas_udf_example(spark: SparkSession) -> None:
# $example on:ser_to_ser_pandas_udf$
import pandas as pd
-
from pyspark.sql.functions import col, pandas_udf
from pyspark.sql.types import LongType
@@ -147,7 +145,6 @@ def iter_ser_to_iter_ser_pandas_udf_example(spark: SparkSession) -> None:
from typing import Iterator
import pandas as pd
-
from pyspark.sql.functions import pandas_udf
pdf = pd.DataFrame([1, 2, 3], columns=["x"])
@@ -175,7 +172,6 @@ def iter_sers_to_iter_ser_pandas_udf_example(spark: SparkSession) -> None:
from typing import Iterator, Tuple
import pandas as pd
-
from pyspark.sql.functions import pandas_udf
pdf = pd.DataFrame([1, 2, 3], columns=["x"])
@@ -202,9 +198,8 @@ def iter_sers_to_iter_ser_pandas_udf_example(spark: SparkSession) -> None:
def ser_to_scalar_pandas_udf_example(spark: SparkSession) -> None:
# $example on:ser_to_scalar_pandas_udf$
import pandas as pd
-
- from pyspark.sql.functions import pandas_udf
from pyspark.sql import Window
+ from pyspark.sql.functions import pandas_udf
df = spark.createDataFrame(
[(1, 1.0), (1, 2.0), (2, 3.0), (2, 5.0), (2, 10.0)],
diff --git a/examples/src/main/python/sql/basic.py b/examples/src/main/python/sql/basic.py
index 4f7ec7ba267d..8514c0b77a5c 100644
--- a/examples/src/main/python/sql/basic.py
+++ b/examples/src/main/python/sql/basic.py
@@ -21,16 +21,15 @@ Run with:
./bin/spark-submit examples/src/main/python/sql/basic.py
"""
# $example on:init_session$
-from pyspark.sql import SparkSession
# $example off:init_session$
-
# $example on:schema_inferring$
-from pyspark.sql import Row
-# $example off:schema_inferring$
+from pyspark.sql import Row, SparkSession
+# $example off:schema_inferring$
# $example on:programmatic_schema$
# Import data types
-from pyspark.sql.types import StringType, StructType, StructField
+from pyspark.sql.types import StringType, StructField, StructType
+
# $example off:programmatic_schema$
diff --git a/examples/src/main/python/sql/datasource.py b/examples/src/main/python/sql/datasource.py
index 0a76376e2c81..0820f09b2853 100644
--- a/examples/src/main/python/sql/datasource.py
+++ b/examples/src/main/python/sql/datasource.py
@@ -20,9 +20,9 @@ A simple example demonstrating Spark SQL data sources.
Run with:
./bin/spark-submit examples/src/main/python/sql/datasource.py
"""
-from pyspark.sql import SparkSession
# $example on:schema_merging$
-from pyspark.sql import Row
+from pyspark.sql import Row, SparkSession
+
# $example off:schema_merging$
diff --git a/examples/src/main/python/sql/hive.py b/examples/src/main/python/sql/hive.py
index fa1b975e2bfd..58c5d44e4d53 100644
--- a/examples/src/main/python/sql/hive.py
+++ b/examples/src/main/python/sql/hive.py
@@ -23,8 +23,8 @@ Run with:
# $example on:spark_hive$
from os.path import abspath
-from pyspark.sql import SparkSession
-from pyspark.sql import Row
+from pyspark.sql import Row, SparkSession
+
# $example off:spark_hive$
diff --git a/examples/src/main/python/sql/jdbc.py b/examples/src/main/python/sql/jdbc.py
index 7e67ab351f63..0956adb072f8 100644
--- a/examples/src/main/python/sql/jdbc.py
+++ b/examples/src/main/python/sql/jdbc.py
@@ -21,8 +21,8 @@ Run with:
./bin/spark-submit examples/src/main/python/sql/jdbc.py [jdbc_url]
"""
import sys
-from pyspark.sql import SparkSession
+from pyspark.sql import SparkSession
if __name__ == "__main__":
if len(sys.argv) < 2:
diff --git a/examples/src/main/python/sql/streaming/structured_kafka_wordcount.py b/examples/src/main/python/sql/streaming/structured_kafka_wordcount.py
index 40a955a46c9b..d241f556022a 100644
--- a/examples/src/main/python/sql/streaming/structured_kafka_wordcount.py
+++ b/examples/src/main/python/sql/streaming/structured_kafka_wordcount.py
@@ -39,8 +39,7 @@
import sys
from pyspark.sql import SparkSession
-from pyspark.sql.functions import explode
-from pyspark.sql.functions import split
+from pyspark.sql.functions import explode, split
if __name__ == "__main__":
if len(sys.argv) != 4:
diff --git a/examples/src/main/python/sql/streaming/structured_network_wordcount.py b/examples/src/main/python/sql/streaming/structured_network_wordcount.py
index c8f43c9dcf2e..6242b541b72e 100644
--- a/examples/src/main/python/sql/streaming/structured_network_wordcount.py
+++ b/examples/src/main/python/sql/streaming/structured_network_wordcount.py
@@ -30,8 +30,7 @@ r"""
import sys
from pyspark.sql import SparkSession
-from pyspark.sql.functions import explode
-from pyspark.sql.functions import split
+from pyspark.sql.functions import explode, split
if __name__ == "__main__":
if len(sys.argv) != 3:
diff --git a/examples/src/main/python/sql/streaming/structured_network_wordcount_session_window.py b/examples/src/main/python/sql/streaming/structured_network_wordcount_session_window.py
index 722d409792bf..f27ad44db55a 100644
--- a/examples/src/main/python/sql/streaming/structured_network_wordcount_session_window.py
+++ b/examples/src/main/python/sql/streaming/structured_network_wordcount_session_window.py
@@ -31,21 +31,19 @@ r"""
localhost 9999`
"""
import sys
-from typing import Iterator, Any
+from typing import Any, Iterator
import pandas as pd
-
from pyspark.sql import SparkSession
-from pyspark.sql.functions import explode
-from pyspark.sql.functions import split
+from pyspark.sql.functions import explode, split
+from pyspark.sql.streaming.state import GroupState, GroupStateTimeout
from pyspark.sql.types import (
LongType,
StringType,
- TimestampType,
- StructType,
StructField,
+ StructType,
+ TimestampType,
)
-from pyspark.sql.streaming.state import GroupStateTimeout, GroupState
if __name__ == "__main__":
if len(sys.argv) != 3:
diff --git a/examples/src/main/python/sql/streaming/structured_network_wordcount_windowed.py b/examples/src/main/python/sql/streaming/structured_network_wordcount_windowed.py
index cc39d8afa6be..28fba29ad04e 100644
--- a/examples/src/main/python/sql/streaming/structured_network_wordcount_windowed.py
+++ b/examples/src/main/python/sql/streaming/structured_network_wordcount_windowed.py
@@ -42,9 +42,7 @@ r"""
import sys
from pyspark.sql import SparkSession
-from pyspark.sql.functions import explode
-from pyspark.sql.functions import split
-from pyspark.sql.functions import window
+from pyspark.sql.functions import explode, split, window
if __name__ == "__main__":
if len(sys.argv) != 5 and len(sys.argv) != 4:
diff --git a/examples/src/main/python/sql/streaming/structured_sessionization.py b/examples/src/main/python/sql/streaming/structured_sessionization.py
index 78cb406650e6..97e441e77c69 100644
--- a/examples/src/main/python/sql/streaming/structured_sessionization.py
+++ b/examples/src/main/python/sql/streaming/structured_sessionization.py
@@ -33,9 +33,7 @@ r"""
import sys
from pyspark.sql import SparkSession
-from pyspark.sql.functions import explode
-from pyspark.sql.functions import split
-from pyspark.sql.functions import count, session_window
+from pyspark.sql.functions import count, explode, session_window, split
if __name__ == "__main__":
if len(sys.argv) != 3 and len(sys.argv) != 2:
diff --git a/examples/src/main/python/sql/udtf.py b/examples/src/main/python/sql/udtf.py
index ecb87c4e72ba..8a36c8d9c47c 100644
--- a/examples/src/main/python/sql/udtf.py
+++ b/examples/src/main/python/sql/udtf.py
@@ -136,6 +136,7 @@ def python_udtf_arrow_example(spark: SparkSession) -> None:
def python_udtf_date_expander_example(spark: SparkSession) -> None:
# $example on:python_udtf_date_expander$
from datetime import datetime, timedelta
+
from pyspark.sql.functions import lit, udtf
@udtf(returnType="date: string")
diff --git a/examples/src/main/python/status_api_demo.py b/examples/src/main/python/status_api_demo.py
index 3bf96ca4466f..15d22ddc8d9c 100644
--- a/examples/src/main/python/status_api_demo.py
+++ b/examples/src/main/python/status_api_demo.py
@@ -15,9 +15,9 @@
# limitations under the License.
#
-import time
-import threading
import queue as Queue
+import threading
+import time
from typing import Any, Callable, List, Tuple
from pyspark import SparkConf, SparkContext
diff --git a/examples/src/main/python/streaming/network_wordjoinsentiments.py b/examples/src/main/python/streaming/network_wordjoinsentiments.py
index fae40a77acaf..a21bf9285341 100644
--- a/examples/src/main/python/streaming/network_wordjoinsentiments.py
+++ b/examples/src/main/python/streaming/network_wordjoinsentiments.py
@@ -33,8 +33,7 @@ r"""
import sys
from typing import Tuple
-from pyspark import SparkContext
-from pyspark import RDD
+from pyspark import RDD, SparkContext
from pyspark.streaming import DStream, StreamingContext
diff --git a/examples/src/main/python/streaming/recoverable_network_wordcount.py b/examples/src/main/python/streaming/recoverable_network_wordcount.py
index 147d3c646799..e9d8513c2eb4 100644
--- a/examples/src/main/python/streaming/recoverable_network_wordcount.py
+++ b/examples/src/main/python/streaming/recoverable_network_wordcount.py
@@ -40,7 +40,7 @@ import os
import sys
from typing import List, Tuple
-from pyspark import SparkContext, Accumulator, Broadcast, RDD
+from pyspark import RDD, Accumulator, Broadcast, SparkContext
from pyspark.streaming import StreamingContext
diff --git a/examples/src/main/python/streaming/sql_network_wordcount.py b/examples/src/main/python/streaming/sql_network_wordcount.py
index bba398c0d610..a1b153063736 100644
--- a/examples/src/main/python/streaming/sql_network_wordcount.py
+++ b/examples/src/main/python/streaming/sql_network_wordcount.py
@@ -27,12 +27,12 @@ r"""
and then run the example
`$ bin/spark-submit examples/src/main/python/streaming/sql_network_wordcount.py localhost 9999`
"""
-import sys
import datetime
+import sys
-from pyspark import SparkConf, SparkContext, RDD
-from pyspark.streaming import StreamingContext
+from pyspark import RDD, SparkConf, SparkContext
from pyspark.sql import Row, SparkSession
+from pyspark.streaming import StreamingContext
def getSparkSessionInstance(sparkConf: SparkConf) -> SparkSession:
diff --git a/examples/src/main/python/wordcount.py b/examples/src/main/python/wordcount.py
index 037c1e8aa379..33ba6982368e 100755
--- a/examples/src/main/python/wordcount.py
+++ b/examples/src/main/python/wordcount.py
@@ -20,7 +20,6 @@ from operator import add
from pyspark.sql import SparkSession
-
if __name__ == "__main__":
if len(sys.argv) != 2:
print("Usage: wordcount <file>", file=sys.stderr)
diff --git a/pyproject.toml b/pyproject.toml
index 02d723547a05..9fcd00e69bf9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -39,6 +39,7 @@ line-length = 100
[tool.ruff.lint]
extend-select = [
+ "I", # isort
"G010", # logging-warn
# ambiguous unicode character
"RUF001", # string
diff --git a/python/benchmarks/bench_eval_type.py b/python/benchmarks/bench_eval_type.py
index 3228d915f44e..85a1bd03b0d3 100644
--- a/python/benchmarks/bench_eval_type.py
+++ b/python/benchmarks/bench_eval_type.py
@@ -24,8 +24,8 @@ by constructing the complete binary protocol that ``worker.py``'s
"""
import io
-import os
import json
+import os
import socket
import struct
import sys
@@ -35,9 +35,8 @@ from typing import Any, Callable, Iterator
import numpy as np
import pyarrow as pa
-
from pyspark.cloudpickle import dumps as cloudpickle_dumps
-from pyspark.serializers import CPickleSerializer, write_int, write_long, SpecialLengths
+from pyspark.serializers import CPickleSerializer, SpecialLengths, write_int, write_long
from pyspark.sql.types import (
BinaryType,
BooleanType,
@@ -52,7 +51,6 @@ from pyspark.sql.types import (
from pyspark.util import PythonEvalType
from pyspark.worker import main as worker_main
-
# ---------------------------------------------------------------------------
# Mock helpers: protocol writer, data factory, UDF factory
# ---------------------------------------------------------------------------
diff --git a/python/benchmarks/bench_local_data_to_arrow.py b/python/benchmarks/bench_local_data_to_arrow.py
index 55f3100311ce..0c41498c8ec1 100644
--- a/python/benchmarks/bench_local_data_to_arrow.py
+++ b/python/benchmarks/bench_local_data_to_arrow.py
@@ -30,6 +30,7 @@ for a scalar column, a one-level ``array`` column, and a two-level
def _build(convert, is_string, leaf, other_val, other_frac, n_rows):
import random
+
from pyspark.sql.types import ArrayType, BinaryType, StringType, StructField, StructType
leaf_type = StringType() if is_string else BinaryType()
@@ -78,6 +79,7 @@ class LocalDataToArrowStringBenchmark:
def setup(self, n_rows, leaf, other, other_frac):
import datetime
import decimal
+
from pyspark.sql.conversion import LocalDataToArrowConversion
self.convert = LocalDataToArrowConversion.convert
diff --git a/python/benchmarks/bench_pipelined_udf.py b/python/benchmarks/bench_pipelined_udf.py
index 31eb9d0b16e3..17b5be698523 100644
--- a/python/benchmarks/bench_pipelined_udf.py
+++ b/python/benchmarks/bench_pipelined_udf.py
@@ -24,7 +24,6 @@ SparkSession to measure the JVM-Python socket I/O pipeline overlap.
"""
import pandas as pd
-
from pyspark import SparkConf
from pyspark.sql import SparkSession
from pyspark.sql.functions import col, pandas_udf
diff --git a/python/conf_viztracer/daemon_viztracer.py b/python/conf_viztracer/daemon_viztracer.py
index 216f6368080d..ea6d736f68aa 100644
--- a/python/conf_viztracer/daemon_viztracer.py
+++ b/python/conf_viztracer/daemon_viztracer.py
@@ -18,11 +18,10 @@
import os
import sys
+import pyspark.worker
import viztracer
from viztracer.main import main
-import pyspark.worker
-
def viztracer_wrapper(func):
def wrapper(*args, **kwargs):
diff --git a/python/conf_viztracer/worker_viztracer.py b/python/conf_viztracer/worker_viztracer.py
index 362532c39dc4..7ffd832e53d9 100644
--- a/python/conf_viztracer/worker_viztracer.py
+++ b/python/conf_viztracer/worker_viztracer.py
@@ -20,7 +20,6 @@ import sys
from viztracer.main import main
-
if __name__ == "__main__":
if os.getenv("SPARK_VIZTRACER_OUTPUT_DIR") is not None:
diff --git a/python/conf_vscode/sitecustomize.py b/python/conf_vscode/sitecustomize.py
index 486d580efdc1..113252c4c5c0 100644
--- a/python/conf_vscode/sitecustomize.py
+++ b/python/conf_vscode/sitecustomize.py
@@ -24,9 +24,10 @@ try:
):
def install_debugpy():
- import debugpy
import fcntl
+ import debugpy
+
lock_file = os.getenv("DEBUGPY_ADAPTER_ENDPOINTS") + ".lock"
try:
fd = os.open(lock_file, os.O_CREAT | os.O_RDWR, 0o600)
diff --git a/python/packaging/classic/setup.py b/python/packaging/classic/setup.py
index c711edcd59b0..add11863f000 100755
--- a/python/packaging/classic/setup.py
+++ b/python/packaging/classic/setup.py
@@ -16,16 +16,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-import importlib.util
+import ctypes
import glob
+import importlib.util
import os
import sys
-import ctypes
+from dataclasses import dataclass
+from pathlib import Path
+from shutil import copyfile, copytree, rmtree
+
from setuptools import setup
from setuptools.command.install import install
-from shutil import copyfile, copytree, rmtree
-from pathlib import Path
-from dataclasses import dataclass
if (
# When we package, the parent directory 'classic' dir
diff --git a/python/packaging/client/setup.py b/python/packaging/client/setup.py
index d1754ff64d96..af9e9475018c 100755
--- a/python/packaging/client/setup.py
+++ b/python/packaging/client/setup.py
@@ -22,12 +22,13 @@
# cd python/packaging/classic
# python setup.py sdist
-import sys
-from setuptools import setup
-import os
-from shutil import copyfile
import glob
+import os
+import sys
from pathlib import Path
+from shutil import copyfile
+
+from setuptools import setup
if (
# When we package, the parent directory 'client' dir
diff --git a/python/packaging/connect/setup.py b/python/packaging/connect/setup.py
index 5ef47a40fc58..d24d0436891b 100755
--- a/python/packaging/connect/setup.py
+++ b/python/packaging/connect/setup.py
@@ -22,12 +22,13 @@
# cd python/packaging/connect
# python setup.py sdist
-import sys
-from setuptools import setup
-import os
-from shutil import copyfile, copytree, rmtree
import glob
+import os
+import sys
from pathlib import Path
+from shutil import copyfile, copytree, rmtree
+
+from setuptools import setup
if (
# When we package, the parent directory 'connect' dir
diff --git a/python/pyspark/__init__.py b/python/pyspark/__init__.py
index f2f8b08b9da7..2dd58e06ec65 100644
--- a/python/pyspark/__init__.py
+++ b/python/pyspark/__init__.py
@@ -48,16 +48,16 @@ Public classes:
import sys
from functools import wraps
-from typing import cast, Any, Callable, TypeVar, Union
+from typing import Any, Callable, TypeVar, Union, cast
from pyspark.util import is_remote_only
if not is_remote_only():
- from pyspark.core.rdd import RDD, RDDBarrier
- from pyspark.core.files import SparkFiles
- from pyspark.core.status import StatusTracker, SparkJobInfo, SparkStageInfo, SparkExecutorInfo
+ from pyspark.core import broadcast, files, rdd, status
from pyspark.core.broadcast import Broadcast
- from pyspark.core import rdd, files, status, broadcast
+ from pyspark.core.files import SparkFiles
+ from pyspark.core.rdd import RDD, RDDBarrier
+ from pyspark.core.status import SparkExecutorInfo, SparkJobInfo, SparkStageInfo, StatusTracker
# for backward compatibility references.
sys.modules["pyspark.rdd"] = rdd
@@ -65,15 +65,15 @@ if not is_remote_only():
sys.modules["pyspark.status"] = status
sys.modules["pyspark.broadcast"] = broadcast
+from pyspark._globals import _NoValue # noqa: F401
+from pyspark.accumulators import Accumulator, AccumulatorParam
from pyspark.conf import SparkConf
-from pyspark.util import InheritableThread, inheritable_thread_target
+from pyspark.profiler import BasicProfiler, Profiler
+from pyspark.serializers import CPickleSerializer, MarshalSerializer
from pyspark.storagelevel import StorageLevel
-from pyspark.accumulators import Accumulator, AccumulatorParam
-from pyspark.serializers import MarshalSerializer, CPickleSerializer
-from pyspark.taskcontext import TaskContext, BarrierTaskContext, BarrierTaskInfo
-from pyspark.profiler import Profiler, BasicProfiler
+from pyspark.taskcontext import BarrierTaskContext, BarrierTaskInfo, TaskContext
+from pyspark.util import InheritableThread, inheritable_thread_target
from pyspark.version import __version__
-from pyspark._globals import _NoValue # noqa: F401
_F = TypeVar("_F", bound=Callable)
@@ -119,14 +119,14 @@ def keyword_only(func: _F) -> _F:
# To avoid circular dependencies
if not is_remote_only():
- from pyspark.core.context import SparkContext
from pyspark.core import context
+ from pyspark.core.context import SparkContext
# for backward compatibility references.
sys.modules["pyspark.context"] = context
# for back compatibility
- from pyspark.sql import SQLContext, HiveContext # noqa: F401
+ from pyspark.sql import HiveContext, SQLContext # noqa: F401
from pyspark.sql import Row # noqa: F401
diff --git a/python/pyspark/_typing.pyi b/python/pyspark/_typing.pyi
index fd4e1e2ad019..a4002788f51b 100644
--- a/python/pyspark/_typing.pyi
+++ b/python/pyspark/_typing.pyi
@@ -17,9 +17,9 @@
# under the License.
from typing import Any, Callable, Iterable, Sized, TypeVar, Union
-from typing_extensions import Literal, Protocol
-from numpy import int32, int64, float32, float64, ndarray
+from numpy import float32, float64, int32, int64, ndarray
+from typing_extensions import Literal, Protocol
F = TypeVar("F", bound=Callable)
T_co = TypeVar("T_co", covariant=True)
diff --git a/python/pyspark/accumulators.py b/python/pyspark/accumulators.py
index fcfa347092ee..e09e987ff417 100644
--- a/python/pyspark/accumulators.py
+++ b/python/pyspark/accumulators.py
@@ -15,22 +15,23 @@
# limitations under the License.
#
-import os
-import sys
import hmac
+import os
import select
-import struct
import socketserver
+import struct
+import sys
import threading
-from typing import Callable, Dict, Generic, Tuple, Type, TYPE_CHECKING, TypeVar, Union, Optional
+from typing import TYPE_CHECKING, Callable, Dict, Generic, Optional, Tuple, Type, TypeVar, Union
-from pyspark.serializers import read_int, CPickleSerializer
from pyspark.errors import PySparkRuntimeError
+from pyspark.serializers import CPickleSerializer, read_int
if TYPE_CHECKING:
- from pyspark._typing import SupportsIAdd
from socketserver import BaseRequestHandler
+ from pyspark._typing import SupportsIAdd
+
__all__ = ["Accumulator", "AccumulatorParam"]
diff --git a/python/pyspark/conf.py b/python/pyspark/conf.py
index 2f109b007b9c..fef0c6fd4c45 100644
--- a/python/pyspark/conf.py
+++ b/python/pyspark/conf.py
@@ -18,13 +18,13 @@
__all__ = ["SparkConf"]
import sys
-from typing import Dict, List, Optional, Tuple, cast, overload, TYPE_CHECKING
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, cast, overload
-from pyspark.util import is_remote_only
from pyspark.errors import PySparkRuntimeError
+from pyspark.util import is_remote_only
if TYPE_CHECKING:
- from py4j.java_gateway import JVMView, JavaObject
+ from py4j.java_gateway import JavaObject, JVMView
class SparkConf:
diff --git a/python/pyspark/core/broadcast.py b/python/pyspark/core/broadcast.py
index 9b0002bdccfe..64cf0ac93700 100644
--- a/python/pyspark/core/broadcast.py
+++ b/python/pyspark/core/broadcast.py
@@ -17,29 +17,29 @@
import gc
import os
+import pickle
import sys
-from tempfile import NamedTemporaryFile
import threading
-import pickle
+from tempfile import NamedTemporaryFile
from typing import (
- overload,
+ IO,
+ TYPE_CHECKING,
Any,
BinaryIO,
Callable,
Dict,
Generic,
- IO,
Iterator,
Optional,
Tuple,
TypeVar,
- TYPE_CHECKING,
Union,
+ overload,
)
-from pyspark.serializers import ChunkedStream, pickle_protocol
-from pyspark.util import print_exec, local_connect_and_auth
from pyspark.errors import PySparkRuntimeError
+from pyspark.serializers import ChunkedStream, pickle_protocol
+from pyspark.util import local_connect_and_auth, print_exec
if TYPE_CHECKING:
from pyspark import SparkContext
@@ -361,8 +361,9 @@ class BroadcastPickleRegistry(threading.local):
def _test() -> None:
import doctest
- from pyspark.sql import SparkSession
+
import pyspark.core.broadcast
+ from pyspark.sql import SparkSession
globs = pyspark.core.broadcast.__dict__.copy()
spark = SparkSession.builder.master("local[4]").appName("broadcast tests").getOrCreate()
diff --git a/python/pyspark/core/context.py b/python/pyspark/core/context.py
index 339c94f54e6f..10e4fe14ab9a 100644
--- a/python/pyspark/core/context.py
+++ b/python/pyspark/core/context.py
@@ -15,21 +15,21 @@
# limitations under the License.
#
-import uuid
+import importlib
import os
import shutil
import signal
import sys
import threading
+import uuid
import warnings
-import importlib
-from threading import RLock
from tempfile import NamedTemporaryFile
+from threading import RLock
from types import TracebackType
from typing import (
+ TYPE_CHECKING,
Any,
Callable,
- cast,
ClassVar,
Dict,
Iterable,
@@ -37,42 +37,42 @@ from typing import (
NoReturn,
Optional,
Sequence,
+ Set,
Tuple,
Type,
- TYPE_CHECKING,
TypeVar,
- Set,
+ cast,
)
from py4j.java_collections import JavaMap
+from py4j.java_gateway import JavaGateway, JavaObject, JVMView, is_instance_of
from py4j.protocol import Py4JError
from pyspark import accumulators
-from pyspark.conf import SparkConf
from pyspark.accumulators import Accumulator
+from pyspark.conf import SparkConf
from pyspark.core.broadcast import Broadcast, BroadcastPickleRegistry
from pyspark.core.files import SparkFiles
+from pyspark.core.rdd import RDD
+from pyspark.core.status import StatusTracker
+from pyspark.errors import PySparkRuntimeError
from pyspark.java_gateway import launch_gateway
+from pyspark.profiler import BasicProfiler, MemoryProfiler, ProfilerCollector, UDFBasicProfiler
+from pyspark.resource.information import ResourceInformation
from pyspark.serializers import (
- CPickleSerializer,
+ AutoBatchedSerializer,
BatchedSerializer,
+ ChunkedStream,
+ CPickleSerializer,
+ NoOpSerializer,
+ PairDeserializer,
Serializer,
UTF8Deserializer,
- PairDeserializer,
- AutoBatchedSerializer,
- NoOpSerializer,
- ChunkedStream,
)
from pyspark.storagelevel import StorageLevel
-from pyspark.resource.information import ResourceInformation
-from pyspark.core.rdd import RDD
-from pyspark.util import _load_from_socket, local_connect_and_auth
from pyspark.taskcontext import TaskContext
from pyspark.traceback_utils import CallSite, first_spark_call
-from pyspark.core.status import StatusTracker
-from pyspark.profiler import ProfilerCollector, BasicProfiler, UDFBasicProfiler, MemoryProfiler
-from pyspark.errors import PySparkRuntimeError
-from py4j.java_gateway import is_instance_of, JavaGateway, JavaObject, JVMView
+from pyspark.util import _load_from_socket, local_connect_and_auth
if TYPE_CHECKING:
from pyspark.accumulators import AccumulatorParam
diff --git a/python/pyspark/core/files.py b/python/pyspark/core/files.py
index aec67a0fe31c..a3c59e768150 100644
--- a/python/pyspark/core/files.py
+++ b/python/pyspark/core/files.py
@@ -19,7 +19,7 @@ import os
__all__ = ["SparkFiles"]
-from typing import cast, ClassVar, Optional, TYPE_CHECKING
+from typing import TYPE_CHECKING, ClassVar, Optional, cast
if TYPE_CHECKING:
from pyspark import SparkContext
@@ -149,6 +149,7 @@ class SparkFiles:
def _test() -> None:
import doctest
import sys
+
from pyspark import SparkContext
globs = globals().copy()
diff --git a/python/pyspark/core/rdd.py b/python/pyspark/core/rdd.py
index 190857bf3cec..629a1bc93e6b 100644
--- a/python/pyspark/core/rdd.py
+++ b/python/pyspark/core/rdd.py
@@ -15,22 +15,24 @@
# limitations under the License.
#
+import bisect
import copy
-import sys
-import os
+import heapq
import operator
+import os
+import random
import shlex
+import sys
import warnings
-import heapq
-import bisect
-import random
-from subprocess import Popen, PIPE
-from threading import Thread
from collections import defaultdict
-from itertools import chain
from functools import reduce
-from math import sqrt, log, isinf, isnan, pow, ceil
+from itertools import chain
+from math import ceil, isinf, isnan, log, pow, sqrt
+from subprocess import PIPE, Popen
+from threading import Thread
from typing import (
+ IO,
+ TYPE_CHECKING,
Any,
Callable,
Dict,
@@ -38,73 +40,71 @@ from typing import (
Hashable,
Iterable,
Iterator,
- IO,
List,
NoReturn,
Optional,
Sequence,
Tuple,
- Union,
TypeVar,
+ Union,
cast,
overload,
- TYPE_CHECKING,
)
+from pyspark.errors import PySparkRuntimeError
+from pyspark.join import (
+ python_cogroup,
+ python_full_outer_join,
+ python_join,
+ python_left_outer_join,
+ python_right_outer_join,
+)
+from pyspark.rddsampler import RDDRangeSampler, RDDSampler, RDDStratifiedSampler
+from pyspark.resource.profile import ResourceProfile
+from pyspark.resource.requests import ExecutorResourceRequests, TaskResourceRequests
+from pyspark.resultiterable import ResultIterable
from pyspark.serializers import (
AutoBatchedSerializer,
BatchedSerializer,
- NoOpSerializer,
CartesianDeserializer,
CloudPickleSerializer,
- PairDeserializer,
CPickleSerializer,
+ NoOpSerializer,
+ PairDeserializer,
Serializer,
pack_long,
)
-from pyspark.join import (
- python_join,
- python_left_outer_join,
- python_right_outer_join,
- python_full_outer_join,
- python_cogroup,
-)
-from pyspark.statcounter import StatCounter
-from pyspark.rddsampler import RDDSampler, RDDRangeSampler, RDDStratifiedSampler
-from pyspark.storagelevel import StorageLevel
-from pyspark.resource.requests import ExecutorResourceRequests, TaskResourceRequests
-from pyspark.resource.profile import ResourceProfile
-from pyspark.resultiterable import ResultIterable
from pyspark.shuffle import (
Aggregator,
+ ExternalGroupBy,
ExternalMerger,
- get_used_memory,
ExternalSorter,
- ExternalGroupBy,
+ get_used_memory,
)
+from pyspark.statcounter import StatCounter
+from pyspark.storagelevel import StorageLevel
from pyspark.traceback_utils import SCCallSiteSync
+
+# for backward compatibility references.
from pyspark.util import (
- fail_on_stopiteration,
- _parse_memory,
+ PythonEvalType, # noqa: F401
_load_from_socket,
_local_iterator_from_socket,
+ _parse_memory,
+ fail_on_stopiteration,
)
-from pyspark.errors import PySparkRuntimeError
-
-# for backward compatibility references.
-from pyspark.util import PythonEvalType # noqa: F401
if TYPE_CHECKING:
from py4j.java_gateway import JavaObject
- from pyspark._typing import S, SizedIterable, NumberOrArray
+ from pyspark._typing import NumberOrArray, S, SizedIterable
from pyspark.core.context import SparkContext
- from pyspark.sql.dataframe import DataFrame
- from pyspark.sql.types import AtomicType, StructType
from pyspark.sql._typing import (
AtomicValue,
RowLike,
)
+ from pyspark.sql.dataframe import DataFrame
+ from pyspark.sql.types import AtomicType, StructType
T = TypeVar("T")
T_co = TypeVar("T_co", covariant=True)
@@ -5356,6 +5356,7 @@ class PipelinedRDD(RDD[U], Generic[T, U]):
def _test() -> None:
import doctest
import tempfile
+
from pyspark.core.context import SparkContext
try:
diff --git a/python/pyspark/daemon.py b/python/pyspark/daemon.py
index 0e321d442e81..ab761d911d14 100644
--- a/python/pyspark/daemon.py
+++ b/python/pyspark/daemon.py
@@ -14,25 +14,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-import uuid
+import faulthandler
+import gc
import os
-import signal
import select
+import signal
import socket
import sys
-import traceback
import time
-import gc
-import faulthandler
-from errno import EINTR, EAGAIN
+import traceback
+import uuid
+from errno import EAGAIN, EINTR
+from signal import SIG_DFL, SIG_IGN, SIGCHLD, SIGHUP, SIGINT, SIGTERM
from socket import AF_INET, AF_INET6, SOCK_STREAM, SOMAXCONN
-from signal import SIGHUP, SIGTERM, SIGCHLD, SIG_DFL, SIG_IGN, SIGINT
from types import FrameType
from typing import Any, Optional
-from pyspark.serializers import read_int, write_int, write_with_length, UTF8Deserializer
-from pyspark.util import enable_faulthandler
from pyspark.errors import PySparkRuntimeError
+from pyspark.serializers import UTF8Deserializer, read_int, write_int, write_with_length
+from pyspark.util import enable_faulthandler
def compute_real_exit_code(exit_code: Any) -> int:
diff --git a/python/pyspark/errors/__init__.py b/python/pyspark/errors/__init__.py
index cde12949c9ca..22f75a6d1b27 100644
--- a/python/pyspark/errors/__init__.py
+++ b/python/pyspark/errors/__init__.py
@@ -20,38 +20,38 @@ PySpark exceptions.
"""
from pyspark.errors.exceptions.base import (
- PySparkException,
AnalysisException,
- SessionNotSameException,
- TempTableAlreadyExistsException,
- ParseException,
- IllegalArgumentException,
ArithmeticException,
- UnsupportedOperationException,
ArrayIndexOutOfBoundsException,
DateTimeException,
+ IllegalArgumentException,
NumberFormatException,
- StreamingQueryException,
- QueryExecutionException,
- PythonException,
- UnknownException,
- SparkRuntimeException,
- SparkUpgradeException,
- SparkNoSuchElementException,
- PySparkTypeError,
- PySparkValueError,
+ ParseException,
+ PickleException,
+ PySparkAssertionError,
+ PySparkAttributeError,
+ PySparkException,
PySparkImportError,
PySparkIndexError,
- PySparkAttributeError,
- PySparkRuntimeError,
- PySparkAssertionError,
+ PySparkKeyError,
PySparkNotImplementedError,
PySparkPicklingError,
- PySparkKeyError,
+ PySparkRuntimeError,
+ PySparkTypeError,
+ PySparkValueError,
+ PythonException,
QueryContext,
QueryContextType,
+ QueryExecutionException,
+ SessionNotSameException,
+ SparkNoSuchElementException,
+ SparkRuntimeException,
+ SparkUpgradeException,
StreamingPythonRunnerInitializationException,
- PickleException,
+ StreamingQueryException,
+ TempTableAlreadyExistsException,
+ UnknownException,
+ UnsupportedOperationException,
)
__all__ = [
diff --git a/python/pyspark/errors/error_classes.py b/python/pyspark/errors/error_classes.py
index 4a840bafb8ae..2f0417b0c8db 100644
--- a/python/pyspark/errors/error_classes.py
+++ b/python/pyspark/errors/error_classes.py
@@ -15,8 +15,8 @@
# limitations under the License.
#
-import json
import importlib.resources
+import json
# Note: Though we call them "error classes" here, the proper name is "error conditions",
# hence why the name of the JSON file is different.
diff --git a/python/pyspark/errors/exceptions/__init__.py b/python/pyspark/errors/exceptions/__init__.py
index 3de00c17500c..b858a3b99672 100644
--- a/python/pyspark/errors/exceptions/__init__.py
+++ b/python/pyspark/errors/exceptions/__init__.py
@@ -19,6 +19,7 @@
def _write_self() -> None:
import json
from pathlib import Path
+
from pyspark.errors import error_classes
ERRORS_DIR = Path(__file__).parents[1]
diff --git a/python/pyspark/errors/exceptions/base.py b/python/pyspark/errors/exceptions/base.py
index 1e9f0fef27a7..aeabb46c2579 100644
--- a/python/pyspark/errors/exceptions/base.py
+++ b/python/pyspark/errors/exceptions/base.py
@@ -17,12 +17,12 @@
import warnings
from abc import ABC, abstractmethod
from enum import Enum
-from typing import Any, Dict, Optional, TypeVar, cast, Iterable, TYPE_CHECKING, List
+from pickle import PicklingError
+from typing import TYPE_CHECKING, Any, Dict, Iterable, List, Optional, TypeVar, cast
from pyspark.errors.exceptions.tblib import Traceback
from pyspark.errors.utils import ErrorClassesReader
from pyspark.logger import PySparkLogger
-from pickle import PicklingError
if TYPE_CHECKING:
from pyspark.sql.types import Row
diff --git a/python/pyspark/errors/exceptions/captured.py b/python/pyspark/errors/exceptions/captured.py
index da755de0d6ec..8aa9d9b624e1 100644
--- a/python/pyspark/errors/exceptions/captured.py
+++ b/python/pyspark/errors/exceptions/captured.py
@@ -16,33 +16,65 @@
#
import warnings
from contextlib import contextmanager
-from typing import Any, Callable, Dict, Iterator, Optional, cast, List, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Optional, cast
from pyspark.errors.exceptions.base import (
AnalysisException as BaseAnalysisException,
- IllegalArgumentException as BaseIllegalArgumentException,
+)
+from pyspark.errors.exceptions.base import (
ArithmeticException as BaseArithmeticException,
- UnsupportedOperationException as BaseUnsupportedOperationException,
+)
+from pyspark.errors.exceptions.base import (
ArrayIndexOutOfBoundsException as BaseArrayIndexOutOfBoundsException,
+)
+from pyspark.errors.exceptions.base import (
DateTimeException as BaseDateTimeException,
+)
+from pyspark.errors.exceptions.base import (
+ IllegalArgumentException as BaseIllegalArgumentException,
+)
+from pyspark.errors.exceptions.base import (
NumberFormatException as BaseNumberFormatException,
+)
+from pyspark.errors.exceptions.base import (
ParseException as BaseParseException,
+)
+from pyspark.errors.exceptions.base import (
PySparkException,
+ QueryContextType,
+ recover_python_exception,
+)
+from pyspark.errors.exceptions.base import (
PythonException as BasePythonException,
+)
+from pyspark.errors.exceptions.base import (
+ QueryContext as BaseQueryContext,
+)
+from pyspark.errors.exceptions.base import (
QueryExecutionException as BaseQueryExecutionException,
+)
+from pyspark.errors.exceptions.base import (
+ SparkNoSuchElementException as BaseNoSuchElementException,
+)
+from pyspark.errors.exceptions.base import (
SparkRuntimeException as BaseSparkRuntimeException,
+)
+from pyspark.errors.exceptions.base import (
SparkUpgradeException as BaseSparkUpgradeException,
- SparkNoSuchElementException as BaseNoSuchElementException,
+)
+from pyspark.errors.exceptions.base import (
StreamingQueryException as BaseStreamingQueryException,
+)
+from pyspark.errors.exceptions.base import (
UnknownException as BaseUnknownException,
- QueryContext as BaseQueryContext,
- QueryContextType,
- recover_python_exception,
+)
+from pyspark.errors.exceptions.base import (
+ UnsupportedOperationException as BaseUnsupportedOperationException,
)
if TYPE_CHECKING:
- from py4j.protocol import Py4JJavaError
from py4j.java_gateway import JavaObject
+ from py4j.protocol import Py4JJavaError
class CapturedException(PySparkException):
@@ -53,9 +85,10 @@ class CapturedException(PySparkException):
cause: Optional["Py4JJavaError"] = None,
origin: Optional["Py4JJavaError"] = None,
):
- from pyspark import SparkContext
from py4j.protocol import Py4JJavaError
+ from pyspark import SparkContext
+
# desc & stackTrace vs origin are mutually exclusive.
# cause is optional.
assert (origin is not None and desc is None and stackTrace is None) or (
@@ -98,9 +131,10 @@ class CapturedException(PySparkException):
return str(desc)
def getCondition(self) -> Optional[str]:
- from pyspark import SparkContext
from py4j.java_gateway import is_instance_of
+ from pyspark import SparkContext
+
assert SparkContext._gateway is not None
assert SparkContext._jvm is not None
@@ -118,9 +152,10 @@ class CapturedException(PySparkException):
return self.getCondition()
def getMessageParameters(self) -> Optional[Dict[str, str]]:
- from pyspark import SparkContext
from py4j.java_gateway import is_instance_of
+ from pyspark import SparkContext
+
assert SparkContext._gateway is not None
assert SparkContext._jvm is not None
@@ -134,9 +169,10 @@ class CapturedException(PySparkException):
return None
def getSqlState(self) -> Optional[str]:
- from pyspark import SparkContext
from py4j.java_gateway import is_instance_of
+ from pyspark import SparkContext
+
assert SparkContext._gateway is not None
assert SparkContext._jvm is not None
gw = SparkContext._gateway
@@ -149,9 +185,10 @@ class CapturedException(PySparkException):
return None
def getMessage(self) -> str:
- from pyspark import SparkContext
from py4j.java_gateway import is_instance_of
+ from pyspark import SparkContext
+
assert SparkContext._gateway is not None
assert SparkContext._jvm is not None
gw = SparkContext._gateway
@@ -172,9 +209,10 @@ class CapturedException(PySparkException):
return ""
def getQueryContext(self) -> List[BaseQueryContext]:
- from pyspark import SparkContext
from py4j.java_gateway import is_instance_of
+ from pyspark import SparkContext
+
assert SparkContext._gateway is not None
assert SparkContext._jvm is not None
@@ -201,9 +239,10 @@ def convert_exception(e: "Py4JJavaError") -> CapturedException:
def _convert_exception(e: "Py4JJavaError") -> CapturedException:
- from pyspark import SparkContext
from py4j.java_gateway import is_instance_of
+ from pyspark import SparkContext
+
assert e is not None
assert SparkContext._jvm is not None
assert SparkContext._gateway is not None
@@ -268,9 +307,10 @@ def capture_sql_exception(f: Callable[..., Any]) -> Callable[..., Any]:
@contextmanager
def unwrap_spark_exception() -> Iterator[Any]:
- from pyspark import SparkContext
- from py4j.protocol import Py4JJavaError
from py4j.java_gateway import is_instance_of
+ from py4j.protocol import Py4JJavaError
+
+ from pyspark import SparkContext
assert SparkContext._gateway is not None
diff --git a/python/pyspark/errors/exceptions/connect.py b/python/pyspark/errors/exceptions/connect.py
index abe436eac80c..794a9d7e73a7 100644
--- a/python/pyspark/errors/exceptions/connect.py
+++ b/python/pyspark/errors/exceptions/connect.py
@@ -14,39 +14,77 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-import grpc
import json
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+import grpc
from grpc import StatusCode
-from typing import Any, Dict, List, Optional, TYPE_CHECKING
from pyspark.errors.exceptions.base import (
AnalysisException as BaseAnalysisException,
- IllegalArgumentException as BaseIllegalArgumentException,
+)
+from pyspark.errors.exceptions.base import (
ArithmeticException as BaseArithmeticException,
- UnsupportedOperationException as BaseUnsupportedOperationException,
+)
+from pyspark.errors.exceptions.base import (
ArrayIndexOutOfBoundsException as BaseArrayIndexOutOfBoundsException,
+)
+from pyspark.errors.exceptions.base import (
DateTimeException as BaseDateTimeException,
+)
+from pyspark.errors.exceptions.base import (
+ IllegalArgumentException as BaseIllegalArgumentException,
+)
+from pyspark.errors.exceptions.base import (
NumberFormatException as BaseNumberFormatException,
+)
+from pyspark.errors.exceptions.base import (
ParseException as BaseParseException,
+)
+from pyspark.errors.exceptions.base import (
+ PickleException as BasePickleException,
+)
+from pyspark.errors.exceptions.base import (
PySparkException,
+ QueryContextType,
+ recover_python_exception,
+)
+from pyspark.errors.exceptions.base import (
PythonException as BasePythonException,
- StreamingQueryException as BaseStreamingQueryException,
+)
+from pyspark.errors.exceptions.base import (
+ QueryContext as BaseQueryContext,
+)
+from pyspark.errors.exceptions.base import (
QueryExecutionException as BaseQueryExecutionException,
- SparkRuntimeException as BaseSparkRuntimeException,
+)
+from pyspark.errors.exceptions.base import (
SparkNoSuchElementException as BaseNoSuchElementException,
+)
+from pyspark.errors.exceptions.base import (
+ SparkRuntimeException as BaseSparkRuntimeException,
+)
+from pyspark.errors.exceptions.base import (
SparkUpgradeException as BaseSparkUpgradeException,
- QueryContext as BaseQueryContext,
- QueryContextType,
+)
+from pyspark.errors.exceptions.base import (
StreamingPythonRunnerInitializationException as BaseStreamingPythonRunnerInitException,
- PickleException as BasePickleException,
+)
+from pyspark.errors.exceptions.base import (
+ StreamingQueryException as BaseStreamingQueryException,
+)
+from pyspark.errors.exceptions.base import (
UnknownException as BaseUnknownException,
- recover_python_exception,
+)
+from pyspark.errors.exceptions.base import (
+ UnsupportedOperationException as BaseUnsupportedOperationException,
)
if TYPE_CHECKING:
- import pyspark.sql.connect.proto as pb2
from google.rpc.error_details_pb2 import ErrorInfo
+ import pyspark.sql.connect.proto as pb2
+
class SparkConnectException(PySparkException):
"""
diff --git a/python/pyspark/errors/tests/test_connect_errors_conversion.py b/python/pyspark/errors/tests/test_connect_errors_conversion.py
index d3bf96c4acb4..1bda1bc19be9 100644
--- a/python/pyspark/errors/tests/test_connect_errors_conversion.py
+++ b/python/pyspark/errors/tests/test_connect_errors_conversion.py
@@ -17,15 +17,15 @@
import unittest
-from pyspark.testing.utils import should_test_connect, connect_requirement_message
+from pyspark.testing.utils import connect_requirement_message, should_test_connect
if should_test_connect:
from pyspark.errors.exceptions.connect import (
- convert_exception,
EXCEPTION_CLASS_MAPPING,
- SparkConnectGrpcException,
- PythonException,
AnalysisException,
+ PythonException,
+ SparkConnectGrpcException,
+ convert_exception,
)
@@ -116,6 +116,7 @@ class ConnectErrorsTest(unittest.TestCase):
def test_convert_exception_with_stacktrace(self):
# Mock FetchErrorDetailsResponse with stacktrace
from google.rpc.error_details_pb2 import ErrorInfo
+
from pyspark.sql.connect.proto import FetchErrorDetailsResponse as pb2
resp = pb2(
@@ -190,10 +191,11 @@ class ConnectErrorsTest(unittest.TestCase):
def test_convert_exception_with_breaking_change_info(self):
"""Test that breaking change info is correctly extracted from protobuf response."""
- import pyspark.sql.connect.proto as pb2
from google.rpc.error_details_pb2 import ErrorInfo
from grpc import StatusCode
+ import pyspark.sql.connect.proto as pb2
+
# Create mock FetchErrorDetailsResponse with breaking change info
resp = pb2.FetchErrorDetailsResponse()
resp.root_error_idx = 0
@@ -245,10 +247,11 @@ class ConnectErrorsTest(unittest.TestCase):
def test_convert_exception_without_breaking_change_info(self):
"""Test that getBreakingChangeInfo returns None when no breaking change info."""
- import pyspark.sql.connect.proto as pb2
from google.rpc.error_details_pb2 import ErrorInfo
from grpc import StatusCode
+ import pyspark.sql.connect.proto as pb2
+
# Create mock FetchErrorDetailsResponse without breaking change info
resp = pb2.FetchErrorDetailsResponse()
resp.root_error_idx = 0
@@ -325,10 +328,11 @@ class ConnectErrorsTest(unittest.TestCase):
def test_breaking_change_info_without_mitigation_config(self):
"""Test breaking change info that only has migration messages."""
- import pyspark.sql.connect.proto as pb2
from google.rpc.error_details_pb2 import ErrorInfo
from grpc import StatusCode
+ import pyspark.sql.connect.proto as pb2
+
# Create mock FetchErrorDetailsResponse with breaking change info (no mitigation config)
resp = pb2.FetchErrorDetailsResponse()
resp.root_error_idx = 0
@@ -367,10 +371,11 @@ class ConnectErrorsTest(unittest.TestCase):
def test_convert_exception_error_class_from_fetch_error_details(self):
"""Test that errorClass is extracted from FetchErrorDetailsResponse
when not present in ErrorInfo metadata (e.g., when messageParameters exceed limit)."""
- import pyspark.sql.connect.proto as pb2
from google.rpc.error_details_pb2 import ErrorInfo
from grpc import StatusCode
+ import pyspark.sql.connect.proto as pb2
+
# Create mock FetchErrorDetailsResponse with errorClass
resp = pb2.FetchErrorDetailsResponse()
resp.root_error_idx = 0
diff --git a/python/pyspark/errors/utils.py b/python/pyspark/errors/utils.py
index 53a5e18901e6..115d9f31cb3b 100644
--- a/python/pyspark/errors/utils.py
+++ b/python/pyspark/errors/utils.py
@@ -14,12 +14,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-import re
import functools
import inspect
import itertools
import os
+import re
import threading
+from types import FrameType
from typing import (
Any,
Callable,
@@ -27,14 +28,13 @@ from typing import (
Iterator,
List,
Match,
- TypeVar,
- Type,
Optional,
+ Type,
+ TypeVar,
Union,
- overload,
cast,
+ overload,
)
-from types import FrameType
import pyspark
from pyspark.errors.error_classes import ERROR_CLASSES_MAP
@@ -264,10 +264,9 @@ def _capture_call_site(depth: int) -> str:
# We try import here since IPython is not a required dependency
try:
- import IPython
-
# ipykernel is required for IPython
import ipykernel
+ import IPython
ipython = IPython.get_ipython()
# Filtering out IPython related frames
diff --git a/python/pyspark/errors_doc_gen.py b/python/pyspark/errors_doc_gen.py
index 6cdd376d20cf..30bd1dc1be35 100644
--- a/python/pyspark/errors_doc_gen.py
+++ b/python/pyspark/errors_doc_gen.py
@@ -1,4 +1,5 @@
import re
+
from pyspark.errors.error_classes import ERROR_CLASSES_MAP
_ERROR_DOC_HEADER = """
diff --git a/python/pyspark/install.py b/python/pyspark/install.py
index 4426a5899b32..fd157f3a08c2 100644
--- a/python/pyspark/install.py
+++ b/python/pyspark/install.py
@@ -23,7 +23,6 @@ import urllib.request
from shutil import rmtree
from typing import TYPE_CHECKING
-
if TYPE_CHECKING:
from http.client import HTTPResponse
diff --git a/python/pyspark/instrumentation_utils.py b/python/pyspark/instrumentation_utils.py
index 1f822207c4af..0ee75d43378c 100644
--- a/python/pyspark/instrumentation_utils.py
+++ b/python/pyspark/instrumentation_utils.py
@@ -16,12 +16,12 @@
#
import functools
+import importlib
import inspect
import threading
-import importlib
import time
from types import ModuleType
-from typing import Tuple, Union, List, Callable, Any, Type
+from typing import Any, Callable, List, Tuple, Type, Union
__all__: List[str] = []
diff --git a/python/pyspark/java_gateway.py b/python/pyspark/java_gateway.py
index 6303a4361857..eccca1f412a7 100644
--- a/python/pyspark/java_gateway.py
+++ b/python/pyspark/java_gateway.py
@@ -17,20 +17,20 @@
import atexit
import os
-import signal
+import platform
import shlex
import shutil
-import platform
+import signal
import tempfile
import time
-from subprocess import Popen, PIPE
+from subprocess import PIPE, Popen
-from py4j.java_gateway import java_import, JavaGateway, JavaObject, GatewayParameters
from py4j.clientserver import ClientServer, JavaParameters, PythonParameters
-from pyspark.serializers import read_int, UTF8Deserializer
+from py4j.java_gateway import GatewayParameters, JavaGateway, JavaObject, java_import
-from pyspark.find_spark_home import _find_spark_home
from pyspark.errors import PySparkRuntimeError
+from pyspark.find_spark_home import _find_spark_home
+from pyspark.serializers import UTF8Deserializer, read_int
# for backward compatibility references.
from pyspark.util import local_connect_and_auth # noqa: F401
diff --git a/python/pyspark/logger/__init__.py b/python/pyspark/logger/__init__.py
index 9e9548e91983..fa01187748a1 100644
--- a/python/pyspark/logger/__init__.py
+++ b/python/pyspark/logger/__init__.py
@@ -19,6 +19,6 @@
PySpark logging
"""
-from pyspark.logger.logger import PySparkLogger, SPARK_LOG_SCHEMA
+from pyspark.logger.logger import SPARK_LOG_SCHEMA, PySparkLogger
__all__ = ["PySparkLogger", "SPARK_LOG_SCHEMA"]
diff --git a/python/pyspark/logger/logger.py b/python/pyspark/logger/logger.py
index 8d08275fe03c..bb390fe8b0cf 100644
--- a/python/pyspark/logger/logger.py
+++ b/python/pyspark/logger/logger.py
@@ -15,11 +15,11 @@
# limitations under the License.
#
-import logging
import json
-import traceback
+import logging
import sys
-from typing import cast, Mapping, Optional, TYPE_CHECKING
+import traceback
+from typing import TYPE_CHECKING, Mapping, Optional, cast
if TYPE_CHECKING:
from logging import _ArgsType, _ExcInfoType
@@ -314,6 +314,7 @@ class PySparkLogger(logging.Logger):
def _test() -> None:
import doctest
+
import pyspark.logger.logger
globs = pyspark.logger.logger.__dict__.copy()
diff --git a/python/pyspark/logger/tests/test_logger.py b/python/pyspark/logger/tests/test_logger.py
index 7c3fd4b1e61e..c3227b01045d 100644
--- a/python/pyspark/logger/tests/test_logger.py
+++ b/python/pyspark/logger/tests/test_logger.py
@@ -14,13 +14,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-import logging
import json
+import logging
import tempfile
from io import StringIO
+
from pyspark.errors import ArithmeticException
-from pyspark.logger.logger import PySparkLogger, SPARK_LOG_SCHEMA
-from pyspark.sql import Row, functions as sf
+from pyspark.logger.logger import SPARK_LOG_SCHEMA, PySparkLogger
+from pyspark.sql import Row
+from pyspark.sql import functions as sf
from pyspark.testing import assertDataFrameEqual
from pyspark.testing.sqlutils import ReusedSQLTestCase
diff --git a/python/pyspark/logger/worker_io.py b/python/pyspark/logger/worker_io.py
index 4b8a7cf15912..b573b38e3ecc 100644
--- a/python/pyspark/logger/worker_io.py
+++ b/python/pyspark/logger/worker_io.py
@@ -15,15 +15,15 @@
# limitations under the License.
#
-from contextlib import contextmanager
import inspect
import io
import logging
import os
import sys
import time
-from typing import BinaryIO, Callable, Generator, Iterable, Iterator, Optional, TextIO, Union
+from contextlib import contextmanager
from types import FrameType, TracebackType
+from typing import BinaryIO, Callable, Generator, Iterable, Iterator, Optional, TextIO, Union
from pyspark.logger.logger import JSONFormatter
diff --git a/python/pyspark/memory_profiler_ext.py b/python/pyspark/memory_profiler_ext.py
index df09377c9d54..a3e75c8e9f40 100644
--- a/python/pyspark/memory_profiler_ext.py
+++ b/python/pyspark/memory_profiler_ext.py
@@ -15,10 +15,10 @@
# limitations under the License.
#
-from types import CodeType
-from typing import Any, Optional, List, Iterator, Tuple, Type, TYPE_CHECKING, Callable
import inspect
import warnings
+from types import CodeType
+from typing import TYPE_CHECKING, Any, Callable, Iterator, List, Optional, Tuple, Type
if TYPE_CHECKING:
has_memory_profiler: bool
diff --git a/python/pyspark/messages/__init__.py b/python/pyspark/messages/__init__.py
index 69cfbf6bd53a..d99409c606ea 100644
--- a/python/pyspark/messages/__init__.py
+++ b/python/pyspark/messages/__init__.py
@@ -15,9 +15,9 @@
# limitations under the License.
#
+from pyspark.messages.socket.spark_socket_message_receiver import SparkSocketMessageReceiver
from pyspark.messages.spark_message_receiver import SparkMessageReceiver
from pyspark.messages.zero_copy_byte_stream import ZeroCopyByteStream
-from pyspark.messages.socket.spark_socket_message_receiver import SparkSocketMessageReceiver
__all__ = [
"SparkMessageReceiver",
diff --git a/python/pyspark/messages/socket/spark_socket_message_receiver.py b/python/pyspark/messages/socket/spark_socket_message_receiver.py
index 7099f0207c1f..3a17a86e8069 100644
--- a/python/pyspark/messages/socket/spark_socket_message_receiver.py
+++ b/python/pyspark/messages/socket/spark_socket_message_receiver.py
@@ -17,11 +17,11 @@
from typing import BinaryIO
-from pyspark.serializers import read_int, SpecialLengths
-from pyspark.messages.zero_copy_byte_stream import ZeroCopyByteStream
from pyspark.messages.spark_message_receiver import (
SparkMessageReceiver,
)
+from pyspark.messages.zero_copy_byte_stream import ZeroCopyByteStream
+from pyspark.serializers import SpecialLengths, read_int
def _assert_message_id(message_id: int, expected: int) -> None:
diff --git a/python/pyspark/messages/spark_message_receiver.py b/python/pyspark/messages/spark_message_receiver.py
index ec6b6fc30624..2266558d4aa3 100644
--- a/python/pyspark/messages/spark_message_receiver.py
+++ b/python/pyspark/messages/spark_message_receiver.py
@@ -15,14 +15,13 @@
# limitations under the License.
#
+from abc import ABC, abstractmethod
from enum import Enum
from functools import wraps
from typing import BinaryIO, Callable, TypeVar
-from abc import ABC, abstractmethod
from pyspark.messages.zero_copy_byte_stream import ZeroCopyByteStream
-
T = TypeVar("T", bound="SparkMessageReceiver")
R = TypeVar("R")
diff --git a/python/pyspark/messages/zero_copy_byte_stream.py b/python/pyspark/messages/zero_copy_byte_stream.py
index 611b4f928be0..69e58905c020 100644
--- a/python/pyspark/messages/zero_copy_byte_stream.py
+++ b/python/pyspark/messages/zero_copy_byte_stream.py
@@ -16,8 +16,8 @@
#
import threading
-from typing import Optional
from collections import deque
+from typing import Optional
class ZeroCopyByteStream:
diff --git a/python/pyspark/ml/__init__.py b/python/pyspark/ml/__init__.py
index 2f692ee2e569..4197a5db2635 100644
--- a/python/pyspark/ml/__init__.py
+++ b/python/pyspark/ml/__init__.py
@@ -20,15 +20,6 @@ DataFrame-based machine learning APIs to let users quickly assemble and configur
machine learning pipelines.
"""
-from pyspark.ml.base import (
- Estimator,
- Model,
- Predictor,
- PredictionModel,
- Transformer,
- UnaryTransformer,
-)
-from pyspark.ml.pipeline import Pipeline, PipelineModel
from pyspark.ml import (
classification,
clustering,
@@ -36,14 +27,23 @@ from pyspark.ml import (
feature,
fpm,
image,
+ linalg,
+ param,
recommendation,
regression,
stat,
tuning,
util,
- linalg,
- param,
)
+from pyspark.ml.base import (
+ Estimator,
+ Model,
+ PredictionModel,
+ Predictor,
+ Transformer,
+ UnaryTransformer,
+)
+from pyspark.ml.pipeline import Pipeline, PipelineModel
from pyspark.ml.torch.distributor import TorchDistributor
__all__ = [
diff --git a/python/pyspark/ml/_typing.pyi b/python/pyspark/ml/_typing.pyi
index c24dfe577350..f7224d5f038f 100644
--- a/python/pyspark/ml/_typing.pyi
+++ b/python/pyspark/ml/_typing.pyi
@@ -16,19 +16,19 @@
# specific language governing permissions and limitations
# under the License.
-from typing import Any, Dict, List, TYPE_CHECKING, TypeVar, Tuple, Union
-from typing_extensions import Literal
+from typing import TYPE_CHECKING, Any, Dict, List, Tuple, TypeVar, Union
from numpy import ndarray
from py4j.java_gateway import JavaObject
+from typing_extensions import Literal
import pyspark.ml.base
import pyspark.ml.param
-from pyspark.ml.linalg import Vector
import pyspark.ml.wrapper
+from pyspark.ml.linalg import Vector
if TYPE_CHECKING:
- from scipy.sparse import spmatrix, sparray
+ from scipy.sparse import sparray, spmatrix
ParamMap = Dict[pyspark.ml.param.Param, Any]
PipelineStage = Union[pyspark.ml.base.Estimator, pyspark.ml.base.Transformer]
diff --git a/python/pyspark/ml/base.py b/python/pyspark/ml/base.py
index 882c9b1f65a0..928976eee257 100644
--- a/python/pyspark/ml/base.py
+++ b/python/pyspark/ml/base.py
@@ -15,10 +15,11 @@
# limitations under the License.
#
-from abc import ABCMeta, abstractmethod
import copy
import threading
+from abc import ABCMeta, abstractmethod
from typing import (
+ TYPE_CHECKING,
Any,
Callable,
Generic,
@@ -31,17 +32,16 @@ from typing import (
Union,
cast,
overload,
- TYPE_CHECKING,
)
from pyspark import since
-from pyspark.ml.param import P
from pyspark.ml.common import inherit_doc
+from pyspark.ml.param import P
from pyspark.ml.param.shared import (
+ HasFeaturesCol,
HasInputCol,
- HasOutputCol,
HasLabelCol,
- HasFeaturesCol,
+ HasOutputCol,
HasPredictionCol,
Params,
)
diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py
index 4b7f2e4da209..0006cbc619ee 100644
--- a/python/pyspark/ml/classification.py
+++ b/python/pyspark/ml/classification.py
@@ -20,88 +20,90 @@ import sys
import uuid
import warnings
from abc import ABCMeta, abstractmethod
-from multiprocessing.pool import ThreadPool
from functools import cached_property
+from multiprocessing.pool import ThreadPool
from typing import (
+ TYPE_CHECKING,
Any,
+ Callable,
Dict,
Generic,
List,
Optional,
+ Tuple,
Type,
TypeVar,
Union,
cast,
overload,
- TYPE_CHECKING,
- Tuple,
- Callable,
)
-from pyspark import keyword_only, since, inheritable_thread_target
-from pyspark.ml import Estimator, Predictor, PredictionModel, Model, functions as MF
+from pyspark import inheritable_thread_target, keyword_only, since
+from pyspark.ml import functions as MF
+from pyspark.ml.base import Estimator, Model, PredictionModel, Predictor, _PredictorParams
+from pyspark.ml.common import inherit_doc
+from pyspark.ml.linalg import Matrix, Vector
from pyspark.ml.param.shared import (
- HasRawPredictionCol,
+ HasAggregationDepth,
+ HasBlockSize,
+ HasElasticNetParam,
+ HasFitIntercept,
+ HasMaxBlockSizeInMB,
+ HasMaxIter,
+ HasParallelism,
HasProbabilityCol,
- HasThresholds,
+ HasRawPredictionCol,
HasRegParam,
- HasMaxIter,
- HasFitIntercept,
- HasTol,
+ HasSeed,
+ HasSolver,
HasStandardization,
- HasWeightCol,
- HasAggregationDepth,
+ HasStepSize,
HasThreshold,
- HasBlockSize,
- HasMaxBlockSizeInMB,
+ HasThresholds,
+ HasTol,
+ HasWeightCol,
Param,
Params,
TypeConverters,
- HasElasticNetParam,
- HasSeed,
- HasStepSize,
- HasSolver,
- HasParallelism,
)
+from pyspark.ml.regression import DecisionTreeRegressionModel, _FactorizationMachinesParams
from pyspark.ml.tree import (
_DecisionTreeModel,
_DecisionTreeParams,
- _TreeEnsembleModel,
- _RandomForestParams,
_GBTParams,
_HasVarianceImpurity,
+ _RandomForestParams,
_TreeClassifierParams,
+ _TreeEnsembleModel,
)
-from pyspark.ml.regression import _FactorizationMachinesParams, DecisionTreeRegressionModel
-from pyspark.ml.base import _PredictorParams
from pyspark.ml.util import (
DefaultParamsReader,
DefaultParamsWriter,
+ HasTrainingSummary,
JavaMLReadable,
JavaMLWritable,
JavaMLWriter,
- MLReader,
MLReadable,
- MLWriter,
+ MLReader,
MLWritable,
- HasTrainingSummary,
+ MLWriter,
+ _cache_spark_dataset,
+ try_remote_attribute_relation,
try_remote_read,
try_remote_write,
- try_remote_attribute_relation,
- _cache_spark_dataset,
)
-from pyspark.ml.wrapper import JavaParams, JavaPredictor, JavaPredictionModel, JavaWrapper
-from pyspark.ml.common import inherit_doc
-from pyspark.ml.linalg import Matrix, Vector
-from pyspark.sql import DataFrame, Row, SparkSession, functions as F
+from pyspark.ml.wrapper import JavaParams, JavaPredictionModel, JavaPredictor, JavaWrapper
+from pyspark.sql import DataFrame, Row, SparkSession
+from pyspark.sql import functions as F
from pyspark.sql.internal import InternalFunction as SF
-from pyspark.storagelevel import StorageLevel
from pyspark.sql.utils import is_remote
+from pyspark.storagelevel import StorageLevel
if TYPE_CHECKING:
- from pyspark.ml._typing import P, ParamMap
from py4j.java_gateway import JavaObject
+
from pyspark.core.context import SparkContext
+ from pyspark.ml._typing import P, ParamMap
T = TypeVar("T")
@@ -4352,6 +4354,7 @@ class FMClassificationTrainingSummary(FMClassificationSummary, _TrainingSummary)
if __name__ == "__main__":
import doctest
+
import pyspark.ml.classification
from pyspark.sql import SparkSession
diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py
index ba3e486cb4ef..525b3e7b3baa 100644
--- a/python/pyspark/ml/clustering.py
+++ b/python/pyspark/ml/clustering.py
@@ -15,51 +15,52 @@
# limitations under the License.
#
+import functools
import sys
import warnings
-from typing import Any, Dict, List, Optional, TYPE_CHECKING
-import functools
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
import numpy as np
-from pyspark import since, keyword_only
+from pyspark import keyword_only, since
+from pyspark.ml.common import inherit_doc
+from pyspark.ml.linalg import Matrix, Vector
from pyspark.ml.param.shared import (
- HasMaxIter,
+ HasAggregationDepth,
+ HasCheckpointInterval,
+ HasDistanceMeasure,
HasFeaturesCol,
- HasSeed,
+ HasIntermediateStorageLevel,
+ HasMaxBlockSizeInMB,
+ HasMaxIter,
HasPredictionCol,
- HasAggregationDepth,
- HasWeightCol,
- HasTol,
HasProbabilityCol,
- HasDistanceMeasure,
- HasCheckpointInterval,
+ HasSeed,
HasSolver,
- HasMaxBlockSizeInMB,
- HasIntermediateStorageLevel,
+ HasTol,
+ HasWeightCol,
Param,
Params,
TypeConverters,
)
+from pyspark.ml.stat import MultivariateGaussian
from pyspark.ml.util import (
- JavaMLWritable,
- JavaMLReadable,
GeneralJavaMLWritable,
HasTrainingSummary,
- try_remote_attribute_relation,
+ JavaMLReadable,
+ JavaMLWritable,
invoke_helper_relation,
+ try_remote_attribute_relation,
)
from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaParams, JavaWrapper
-from pyspark.ml.common import inherit_doc
-from pyspark.ml.stat import MultivariateGaussian
from pyspark.sql import DataFrame
-from pyspark.ml.linalg import Vector, Matrix
from pyspark.sql.utils import is_remote
if TYPE_CHECKING:
- from pyspark.ml._typing import M
from py4j.java_gateway import JavaObject
+ from pyspark.ml._typing import M
+
__all__ = [
"BisectingKMeans",
@@ -2180,7 +2181,9 @@ class PowerIterationClustering(
if __name__ == "__main__":
import doctest
+
import numpy
+
import pyspark.ml.clustering
from pyspark.sql import SparkSession
diff --git a/python/pyspark/ml/common.py b/python/pyspark/ml/common.py
index 2417df6ab9eb..a0a4b02ee13a 100644
--- a/python/pyspark/ml/common.py
+++ b/python/pyspark/ml/common.py
@@ -15,19 +15,19 @@
# limitations under the License.
#
-from typing import Any, Callable, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Callable
-from pyspark.util import is_remote_only
-from pyspark.serializers import CPickleSerializer, AutoBatchedSerializer
+from pyspark.serializers import AutoBatchedSerializer, CPickleSerializer
from pyspark.sql import DataFrame, SparkSession
+from pyspark.util import is_remote_only
if TYPE_CHECKING:
import py4j.protocol
from py4j.java_gateway import JavaObject
import pyspark.core.context
- from pyspark.core.rdd import RDD
from pyspark.core.context import SparkContext
+ from pyspark.core.rdd import RDD
from pyspark.ml._typing import C, JavaObjectOrPickleDump
@@ -80,8 +80,9 @@ def _to_java_object_rdd(rdd: "RDD") -> "JavaObject":
def _py2java(sc: "SparkContext", obj: Any) -> "JavaObject":
"""Convert Python object into Java"""
from py4j.java_gateway import JavaObject
- from pyspark.core.rdd import RDD
+
from pyspark.core.context import SparkContext
+ from pyspark.core.rdd import RDD
if isinstance(obj, RDD):
obj = _to_java_object_rdd(obj)
@@ -103,9 +104,9 @@ def _py2java(sc: "SparkContext", obj: Any) -> "JavaObject":
def _java2py(sc: "SparkContext", r: "JavaObjectOrPickleDump", encoding: str = "bytes") -> Any:
- from py4j.protocol import Py4JJavaError
- from py4j.java_gateway import JavaObject
from py4j.java_collections import JavaArray, JavaList
+ from py4j.java_gateway import JavaObject
+ from py4j.protocol import Py4JJavaError
if isinstance(r, JavaObject):
clsName = r.getClass().getSimpleName()
diff --git a/python/pyspark/ml/connect/__init__.py b/python/pyspark/ml/connect/__init__.py
index c4bc8c9d84d2..0ce207e5d865 100644
--- a/python/pyspark/ml/connect/__init__.py
+++ b/python/pyspark/ml/connect/__init__.py
@@ -21,16 +21,16 @@ from pyspark.sql.connect.utils import check_dependencies
check_dependencies()
-from pyspark.ml.connect.base import (
- Estimator,
- Transformer,
- Model,
-)
from pyspark.ml.connect import (
- feature,
evaluation,
+ feature,
tuning,
)
+from pyspark.ml.connect.base import (
+ Estimator,
+ Model,
+ Transformer,
+)
from pyspark.ml.connect.evaluation import Evaluator
from pyspark.ml.connect.pipeline import Pipeline, PipelineModel
diff --git a/python/pyspark/ml/connect/base.py b/python/pyspark/ml/connect/base.py
index 92fe1ae4e32c..3497aa9c87fa 100644
--- a/python/pyspark/ml/connect/base.py
+++ b/python/pyspark/ml/connect/base.py
@@ -17,28 +17,28 @@
from abc import ABCMeta, abstractmethod
from typing import (
+ TYPE_CHECKING,
Any,
+ Callable,
Generic,
List,
Optional,
+ Tuple,
TypeVar,
Union,
- TYPE_CHECKING,
- Tuple,
- Callable,
)
import pandas as pd
from pyspark import since
from pyspark.ml.common import inherit_doc
-from pyspark.sql.dataframe import DataFrame
from pyspark.ml.param import Params
from pyspark.ml.param.shared import (
- HasLabelCol,
HasFeaturesCol,
+ HasLabelCol,
HasPredictionCol,
)
+from pyspark.sql.dataframe import DataFrame
if TYPE_CHECKING:
from pyspark.ml._typing import ParamMap
diff --git a/python/pyspark/ml/connect/classification.py b/python/pyspark/ml/connect/classification.py
index 3263f47e6135..71efe29cdd30 100644
--- a/python/pyspark/ml/connect/classification.py
+++ b/python/pyspark/ml/connect/classification.py
@@ -14,31 +14,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from typing import Any, Dict, Union, List, Tuple, Callable, Optional
import math
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
import numpy as np
import pandas as pd
from pyspark import keyword_only
-from pyspark.ml.connect.base import _PredictorParams
-from pyspark.ml.param.shared import HasProbabilityCol
-from pyspark.sql import DataFrame
from pyspark.ml.common import inherit_doc
-from pyspark.ml.torch.distributor import TorchDistributor
+from pyspark.ml.connect.base import PredictionModel, Predictor, _PredictorParams
+from pyspark.ml.connect.io_utils import CoreModelReadWrite, ParamsReadWrite
from pyspark.ml.param.shared import (
- HasMaxIter,
- HasFitIntercept,
- HasTol,
- HasWeightCol,
- HasSeed,
- HasNumTrainWorkers,
HasBatchSize,
+ HasFitIntercept,
HasLearningRate,
+ HasMaxIter,
HasMomentum,
+ HasNumTrainWorkers,
+ HasProbabilityCol,
+ HasSeed,
+ HasTol,
+ HasWeightCol,
)
-from pyspark.ml.connect.base import Predictor, PredictionModel
-from pyspark.ml.connect.io_utils import ParamsReadWrite, CoreModelReadWrite
+from pyspark.ml.torch.distributor import TorchDistributor
+from pyspark.sql import DataFrame
from pyspark.sql import functions as sf
@@ -86,12 +85,13 @@ def _train_logistic_regression_model_worker_fn(
fit_intercept: bool,
seed: int,
) -> Any:
- from pyspark.ml.torch.distributor import _get_spark_partition_data_loader
import torch
- import torch.nn as torch_nn
- from torch.nn.parallel import DistributedDataParallel as DDP
import torch.distributed
+ import torch.nn as torch_nn
import torch.optim as optim
+ from torch.nn.parallel import DistributedDataParallel as DDP
+
+ from pyspark.ml.torch.distributor import _get_spark_partition_data_loader
# TODO: add a setting seed param.
torch.manual_seed(seed)
diff --git a/python/pyspark/ml/connect/evaluation.py b/python/pyspark/ml/connect/evaluation.py
index f324bb193c0c..e4d4acb9cf60 100644
--- a/python/pyspark/ml/connect/evaluation.py
+++ b/python/pyspark/ml/connect/evaluation.py
@@ -14,16 +14,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from typing import Any, Union, List, Tuple
+from typing import Any, List, Tuple, Union
import numpy as np
import pandas as pd
from pyspark import keyword_only
-from pyspark.ml.param import Param, Params, TypeConverters
-from pyspark.ml.param.shared import HasLabelCol, HasPredictionCol, HasProbabilityCol
from pyspark.ml.connect.base import Evaluator
from pyspark.ml.connect.io_utils import ParamsReadWrite
+from pyspark.ml.param import Param, Params, TypeConverters
+from pyspark.ml.param.shared import HasLabelCol, HasPredictionCol, HasProbabilityCol
from pyspark.sql import DataFrame
diff --git a/python/pyspark/ml/connect/feature.py b/python/pyspark/ml/connect/feature.py
index 2184b3c7f332..e018522098b6 100644
--- a/python/pyspark/ml/connect/feature.py
+++ b/python/pyspark/ml/connect/feature.py
@@ -15,26 +15,26 @@
# limitations under the License.
#
-from typing import Any, Union, List, Tuple, Callable, Dict, Optional
+from typing import Any, Callable, Dict, List, Optional, Tuple, Union
import numpy as np
import pandas as pd
import pyarrow as pa
from pyspark import keyword_only
-from pyspark.sql import DataFrame
+from pyspark.ml.connect.base import Estimator, Model, Transformer
+from pyspark.ml.connect.io_utils import CoreModelReadWrite, ParamsReadWrite
from pyspark.ml.param.shared import (
+ HasFeatureSizes,
+ HasHandleInvalid,
HasInputCol,
HasInputCols,
HasOutputCol,
- HasFeatureSizes,
- HasHandleInvalid,
Param,
Params,
TypeConverters,
)
-from pyspark.ml.connect.base import Estimator, Model, Transformer
-from pyspark.ml.connect.io_utils import ParamsReadWrite, CoreModelReadWrite
+from pyspark.sql import DataFrame
class MaxAbsScaler(Estimator, HasInputCol, HasOutputCol, ParamsReadWrite):
diff --git a/python/pyspark/ml/connect/functions.py b/python/pyspark/ml/connect/functions.py
index 5f2738cb2b3e..3a66253fd357 100644
--- a/python/pyspark/ml/connect/functions.py
+++ b/python/pyspark/ml/connect/functions.py
@@ -15,7 +15,7 @@
# limitations under the License.
#
-from typing import Any, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
from pyspark.ml import functions as PyMLFunctions
from pyspark.sql.column import Column
@@ -64,8 +64,9 @@ def _test() -> None:
sys.exit(0)
import doctest
- from pyspark.sql import SparkSession as PySparkSession
+
import pyspark.ml.connect.functions
+ from pyspark.sql import SparkSession as PySparkSession
globs = pyspark.ml.connect.functions.__dict__.copy()
diff --git a/python/pyspark/ml/connect/io_utils.py b/python/pyspark/ml/connect/io_utils.py
index f6108934f399..c6104d12d094 100644
--- a/python/pyspark/ml/connect/io_utils.py
+++ b/python/pyspark/ml/connect/io_utils.py
@@ -16,17 +16,17 @@
#
import json
-import shutil
import os
+import shutil
import tempfile
import time
-from urllib.parse import urlparse
from typing import Any, Dict, List
+from urllib.parse import urlparse
+from pyspark import __version__ as pyspark_version
from pyspark.ml.base import Params
from pyspark.sql import SparkSession
from pyspark.sql.utils import is_remote
-from pyspark import __version__ as pyspark_version
_META_DATA_FILE_NAME = "metadata.json"
diff --git a/python/pyspark/ml/connect/pipeline.py b/python/pyspark/ml/connect/pipeline.py
index 55e57b88da04..029dc40c9b4b 100644
--- a/python/pyspark/ml/connect/pipeline.py
+++ b/python/pyspark/ml/connect/pipeline.py
@@ -14,18 +14,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from typing import Any, Dict, List, Optional, Union, cast, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union, cast
import pandas as pd
from pyspark import keyword_only, since
+from pyspark.ml.common import inherit_doc
from pyspark.ml.connect.base import Estimator, Model, Transformer
from pyspark.ml.connect.io_utils import (
- ParamsReadWrite,
MetaAlgorithmReadWrite,
+ ParamsReadWrite,
)
from pyspark.ml.param import Param, Params
-from pyspark.ml.common import inherit_doc
from pyspark.sql.dataframe import DataFrame
if TYPE_CHECKING:
diff --git a/python/pyspark/ml/connect/proto.py b/python/pyspark/ml/connect/proto.py
index eecf971440fb..6d274b30390b 100644
--- a/python/pyspark/ml/connect/proto.py
+++ b/python/pyspark/ml/connect/proto.py
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from typing import Optional, TYPE_CHECKING, List
+from typing import TYPE_CHECKING, List, Optional
import pyspark.sql.connect.proto as pb2
from pyspark.sql.connect.plan import LogicalPlan
diff --git a/python/pyspark/ml/connect/readwrite.py b/python/pyspark/ml/connect/readwrite.py
index 08802a0fceae..8f95f55352fd 100644
--- a/python/pyspark/ml/connect/readwrite.py
+++ b/python/pyspark/ml/connect/readwrite.py
@@ -14,17 +14,24 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from typing import cast, Type, TYPE_CHECKING, Union, Dict, Any
+
+# TODO: The type design of RemoteMLWriter is not very good. In saveInstance,
+# we try to call session() with a connect session, but the session() method
+# accepts a classic session. What's make it worse is that mypy considers some
+# of the branches as unreachable. We need a full re-design of this module.
+# mypy: disable-error-code="arg-type"
+
+from typing import TYPE_CHECKING, Any, Dict, Type, Union, cast
import pyspark.sql.connect.proto as pb2
-from pyspark.ml.connect.serialize import serialize_ml_params, deserialize, deserialize_param
-from pyspark.ml.util import MLWriter, MLReader, RL
+from pyspark.ml.connect.serialize import deserialize, deserialize_param, serialize_ml_params
+from pyspark.ml.util import RL, MLReader, MLWriter
from pyspark.ml.wrapper import JavaWrapper
if TYPE_CHECKING:
from pyspark.core.context import SparkContext
- from pyspark.sql.connect.session import SparkSession
from pyspark.ml.util import JavaMLReadable, JavaMLWritable
+ from pyspark.sql.connect.session import SparkSession
class RemoteMLWriter(MLWriter):
@@ -58,17 +65,17 @@ class RemoteMLWriter(MLWriter):
shouldOverwrite: bool = False,
optionMap: Dict[str, Any] = {},
) -> None:
- from pyspark.ml.wrapper import JavaModel, JavaEstimator, JavaTransformer
- from pyspark.ml.evaluation import JavaEvaluator
- from pyspark.ml.pipeline import Pipeline, PipelineModel
from pyspark.ml.classification import OneVsRest, OneVsRestModel
from pyspark.ml.clustering import PowerIterationClustering
+ from pyspark.ml.evaluation import JavaEvaluator
+ from pyspark.ml.pipeline import Pipeline, PipelineModel
from pyspark.ml.tuning import (
CrossValidator,
CrossValidatorModel,
TrainValidationSplit,
TrainValidationSplitModel,
)
+ from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaTransformer
# Spark Connect ML is built on scala Spark.ML, that means we're only
# supporting JavaModel or JavaEstimator or JavaEvaluator
@@ -119,21 +126,21 @@ class RemoteMLWriter(MLWriter):
RemoteMLWriter.handleOverwrite(path, shouldOverwrite)
pl_writer = PipelineWriter(instance)
- pl_writer.session(session) # type: ignore[arg-type]
+ pl_writer.session(session)
pl_writer.save(path)
elif isinstance(instance, PipelineModel):
from pyspark.ml.pipeline import PipelineModelWriter
RemoteMLWriter.handleOverwrite(path, shouldOverwrite)
plm_writer = PipelineModelWriter(instance)
- plm_writer.session(session) # type: ignore[arg-type]
+ plm_writer.session(session)
plm_writer.save(path)
elif isinstance(instance, CrossValidator):
from pyspark.ml.tuning import CrossValidatorWriter
RemoteMLWriter.handleOverwrite(path, shouldOverwrite)
cv_writer = CrossValidatorWriter(instance)
- cv_writer.session(session) # type: ignore[arg-type]
+ cv_writer.session(session)
cv_writer.save(path)
elif isinstance(instance, CrossValidatorModel):
from pyspark.ml.tuning import CrossValidatorModelWriter
@@ -141,7 +148,7 @@ class RemoteMLWriter(MLWriter):
RemoteMLWriter.handleOverwrite(path, shouldOverwrite)
cvm_writer = CrossValidatorModelWriter(instance)
cvm_writer.optionMap = optionMap
- cvm_writer.session(session) # type: ignore[arg-type]
+ cvm_writer.session(session)
cvm_writer.save(path)
elif isinstance(instance, TrainValidationSplit):
from pyspark.ml.tuning import TrainValidationSplitWriter
@@ -155,21 +162,21 @@ class RemoteMLWriter(MLWriter):
RemoteMLWriter.handleOverwrite(path, shouldOverwrite)
tvsm_writer = TrainValidationSplitModelWriter(instance)
tvsm_writer.optionMap = optionMap
- tvsm_writer.session(session) # type: ignore[arg-type]
+ tvsm_writer.session(session)
tvsm_writer.save(path)
elif isinstance(instance, OneVsRest):
from pyspark.ml.classification import OneVsRestWriter
RemoteMLWriter.handleOverwrite(path, shouldOverwrite)
ovr_writer = OneVsRestWriter(instance)
- ovr_writer.session(session) # type: ignore[arg-type]
+ ovr_writer.session(session)
ovr_writer.save(path)
elif isinstance(instance, OneVsRestModel):
from pyspark.ml.classification import OneVsRestModelWriter
RemoteMLWriter.handleOverwrite(path, shouldOverwrite)
ovrm_writer = OneVsRestModelWriter(instance)
- ovrm_writer.session(session) # type: ignore[arg-type]
+ ovrm_writer.session(session)
ovrm_writer.save(path)
elif isinstance(instance, PowerIterationClustering):
@@ -179,7 +186,7 @@ class RemoteMLWriter(MLWriter):
transformer._resetUid(instance.uid)
transformer._paramMap = instance._paramMap
RemoteMLWriter.saveInstance(
- transformer, # type: ignore[arg-type]
+ transformer,
path,
session,
shouldOverwrite,
@@ -217,17 +224,17 @@ class RemoteMLReader(MLReader[RL]):
path: str,
session: "SparkSession",
) -> RL:
- from pyspark.ml.wrapper import JavaModel, JavaEstimator, JavaTransformer
- from pyspark.ml.evaluation import JavaEvaluator
- from pyspark.ml.pipeline import Pipeline, PipelineModel
from pyspark.ml.classification import OneVsRest, OneVsRestModel
from pyspark.ml.clustering import PowerIterationClustering
+ from pyspark.ml.evaluation import JavaEvaluator
+ from pyspark.ml.pipeline import Pipeline, PipelineModel
from pyspark.ml.tuning import (
CrossValidator,
CrossValidatorModel,
TrainValidationSplit,
TrainValidationSplitModel,
)
+ from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaTransformer
if (
issubclass(clazz, JavaModel)
diff --git a/python/pyspark/ml/connect/serialize.py b/python/pyspark/ml/connect/serialize.py
index 42bedfb330b1..edd95d589898 100644
--- a/python/pyspark/ml/connect/serialize.py
+++ b/python/pyspark/ml/connect/serialize.py
@@ -14,20 +14,20 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from typing import Any, List, TYPE_CHECKING, Mapping, Dict
+from typing import TYPE_CHECKING, Any, Dict, List, Mapping
import pyspark.sql.connect.proto as pb2
-from pyspark.sql.types import DataType
from pyspark.ml.linalg import (
- DenseVector,
- SparseVector,
DenseMatrix,
+ DenseVector,
SparseMatrix,
+ SparseVector,
)
+from pyspark.sql.types import DataType
if TYPE_CHECKING:
- from pyspark.sql.connect.client import SparkConnectClient
from pyspark.ml.param import Params
+ from pyspark.sql.connect.client import SparkConnectClient
def literal_null() -> pb2.Expression.Literal:
diff --git a/python/pyspark/ml/connect/summarizer.py b/python/pyspark/ml/connect/summarizer.py
index 2f3e16ef4d18..92f3aaed438e 100644
--- a/python/pyspark/ml/connect/summarizer.py
+++ b/python/pyspark/ml/connect/summarizer.py
@@ -15,13 +15,13 @@
# limitations under the License.
#
-from typing import Any, Union, List, Dict
+from typing import Any, Dict, List, Union
import numpy as np
import pandas as pd
-from pyspark.sql import DataFrame
from pyspark.ml.connect.util import aggregate_dataframe
+from pyspark.sql import DataFrame
class SummarizerAggState:
diff --git a/python/pyspark/ml/connect/tuning.py b/python/pyspark/ml/connect/tuning.py
index 9d6a008c99e2..d31e0fdad1df 100644
--- a/python/pyspark/ml/connect/tuning.py
+++ b/python/pyspark/ml/connect/tuning.py
@@ -17,6 +17,7 @@
from multiprocessing.pool import ThreadPool
from typing import (
+ TYPE_CHECKING,
Any,
Callable,
Dict,
@@ -26,24 +27,22 @@ from typing import (
Tuple,
Union,
cast,
- TYPE_CHECKING,
)
import numpy as np
import pandas as pd
-from pyspark import keyword_only, since, inheritable_thread_target
-from pyspark.ml.connect import Estimator, Model
-from pyspark.ml.connect.base import Evaluator
+from pyspark import inheritable_thread_target, keyword_only, since
+from pyspark.ml.connect.base import Estimator, Evaluator, Model
from pyspark.ml.connect.io_utils import (
MetaAlgorithmReadWrite,
ParamsReadWrite,
)
-from pyspark.ml.param import Params, Param, TypeConverters
+from pyspark.ml.param import Param, Params, TypeConverters
from pyspark.ml.param.shared import HasParallelism, HasSeed
-from pyspark.sql.functions import col, lit, rand
-from pyspark.sql.dataframe import DataFrame
from pyspark.sql import SparkSession
+from pyspark.sql.dataframe import DataFrame
+from pyspark.sql.functions import col, lit, rand
from pyspark.sql.utils import is_remote
if TYPE_CHECKING:
diff --git a/python/pyspark/ml/connect/util.py b/python/pyspark/ml/connect/util.py
index 17b072ff57ee..4f22ba494820 100644
--- a/python/pyspark/ml/connect/util.py
+++ b/python/pyspark/ml/connect/util.py
@@ -15,7 +15,7 @@
# limitations under the License.
#
-from typing import Any, TypeVar, Callable, List, Tuple, Union, Iterator, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Callable, Iterator, List, Tuple, TypeVar, Union
import pandas as pd
diff --git a/python/pyspark/ml/deepspeed/deepspeed_distributor.py b/python/pyspark/ml/deepspeed/deepspeed_distributor.py
index 162cd2adda52..cbfbf3702027 100644
--- a/python/pyspark/ml/deepspeed/deepspeed_distributor.py
+++ b/python/pyspark/ml/deepspeed/deepspeed_distributor.py
@@ -18,12 +18,12 @@ import json
import sys
import tempfile
from typing import (
- Union,
+ Any,
Callable,
- List,
Dict,
+ List,
Optional,
- Any,
+ Union,
)
from pyspark.ml.torch.distributor import TorchDistributor
diff --git a/python/pyspark/ml/evaluation.py b/python/pyspark/ml/evaluation.py
index 801d23db4eef..4b1ed21c3f72 100644
--- a/python/pyspark/ml/evaluation.py
+++ b/python/pyspark/ml/evaluation.py
@@ -16,32 +16,32 @@
#
import sys
-from abc import abstractmethod, ABCMeta
-from typing import Any, Dict, Optional, TYPE_CHECKING
+from abc import ABCMeta, abstractmethod
+from typing import TYPE_CHECKING, Any, Dict, Optional
-from pyspark import since, keyword_only
-from pyspark.ml.wrapper import JavaParams
+from pyspark import keyword_only, since
+from pyspark.ml.common import inherit_doc
from pyspark.ml.param import Param, Params, TypeConverters
from pyspark.ml.param.shared import (
+ HasFeaturesCol,
HasLabelCol,
HasPredictionCol,
HasProbabilityCol,
HasRawPredictionCol,
- HasFeaturesCol,
HasWeightCol,
)
-from pyspark.ml.common import inherit_doc
from pyspark.ml.util import JavaMLReadable, JavaMLWritable, try_remote_evaluate
+from pyspark.ml.wrapper import JavaParams
from pyspark.sql.dataframe import DataFrame
if TYPE_CHECKING:
from pyspark.ml._typing import (
- ParamMap,
BinaryClassificationEvaluatorMetricType,
ClusteringEvaluatorDistanceMeasureType,
ClusteringEvaluatorMetricType,
MulticlassClassificationEvaluatorMetricType,
MultilabelClassificationEvaluatorMetricType,
+ ParamMap,
RankingEvaluatorMetricType,
RegressionEvaluatorMetricType,
)
@@ -1171,6 +1171,7 @@ class RankingEvaluator(
if __name__ == "__main__":
import doctest
import tempfile
+
import pyspark.ml.evaluation
from pyspark.sql import SparkSession
diff --git a/python/pyspark/ml/feature.py b/python/pyspark/ml/feature.py
index 4024c0c07f4f..4167249afaa8 100755
--- a/python/pyspark/ml/feature.py
+++ b/python/pyspark/ml/feature.py
@@ -15,8 +15,7 @@
# limitations under the License.
#
from typing import (
- cast,
- overload,
+ TYPE_CHECKING,
Any,
Dict,
Generic,
@@ -25,36 +24,38 @@ from typing import (
Tuple,
TypeVar,
Union,
- TYPE_CHECKING,
+ cast,
+ overload,
)
from pyspark import keyword_only, since
-from pyspark.ml.linalg import _convert_to_vector, DenseMatrix, DenseVector, Vector
-from pyspark.sql.dataframe import DataFrame
+from pyspark.ml.common import inherit_doc
+from pyspark.ml.linalg import DenseMatrix, DenseVector, Vector, _convert_to_vector
from pyspark.ml.param.shared import (
- HasThreshold,
- HasThresholds,
+ HasFeaturesCol,
+ HasHandleInvalid,
HasInputCol,
- HasOutputCol,
HasInputCols,
+ HasLabelCol,
+ HasMaxIter,
+ HasNumFeatures,
+ HasOutputCol,
HasOutputCols,
- HasHandleInvalid,
HasRelativeError,
- HasFeaturesCol,
- HasLabelCol,
HasSeed,
- HasNumFeatures,
HasStepSize,
- HasMaxIter,
- TypeConverters,
+ HasThreshold,
+ HasThresholds,
Param,
Params,
+ TypeConverters,
)
from pyspark.ml.util import (
JavaMLReadable,
JavaMLWritable,
- try_remote_attribute_relation,
+ RemoteModelRef,
invoke_helper_attr,
+ try_remote_attribute_relation,
)
from pyspark.ml.wrapper import (
JavaEstimator,
@@ -63,8 +64,7 @@ from pyspark.ml.wrapper import (
JavaTransformer,
_jvm,
)
-from pyspark.ml.common import inherit_doc
-from pyspark.ml.util import RemoteModelRef
+from pyspark.sql.dataframe import DataFrame
from pyspark.sql.types import ArrayType, StringType
from pyspark.sql.utils import is_remote
diff --git a/python/pyspark/ml/fpm.py b/python/pyspark/ml/fpm.py
index 7ad28f69b7c7..fc250db37fe0 100644
--- a/python/pyspark/ml/fpm.py
+++ b/python/pyspark/ml/fpm.py
@@ -16,18 +16,18 @@
#
import sys
-from typing import Any, Dict, Optional, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Dict, Optional
from pyspark import keyword_only, since
-from pyspark.sql import DataFrame
+from pyspark.ml.param.shared import HasPredictionCol, Param, Params, TypeConverters
from pyspark.ml.util import (
- JavaMLWritable,
JavaMLReadable,
- try_remote_attribute_relation,
+ JavaMLWritable,
invoke_helper_relation,
+ try_remote_attribute_relation,
)
from pyspark.ml.wrapper import JavaEstimator, JavaModel, JavaParams
-from pyspark.ml.param.shared import HasPredictionCol, Param, TypeConverters, Params
+from pyspark.sql import DataFrame
if TYPE_CHECKING:
from py4j.java_gateway import JavaObject
@@ -533,6 +533,7 @@ class PrefixSpan(JavaParams):
if __name__ == "__main__":
import doctest
+
import pyspark.ml.fpm
from pyspark.sql import SparkSession
diff --git a/python/pyspark/ml/functions.py b/python/pyspark/ml/functions.py
index b3cd3978df44..51a735f0862f 100644
--- a/python/pyspark/ml/functions.py
+++ b/python/pyspark/ml/functions.py
@@ -18,7 +18,7 @@ from __future__ import annotations
import inspect
import uuid
-from typing import Any, Callable, Iterator, List, Mapping, TYPE_CHECKING, Tuple, Union, Optional
+from typing import TYPE_CHECKING, Any, Callable, Iterator, List, Mapping, Optional, Tuple, Union
import numpy as np
@@ -27,8 +27,9 @@ try:
except ImportError:
pass # Let it throw a better error message later when the API is invoked.
-from pyspark.sql.functions import pandas_udf
+from pyspark.ml.util import try_remote_functions
from pyspark.sql.column import Column
+from pyspark.sql.functions import pandas_udf
from pyspark.sql.types import (
ArrayType,
ByteType,
@@ -41,7 +42,6 @@ from pyspark.sql.types import (
StringType,
StructType,
)
-from pyspark.ml.util import try_remote_functions
if TYPE_CHECKING:
from pyspark.sql._typing import UserDefinedFunctionLike
@@ -833,10 +833,10 @@ def predict_batch_udf(
def _test() -> None:
import doctest
- from pyspark.sql import SparkSession
- import pyspark.ml.functions
import sys
+ import pyspark.ml.functions
+ from pyspark.sql import SparkSession
from pyspark.sql.pandas.utils import (
require_minimum_pandas_version,
require_minimum_pyarrow_version,
diff --git a/python/pyspark/ml/image.py b/python/pyspark/ml/image.py
index d4f59d2d0273..713af8b58b3a 100644
--- a/python/pyspark/ml/image.py
+++ b/python/pyspark/ml/image.py
@@ -25,13 +25,13 @@
"""
import sys
-from typing import Any, Dict, List, NoReturn, cast
from functools import cached_property
+from typing import Any, Dict, List, NoReturn, cast
import numpy as np
-from pyspark.sql.types import Row, StructType, _create_row, _parse_datatype_json_string
from pyspark.sql import SparkSession
+from pyspark.sql.types import Row, StructType, _create_row, _parse_datatype_json_string
__all__ = ["ImageSchema"]
@@ -231,6 +231,7 @@ _ImageSchema.__init__ = _disallow_instance # type: ignore[assignment]
def _test() -> None:
import doctest
+
import pyspark.ml.image
globs = pyspark.ml.image.__dict__.copy()
diff --git a/python/pyspark/ml/linalg/__init__.py b/python/pyspark/ml/linalg/__init__.py
index 4a07edbadf40..a1d7278bd2e6 100644
--- a/python/pyspark/ml/linalg/__init__.py
+++ b/python/pyspark/ml/linalg/__init__.py
@@ -23,37 +23,37 @@ object from MLlib or pass SciPy `scipy.sparse` column vectors if
SciPy is available in their environment.
"""
-import sys
import array
import struct
+import sys
from typing import (
+ TYPE_CHECKING,
Any,
Callable,
- cast,
Dict,
Iterable,
List,
Optional,
- overload,
Sequence,
Tuple,
Type,
- TYPE_CHECKING,
Union,
+ cast,
+ overload,
)
import numpy as np
from pyspark.sql.types import (
- DataTypeSingleton,
- UserDefinedType,
- StructField,
- StructType,
ArrayType,
+ BooleanType,
+ ByteType,
+ DataTypeSingleton,
DoubleType,
IntegerType,
- ByteType,
- BooleanType,
+ StructField,
+ StructType,
+ UserDefinedType,
)
__all__ = [
@@ -68,8 +68,8 @@ __all__ = [
]
if TYPE_CHECKING:
- from pyspark.mllib._typing import NormType
from pyspark.ml._typing import VectorLike
+ from pyspark.mllib._typing import NormType
# Check whether we have SciPy. MLlib works without it too, but if we have it, some methods,
diff --git a/python/pyspark/ml/param/__init__.py b/python/pyspark/ml/param/__init__.py
index cf0da7dce827..db4f3a5bbb17 100644
--- a/python/pyspark/ml/param/__init__.py
+++ b/python/pyspark/ml/param/__init__.py
@@ -15,25 +15,25 @@
# limitations under the License.
#
import array
-from abc import ABCMeta
import copy
+from abc import ABCMeta
from typing import (
+ TYPE_CHECKING,
Any,
Callable,
Generic,
List,
Optional,
- overload,
TypeVar,
Union,
- TYPE_CHECKING,
+ overload,
)
import numpy as np
-from pyspark.util import is_remote_only
-from pyspark.ml.linalg import DenseVector, Vector, Matrix
+from pyspark.ml.linalg import DenseVector, Matrix, Vector
from pyspark.ml.util import Identifiable
+from pyspark.util import is_remote_only
if TYPE_CHECKING:
from pyspark.ml._typing import ParamMap
diff --git a/python/pyspark/ml/pipeline.py b/python/pyspark/ml/pipeline.py
index f5f0d12bc836..e4facbcc7d5d 100644
--- a/python/pyspark/ml/pipeline.py
+++ b/python/pyspark/ml/pipeline.py
@@ -15,33 +15,33 @@
# limitations under the License.
#
import os
-
-from typing import Any, Dict, List, Optional, Tuple, Type, Union, cast, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Type, Union, cast
from pyspark import keyword_only, since
from pyspark.ml.base import Estimator, Model, Transformer
+from pyspark.ml.common import inherit_doc
from pyspark.ml.param import Param, Params
from pyspark.ml.util import (
- MLReadable,
- MLWritable,
- JavaMLWriter,
DefaultParamsReader,
DefaultParamsWriter,
- MLWriter,
- MLReader,
JavaMLWritable,
+ JavaMLWriter,
+ MLReadable,
+ MLReader,
+ MLWritable,
+ MLWriter,
try_remote_read,
try_remote_write,
)
from pyspark.ml.wrapper import JavaParams
-from pyspark.ml.common import inherit_doc
from pyspark.sql import SparkSession
from pyspark.sql.dataframe import DataFrame
if TYPE_CHECKING:
- from pyspark.ml._typing import ParamMap, PipelineStage
from py4j.java_gateway import JavaObject
+
from pyspark.core.context import SparkContext
+ from pyspark.ml._typing import ParamMap, PipelineStage
@inherit_doc
diff --git a/python/pyspark/ml/recommendation.py b/python/pyspark/ml/recommendation.py
index ee4a0c8d2e96..273b37f83ab1 100644
--- a/python/pyspark/ml/recommendation.py
+++ b/python/pyspark/ml/recommendation.py
@@ -16,22 +16,22 @@
#
import sys
-from typing import Any, Dict, Optional, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Dict, Optional
-from pyspark import since, keyword_only
+from pyspark import keyword_only, since
+from pyspark.ml.common import inherit_doc
+from pyspark.ml.param import Param, Params, TypeConverters
from pyspark.ml.param.shared import (
- HasPredictionCol,
HasBlockSize,
+ HasCheckpointInterval,
+ HasIntermediateStorageLevel,
HasMaxIter,
+ HasPredictionCol,
HasRegParam,
- HasCheckpointInterval,
HasSeed,
- HasIntermediateStorageLevel,
)
+from pyspark.ml.util import JavaMLReadable, JavaMLWritable, try_remote_attribute_relation
from pyspark.ml.wrapper import JavaEstimator, JavaModel
-from pyspark.ml.common import inherit_doc
-from pyspark.ml.param import Params, TypeConverters, Param
-from pyspark.ml.util import JavaMLWritable, JavaMLReadable, try_remote_attribute_relation
from pyspark.sql import DataFrame
if TYPE_CHECKING:
@@ -720,6 +720,7 @@ class ALSModel(JavaModel, _ALSModelParams, JavaMLWritable, JavaMLReadable["ALSMo
if __name__ == "__main__":
import doctest
+
import pyspark.ml.recommendation
from pyspark.sql import SparkSession
diff --git a/python/pyspark/ml/regression.py b/python/pyspark/ml/regression.py
index d910db6c6d30..f3f780aa360c 100644
--- a/python/pyspark/ml/regression.py
+++ b/python/pyspark/ml/regression.py
@@ -16,61 +16,59 @@
#
import sys
-from typing import Any, Dict, Generic, List, Optional, TypeVar, TYPE_CHECKING
from abc import ABCMeta
from functools import cached_property
+from typing import TYPE_CHECKING, Any, Dict, Generic, List, Optional, TypeVar
from pyspark import keyword_only, since
-from pyspark.ml import Predictor, PredictionModel
-from pyspark.ml.base import _PredictorParams
+from pyspark.ml.base import PredictionModel, Predictor, Transformer, _PredictorParams
+from pyspark.ml.common import inherit_doc
+from pyspark.ml.linalg import Matrix, Vector
from pyspark.ml.param.shared import (
+ HasAggregationDepth,
+ HasElasticNetParam,
HasFeaturesCol,
- HasLabelCol,
- HasPredictionCol,
- HasWeightCol,
- Param,
- Params,
- TypeConverters,
- HasMaxIter,
- HasTol,
HasFitIntercept,
- HasAggregationDepth,
+ HasLabelCol,
+ HasLoss,
HasMaxBlockSizeInMB,
+ HasMaxIter,
+ HasPredictionCol,
HasRegParam,
- HasSolver,
- HasStepSize,
HasSeed,
- HasElasticNetParam,
+ HasSolver,
HasStandardization,
- HasLoss,
+ HasStepSize,
+ HasTol,
HasVarianceCol,
+ HasWeightCol,
+ Param,
+ Params,
+ TypeConverters,
)
-from pyspark.ml.util import try_remote_attribute_relation
from pyspark.ml.tree import (
_DecisionTreeModel,
_DecisionTreeParams,
- _TreeEnsembleModel,
- _RandomForestParams,
_GBTParams,
+ _RandomForestParams,
+ _TreeEnsembleModel,
_TreeRegressorParams,
)
-from pyspark.ml.base import Transformer
-from pyspark.ml.linalg import Vector, Matrix
from pyspark.ml.util import (
- JavaMLWritable,
- JavaMLReadable,
- HasTrainingSummary,
GeneralJavaMLWritable,
+ HasTrainingSummary,
+ JavaMLReadable,
+ JavaMLWritable,
+ try_remote_attribute_relation,
)
from pyspark.ml.wrapper import (
JavaEstimator,
JavaModel,
- JavaPredictor,
JavaPredictionModel,
+ JavaPredictor,
JavaTransformer,
JavaWrapper,
)
-from pyspark.ml.common import inherit_doc
from pyspark.sql import DataFrame
from pyspark.sql.utils import is_remote
@@ -3314,6 +3312,7 @@ class FMRegressionModel(
if __name__ == "__main__":
import doctest
+
import pyspark.ml.regression
from pyspark.sql import SparkSession
diff --git a/python/pyspark/ml/stat.py b/python/pyspark/ml/stat.py
index 99c4b80018cf..50c0fcb45682 100644
--- a/python/pyspark/ml/stat.py
+++ b/python/pyspark/ml/stat.py
@@ -16,13 +16,13 @@
#
import sys
-from typing import Optional, Tuple, TYPE_CHECKING
+from typing import TYPE_CHECKING, Optional, Tuple
from pyspark import since
from pyspark.ml.common import _java2py, _py2java
from pyspark.ml.linalg import Matrix, Vector
-from pyspark.ml.wrapper import JavaWrapper, _jvm
from pyspark.ml.util import invoke_helper_relation
+from pyspark.ml.wrapper import JavaWrapper, _jvm
from pyspark.sql.column import Column
from pyspark.sql.dataframe import DataFrame
from pyspark.sql.functions import lit
@@ -559,7 +559,9 @@ class MultivariateGaussian:
if __name__ == "__main__":
import doctest
+
import numpy
+
import pyspark.ml.stat
from pyspark.sql import SparkSession
diff --git a/python/pyspark/ml/tests/connect/test_connect_cache.py b/python/pyspark/ml/tests/connect/test_connect_cache.py
index ad3ee04e812e..23c17bd2ecbd 100644
--- a/python/pyspark/ml/tests/connect/test_connect_cache.py
+++ b/python/pyspark/ml/tests/connect/test_connect_cache.py
@@ -17,8 +17,8 @@
import json
-from pyspark.ml.linalg import Vectors
from pyspark.ml.classification import LinearSVC
+from pyspark.ml.linalg import Vectors
from pyspark.testing.connectutils import ReusedConnectTestCase
diff --git a/python/pyspark/ml/tests/connect/test_connect_classification.py b/python/pyspark/ml/tests/connect/test_connect_classification.py
index 7540c0e276a1..a173b49a3cff 100644
--- a/python/pyspark/ml/tests/connect/test_connect_classification.py
+++ b/python/pyspark/ml/tests/connect/test_connect_classification.py
@@ -15,14 +15,17 @@
# limitations under the License.
#
-import unittest
import os
+import unittest
-from pyspark.util import is_remote_only
from pyspark.ml.tests.connect.test_legacy_mode_classification import ClassificationTestsMixin
-from pyspark.testing.connectutils import should_test_connect, connect_requirement_message
+from pyspark.testing.connectutils import (
+ ReusedConnectTestCase,
+ connect_requirement_message,
+ should_test_connect,
+)
from pyspark.testing.utils import have_torch, torch_requirement_message
-from pyspark.testing.connectutils import ReusedConnectTestCase
+from pyspark.util import is_remote_only
@unittest.skipIf(
diff --git a/python/pyspark/ml/tests/connect/test_connect_evaluation.py b/python/pyspark/ml/tests/connect/test_connect_evaluation.py
index adf57ec842c2..693995cd8dbf 100644
--- a/python/pyspark/ml/tests/connect/test_connect_evaluation.py
+++ b/python/pyspark/ml/tests/connect/test_connect_evaluation.py
@@ -18,8 +18,7 @@
import os
import unittest
-from pyspark.testing.connectutils import should_test_connect
-from pyspark.testing.connectutils import ReusedConnectTestCase
+from pyspark.testing.connectutils import ReusedConnectTestCase, should_test_connect
if should_test_connect:
from pyspark.ml.tests.connect.test_legacy_mode_evaluation import EvaluationTestsMixin
diff --git a/python/pyspark/ml/tests/connect/test_connect_feature.py b/python/pyspark/ml/tests/connect/test_connect_feature.py
index 0d0109d02c8d..9ec9583a8caf 100644
--- a/python/pyspark/ml/tests/connect/test_connect_feature.py
+++ b/python/pyspark/ml/tests/connect/test_connect_feature.py
@@ -18,9 +18,12 @@
import os
import unittest
-from pyspark.testing.connectutils import should_test_connect, connect_requirement_message
+from pyspark.testing.connectutils import (
+ ReusedConnectTestCase,
+ connect_requirement_message,
+ should_test_connect,
+)
from pyspark.testing.utils import have_sklearn, sklearn_requirement_message
-from pyspark.testing.connectutils import ReusedConnectTestCase
if should_test_connect:
from pyspark.ml.tests.connect.test_legacy_mode_feature import FeatureTestsMixin
diff --git a/python/pyspark/ml/tests/connect/test_connect_function.py b/python/pyspark/ml/tests/connect/test_connect_function.py
index a18aa619249c..59df2bbfd13b 100644
--- a/python/pyspark/ml/tests/connect/test_connect_function.py
+++ b/python/pyspark/ml/tests/connect/test_connect_function.py
@@ -16,13 +16,13 @@
#
import unittest
-from pyspark.util import is_remote_only
from pyspark.ml import functions as SF
from pyspark.testing.connectutils import (
- should_test_connect,
ReusedMixedTestCase,
+ should_test_connect,
)
from pyspark.testing.pandasutils import PandasOnSparkTestUtils
+from pyspark.util import is_remote_only
if should_test_connect:
from pyspark.ml.connect import functions as CF
diff --git a/python/pyspark/ml/tests/connect/test_connect_model_offloading.py b/python/pyspark/ml/tests/connect/test_connect_model_offloading.py
index 9d78ab8e45da..ddc0c0ef869c 100644
--- a/python/pyspark/ml/tests/connect/test_connect_model_offloading.py
+++ b/python/pyspark/ml/tests/connect/test_connect_model_offloading.py
@@ -19,28 +19,28 @@ import json
import numpy as np
-from pyspark.sql import functions as sf
-from pyspark.ml.linalg import Vectors
from pyspark.ml.classification import (
LinearSVC,
LinearSVCSummary,
LinearSVCTrainingSummary,
)
-from pyspark.ml.regression import (
- LinearRegression,
- LinearRegressionSummary,
- LinearRegressionTrainingSummary,
-)
from pyspark.ml.clustering import (
LDA,
+ DistributedLDAModel,
LDAModel,
LocalLDAModel,
- DistributedLDAModel,
)
from pyspark.ml.fpm import (
FPGrowth,
FPGrowthModel,
)
+from pyspark.ml.linalg import Vectors
+from pyspark.ml.regression import (
+ LinearRegression,
+ LinearRegressionSummary,
+ LinearRegressionTrainingSummary,
+)
+from pyspark.sql import functions as sf
from pyspark.testing.connectutils import ReusedConnectTestCase
diff --git a/python/pyspark/ml/tests/connect/test_connect_pipeline.py b/python/pyspark/ml/tests/connect/test_connect_pipeline.py
index dcdc1de62d29..a98b6b3b6ff5 100644
--- a/python/pyspark/ml/tests/connect/test_connect_pipeline.py
+++ b/python/pyspark/ml/tests/connect/test_connect_pipeline.py
@@ -18,10 +18,13 @@
import os
import unittest
-from pyspark.util import is_remote_only
-from pyspark.testing.connectutils import should_test_connect, connect_requirement_message
+from pyspark.testing.connectutils import (
+ ReusedConnectTestCase,
+ connect_requirement_message,
+ should_test_connect,
+)
from pyspark.testing.utils import have_torch, torch_requirement_message
-from pyspark.testing.connectutils import ReusedConnectTestCase
+from pyspark.util import is_remote_only
if should_test_connect:
from pyspark.ml.tests.connect.test_legacy_mode_pipeline import PipelineTestsMixin
diff --git a/python/pyspark/ml/tests/connect/test_connect_summarizer.py b/python/pyspark/ml/tests/connect/test_connect_summarizer.py
index f68c18943bc4..2766a25bb0ed 100644
--- a/python/pyspark/ml/tests/connect/test_connect_summarizer.py
+++ b/python/pyspark/ml/tests/connect/test_connect_summarizer.py
@@ -18,8 +18,11 @@
import os
import unittest
-from pyspark.testing.connectutils import should_test_connect, connect_requirement_message
-from pyspark.testing.connectutils import ReusedConnectTestCase
+from pyspark.testing.connectutils import (
+ ReusedConnectTestCase,
+ connect_requirement_message,
+ should_test_connect,
+)
if should_test_connect:
from pyspark.ml.tests.connect.test_legacy_mode_summarizer import SummarizerTestsMixin
diff --git a/python/pyspark/ml/tests/connect/test_connect_tuning.py b/python/pyspark/ml/tests/connect/test_connect_tuning.py
index 21ada5443112..3a25c4e7af9c 100644
--- a/python/pyspark/ml/tests/connect/test_connect_tuning.py
+++ b/python/pyspark/ml/tests/connect/test_connect_tuning.py
@@ -18,10 +18,13 @@
import os
import unittest
-from pyspark.util import is_remote_only
-from pyspark.testing.connectutils import should_test_connect, connect_requirement_message
+from pyspark.testing.connectutils import (
+ ReusedConnectTestCase,
+ connect_requirement_message,
+ should_test_connect,
+)
from pyspark.testing.utils import have_torch, torch_requirement_message
-from pyspark.testing.connectutils import ReusedConnectTestCase
+from pyspark.util import is_remote_only
if should_test_connect:
from pyspark.ml.tests.connect.test_legacy_mode_tuning import CrossValidatorTestsMixin
diff --git a/python/pyspark/ml/tests/connect/test_legacy_mode_classification.py b/python/pyspark/ml/tests/connect/test_legacy_mode_classification.py
index 0028feb6cce7..854326101f8f 100644
--- a/python/pyspark/ml/tests/connect/test_legacy_mode_classification.py
+++ b/python/pyspark/ml/tests/connect/test_legacy_mode_classification.py
@@ -20,17 +20,20 @@ import unittest
import numpy as np
-from pyspark.util import is_remote_only
-from pyspark.testing.connectutils import should_test_connect, connect_requirement_message
-from pyspark.testing.utils import have_torch, torch_requirement_message
+from pyspark.testing.connectutils import connect_requirement_message, should_test_connect
from pyspark.testing.sqlutils import ReusedSQLTestCase
+from pyspark.testing.utils import have_torch, torch_requirement_message
+from pyspark.util import is_remote_only
if should_test_connect:
+ import pandas as pd
+
from pyspark.ml.connect.classification import (
LogisticRegression as LORV2,
+ )
+ from pyspark.ml.connect.classification import (
LogisticRegressionModel as LORV2Model,
)
- import pandas as pd
class ClassificationTestsMixin:
diff --git a/python/pyspark/ml/tests/connect/test_legacy_mode_evaluation.py b/python/pyspark/ml/tests/connect/test_legacy_mode_evaluation.py
index cc7822c43930..7220adca15a7 100644
--- a/python/pyspark/ml/tests/connect/test_legacy_mode_evaluation.py
+++ b/python/pyspark/ml/tests/connect/test_legacy_mode_evaluation.py
@@ -14,21 +14,21 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-import unittest
import tempfile
+import unittest
import numpy as np
-from pyspark.util import is_remote_only
-from pyspark.testing.connectutils import should_test_connect, connect_requirement_message
-from pyspark.testing.utils import have_torcheval, torcheval_requirement_message
+from pyspark.testing.connectutils import connect_requirement_message, should_test_connect
from pyspark.testing.sqlutils import ReusedSQLTestCase
+from pyspark.testing.utils import have_torcheval, torcheval_requirement_message
+from pyspark.util import is_remote_only
if should_test_connect:
from pyspark.ml.connect.evaluation import (
- RegressionEvaluator,
BinaryClassificationEvaluator,
MulticlassClassificationEvaluator,
+ RegressionEvaluator,
)
diff --git a/python/pyspark/ml/tests/connect/test_legacy_mode_feature.py b/python/pyspark/ml/tests/connect/test_legacy_mode_feature.py
index de65858e2f51..5bf8b479ddef 100644
--- a/python/pyspark/ml/tests/connect/test_legacy_mode_feature.py
+++ b/python/pyspark/ml/tests/connect/test_legacy_mode_feature.py
@@ -21,20 +21,21 @@ import unittest
import numpy as np
-from pyspark.util import is_remote_only
-from pyspark.testing.connectutils import should_test_connect, connect_requirement_message
-from pyspark.testing.utils import have_torch, torch_requirement_message
+from pyspark.testing.connectutils import connect_requirement_message, should_test_connect
from pyspark.testing.sqlutils import ReusedSQLTestCase
+from pyspark.testing.utils import have_torch, torch_requirement_message
+from pyspark.util import is_remote_only
if should_test_connect:
+ import pandas as pd
+
from pyspark.ml.connect.feature import (
+ ArrayAssembler,
MaxAbsScaler,
MaxAbsScalerModel,
StandardScaler,
StandardScalerModel,
- ArrayAssembler,
)
- import pandas as pd
class FeatureTestsMixin:
diff --git a/python/pyspark/ml/tests/connect/test_legacy_mode_pipeline.py b/python/pyspark/ml/tests/connect/test_legacy_mode_pipeline.py
index bc85f98619a6..0cec1b6565d3 100644
--- a/python/pyspark/ml/tests/connect/test_legacy_mode_pipeline.py
+++ b/python/pyspark/ml/tests/connect/test_legacy_mode_pipeline.py
@@ -20,16 +20,17 @@ import unittest
import numpy as np
-from pyspark.util import is_remote_only
-from pyspark.testing.connectutils import should_test_connect, connect_requirement_message
-from pyspark.testing.utils import have_torch, torch_requirement_message
+from pyspark.testing.connectutils import connect_requirement_message, should_test_connect
from pyspark.testing.sqlutils import ReusedSQLTestCase
+from pyspark.testing.utils import have_torch, torch_requirement_message
+from pyspark.util import is_remote_only
if should_test_connect:
- from pyspark.ml.connect.feature import StandardScaler
+ import pandas as pd
+
from pyspark.ml.connect.classification import LogisticRegression as LORV2
+ from pyspark.ml.connect.feature import StandardScaler
from pyspark.ml.connect.pipeline import Pipeline
- import pandas as pd
class PipelineTestsMixin:
diff --git a/python/pyspark/ml/tests/connect/test_legacy_mode_summarizer.py b/python/pyspark/ml/tests/connect/test_legacy_mode_summarizer.py
index 567f6d854d3b..9eb03fa70ef2 100644
--- a/python/pyspark/ml/tests/connect/test_legacy_mode_summarizer.py
+++ b/python/pyspark/ml/tests/connect/test_legacy_mode_summarizer.py
@@ -19,9 +19,9 @@ import unittest
import numpy as np
-from pyspark.util import is_remote_only
-from pyspark.testing.connectutils import should_test_connect, connect_requirement_message
+from pyspark.testing.connectutils import connect_requirement_message, should_test_connect
from pyspark.testing.sqlutils import ReusedSQLTestCase
+from pyspark.util import is_remote_only
if should_test_connect:
from pyspark.ml.connect.summarizer import summarize_dataframe
diff --git a/python/pyspark/ml/tests/connect/test_legacy_mode_tuning.py b/python/pyspark/ml/tests/connect/test_legacy_mode_tuning.py
index 4a819520963b..2912b0d96ac4 100644
--- a/python/pyspark/ml/tests/connect/test_legacy_mode_tuning.py
+++ b/python/pyspark/ml/tests/connect/test_legacy_mode_tuning.py
@@ -15,35 +15,36 @@
# limitations under the License.
#
+import sys
import tempfile
import unittest
-import sys
import numpy as np
-from pyspark.util import is_remote_only
from pyspark.ml.param import Param, Params
from pyspark.ml.tuning import ParamGridBuilder
from pyspark.sql.functions import rand
-from pyspark.testing.connectutils import should_test_connect, connect_requirement_message
+from pyspark.testing.connectutils import connect_requirement_message, should_test_connect
+from pyspark.testing.sqlutils import ReusedSQLTestCase
from pyspark.testing.utils import (
have_sklearn,
- sklearn_requirement_message,
have_torch,
- torch_requirement_message,
have_torcheval,
+ sklearn_requirement_message,
+ torch_requirement_message,
torcheval_requirement_message,
)
-from pyspark.testing.sqlutils import ReusedSQLTestCase
+from pyspark.util import is_remote_only
if should_test_connect:
import pandas as pd
- from pyspark.ml.connect import Model, Estimator
- from pyspark.ml.connect.feature import StandardScaler
+
+ from pyspark.ml.connect import Estimator, Model
from pyspark.ml.connect.classification import LogisticRegression as LORV2
+ from pyspark.ml.connect.evaluation import BinaryClassificationEvaluator, RegressionEvaluator
+ from pyspark.ml.connect.feature import StandardScaler
from pyspark.ml.connect.pipeline import Pipeline
from pyspark.ml.connect.tuning import CrossValidator, CrossValidatorModel
- from pyspark.ml.connect.evaluation import BinaryClassificationEvaluator, RegressionEvaluator
class HasInducedError(Params):
def __init__(self):
diff --git a/python/pyspark/ml/tests/connect/test_parity_torch_data_loader.py b/python/pyspark/ml/tests/connect/test_parity_torch_data_loader.py
index e7b96aab585d..256c4319dc81 100644
--- a/python/pyspark/ml/tests/connect/test_parity_torch_data_loader.py
+++ b/python/pyspark/ml/tests/connect/test_parity_torch_data_loader.py
@@ -17,8 +17,8 @@
import unittest
-from pyspark.util import is_remote_only
from pyspark.sql import SparkSession
+from pyspark.util import is_remote_only
if not is_remote_only():
from pyspark.ml.torch.tests.test_data_loader import TorchDistributorDataLoaderUnitTests
diff --git a/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py b/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py
index ce0d64c66f88..3a6d27183453 100644
--- a/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py
+++ b/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py
@@ -19,24 +19,24 @@ import os
import shutil
import unittest
-from pyspark.util import is_remote_only
from pyspark.sql import SparkSession
from pyspark.testing.utils import (
+ connect_requirement_message,
have_torch,
- torch_requirement_message,
should_test_connect,
- connect_requirement_message,
+ torch_requirement_message,
)
+from pyspark.util import is_remote_only
if not is_remote_only() and should_test_connect:
from pyspark.ml.torch.tests.test_distributor import (
TorchDistributorBaselineUnitTestsMixin,
- TorchDistributorLocalUnitTestsMixin,
TorchDistributorDistributedUnitTestsMixin,
+ TorchDistributorLocalUnitTestsMixin,
TorchWrapperUnitTestsMixin,
- set_up_test_dirs,
- get_local_mode_conf,
get_distributed_mode_conf,
+ get_local_mode_conf,
+ set_up_test_dirs,
)
@unittest.skipIf(
diff --git a/python/pyspark/ml/tests/test_algorithms.py b/python/pyspark/ml/tests/test_algorithms.py
index 8c9f18d62784..7d630f71e470 100644
--- a/python/pyspark/ml/tests/test_algorithms.py
+++ b/python/pyspark/ml/tests/test_algorithms.py
@@ -15,8 +15,8 @@
# limitations under the License.
#
import os
-from shutil import rmtree
import tempfile
+from shutil import rmtree
import numpy as np
@@ -26,9 +26,9 @@ from pyspark.ml.classification import (
MultilayerPerceptronClassifier,
OneVsRest,
)
-from pyspark.ml.clustering import DistributedLDAModel, KMeans, LocalLDAModel, LDA, LDAModel
+from pyspark.ml.clustering import LDA, DistributedLDAModel, KMeans, LDAModel, LocalLDAModel
from pyspark.ml.fpm import FPGrowth
-from pyspark.ml.linalg import Vectors, DenseVector
+from pyspark.ml.linalg import DenseVector, Vectors
from pyspark.ml.recommendation import ALS
from pyspark.ml.regression import GeneralizedLinearRegression, LinearRegression
from pyspark.sql import Row
diff --git a/python/pyspark/ml/tests/test_base.py b/python/pyspark/ml/tests/test_base.py
index cf4cec224e24..7bf662dc3b7f 100644
--- a/python/pyspark/ml/tests/test_base.py
+++ b/python/pyspark/ml/tests/test_base.py
@@ -21,8 +21,8 @@ from pyspark.sql.types import DoubleType, IntegerType
from pyspark.testing.mlutils import (
MockDataset,
MockEstimator,
- MockUnaryTransformer,
MockTransformer,
+ MockUnaryTransformer,
SparkSessionTestCase,
)
diff --git a/python/pyspark/ml/tests/test_classification.py b/python/pyspark/ml/tests/test_classification.py
index c8785d0e9ce1..761ec650fac1 100644
--- a/python/pyspark/ml/tests/test_classification.py
+++ b/python/pyspark/ml/tests/test_classification.py
@@ -21,11 +21,18 @@ from shutil import rmtree
import numpy as np
from pyspark.errors import PySparkException
-from pyspark.ml.linalg import Vectors, Matrices
-from pyspark.sql import DataFrame, Row
from pyspark.ml.classification import (
- NaiveBayes,
- NaiveBayesModel,
+ BinaryLogisticRegressionSummary,
+ BinaryRandomForestClassificationSummary,
+ BinaryRandomForestClassificationTrainingSummary,
+ DecisionTreeClassificationModel,
+ DecisionTreeClassifier,
+ FMClassificationModel,
+ FMClassificationSummary,
+ FMClassificationTrainingSummary,
+ FMClassifier,
+ GBTClassificationModel,
+ GBTClassifier,
LinearSVC,
LinearSVCModel,
LinearSVCSummary,
@@ -33,27 +40,20 @@ from pyspark.ml.classification import (
LogisticRegression,
LogisticRegressionModel,
LogisticRegressionSummary,
- BinaryLogisticRegressionSummary,
- FMClassifier,
- FMClassificationModel,
- FMClassificationSummary,
- FMClassificationTrainingSummary,
- DecisionTreeClassifier,
- DecisionTreeClassificationModel,
- RandomForestClassifier,
- RandomForestClassificationModel,
- RandomForestClassificationSummary,
- RandomForestClassificationTrainingSummary,
- BinaryRandomForestClassificationSummary,
- BinaryRandomForestClassificationTrainingSummary,
- GBTClassifier,
- GBTClassificationModel,
- MultilayerPerceptronClassifier,
MultilayerPerceptronClassificationModel,
MultilayerPerceptronClassificationSummary,
MultilayerPerceptronClassificationTrainingSummary,
+ MultilayerPerceptronClassifier,
+ NaiveBayes,
+ NaiveBayesModel,
+ RandomForestClassificationModel,
+ RandomForestClassificationSummary,
+ RandomForestClassificationTrainingSummary,
+ RandomForestClassifier,
)
+from pyspark.ml.linalg import Matrices, Vectors
from pyspark.ml.regression import DecisionTreeRegressionModel
+from pyspark.sql import DataFrame, Row
from pyspark.testing.sqlutils import ReusedSQLTestCase
diff --git a/python/pyspark/ml/tests/test_clustering.py b/python/pyspark/ml/tests/test_clustering.py
index e22e97a5e7f1..6b753d591027 100644
--- a/python/pyspark/ml/tests/test_clustering.py
+++ b/python/pyspark/ml/tests/test_clustering.py
@@ -19,23 +19,23 @@ import tempfile
import numpy as np
-from pyspark.ml.linalg import Vectors, SparseVector
from pyspark.ml.clustering import (
- KMeans,
- KMeansModel,
- KMeansSummary,
+ LDA,
BisectingKMeans,
BisectingKMeansModel,
BisectingKMeansSummary,
+ DistributedLDAModel,
GaussianMixture,
GaussianMixtureModel,
GaussianMixtureSummary,
- LDA,
+ KMeans,
+ KMeansModel,
+ KMeansSummary,
LDAModel,
LocalLDAModel,
- DistributedLDAModel,
PowerIterationClustering,
)
+from pyspark.ml.linalg import SparseVector, Vectors
from pyspark.testing.sqlutils import ReusedSQLTestCase
diff --git a/python/pyspark/ml/tests/test_dl_util.py b/python/pyspark/ml/tests/test_dl_util.py
index ba9dc31b204e..7bff80dc606d 100644
--- a/python/pyspark/ml/tests/test_dl_util.py
+++ b/python/pyspark/ml/tests/test_dl_util.py
@@ -14,11 +14,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from contextlib import contextmanager
import os
import textwrap
-from typing import Any, BinaryIO, Callable, Iterator
import unittest
+from contextlib import contextmanager
+from typing import Any, BinaryIO, Callable, Iterator
from pyspark import cloudpickle
from pyspark.ml.dl_util import FunctionPickler
diff --git a/python/pyspark/ml/tests/test_evaluation.py b/python/pyspark/ml/tests/test_evaluation.py
index 5c227ccb423e..3c2947dbf30a 100644
--- a/python/pyspark/ml/tests/test_evaluation.py
+++ b/python/pyspark/ml/tests/test_evaluation.py
@@ -19,12 +19,12 @@ import tempfile
import numpy as np
from pyspark.ml.evaluation import (
- ClusteringEvaluator,
- RegressionEvaluator,
BinaryClassificationEvaluator,
+ ClusteringEvaluator,
MulticlassClassificationEvaluator,
MultilabelClassificationEvaluator,
RankingEvaluator,
+ RegressionEvaluator,
)
from pyspark.ml.linalg import Vectors
from pyspark.sql import Row
diff --git a/python/pyspark/ml/tests/test_feature.py b/python/pyspark/ml/tests/test_feature.py
index 0047d0810699..13e36ce76c0e 100644
--- a/python/pyspark/ml/tests/test_feature.py
+++ b/python/pyspark/ml/tests/test_feature.py
@@ -16,68 +16,68 @@
#
import tempfile
-from typing import List, Tuple, Any
+from typing import Any, List, Tuple
import numpy as np
from pyspark.ml.feature import (
DCT,
+ IDF,
+ PCA,
Binarizer,
+ BucketedRandomProjectionLSH,
+ BucketedRandomProjectionLSHModel,
Bucketizer,
- QuantileDiscretizer,
+ ChiSqSelector,
+ ChiSqSelectorModel,
CountVectorizer,
CountVectorizerModel,
- OneHotEncoder,
- OneHotEncoderModel,
- FeatureHasher,
ElementwiseProduct,
+ FeatureHasher,
HashingTF,
- IDF,
IDFModel,
Imputer,
ImputerModel,
- NGram,
- Normalizer,
+ IndexToString,
Interaction,
- RFormula,
- RFormulaModel,
- Tokenizer,
- SQLTransformer,
- RegexTokenizer,
- StandardScaler,
- StandardScalerModel,
MaxAbsScaler,
MaxAbsScalerModel,
+ MinHashLSH,
+ MinHashLSHModel,
MinMaxScaler,
MinMaxScalerModel,
+ NGram,
+ Normalizer,
+ OneHotEncoder,
+ OneHotEncoderModel,
+ PCAModel,
+ PolynomialExpansion,
+ QuantileDiscretizer,
+ RegexTokenizer,
+ RFormula,
+ RFormulaModel,
RobustScaler,
RobustScalerModel,
- ChiSqSelector,
- ChiSqSelectorModel,
+ SQLTransformer,
+ StandardScaler,
+ StandardScalerModel,
+ StopWordsRemover,
+ StringIndexer,
+ StringIndexerModel,
+ TargetEncoder,
+ TargetEncoderModel,
+ Tokenizer,
UnivariateFeatureSelector,
UnivariateFeatureSelectorModel,
VarianceThresholdSelector,
VarianceThresholdSelectorModel,
- StopWordsRemover,
- StringIndexer,
- StringIndexerModel,
+ VectorAssembler,
VectorIndexer,
VectorIndexerModel,
- TargetEncoder,
- TargetEncoderModel,
VectorSizeHint,
VectorSlicer,
- VectorAssembler,
- PCA,
- PCAModel,
Word2Vec,
Word2VecModel,
- BucketedRandomProjectionLSH,
- BucketedRandomProjectionLSHModel,
- MinHashLSH,
- MinHashLSHModel,
- IndexToString,
- PolynomialExpansion,
)
from pyspark.ml.linalg import DenseVector, SparseVector, Vectors
from pyspark.sql import Row
diff --git a/python/pyspark/ml/tests/test_fpm.py b/python/pyspark/ml/tests/test_fpm.py
index b54f55738111..5d486042a006 100644
--- a/python/pyspark/ml/tests/test_fpm.py
+++ b/python/pyspark/ml/tests/test_fpm.py
@@ -17,13 +17,13 @@
import tempfile
-from pyspark.sql import Row
import pyspark.sql.functions as sf
from pyspark.ml.fpm import (
FPGrowth,
FPGrowthModel,
PrefixSpan,
)
+from pyspark.sql import Row
from pyspark.testing.sqlutils import ReusedSQLTestCase
diff --git a/python/pyspark/ml/tests/test_functions.py b/python/pyspark/ml/tests/test_functions.py
index 6e1d25a0cfda..82eb5bc02b03 100644
--- a/python/pyspark/ml/tests/test_functions.py
+++ b/python/pyspark/ml/tests/test_functions.py
@@ -19,10 +19,10 @@ import unittest
import numpy as np
+from pyspark.ml.functions import array_to_vector, predict_batch_udf, vector_to_array
from pyspark.ml.linalg import DenseVector
-from pyspark.ml.functions import array_to_vector, vector_to_array, predict_batch_udf
-from pyspark.sql.functions import array, struct, col
-from pyspark.sql.types import ArrayType, DoubleType, IntegerType, StructType, StructField, FloatType
+from pyspark.sql.functions import array, col, struct
+from pyspark.sql.types import ArrayType, DoubleType, FloatType, IntegerType, StructField, StructType
from pyspark.testing.sqlutils import ReusedSQLTestCase
from pyspark.testing.utils import (
have_pandas,
diff --git a/python/pyspark/ml/tests/test_image.py b/python/pyspark/ml/tests/test_image.py
index 7cacde935400..8b98ad6f5bc8 100644
--- a/python/pyspark/ml/tests/test_image.py
+++ b/python/pyspark/ml/tests/test_image.py
@@ -16,8 +16,8 @@
#
from pyspark.ml.image import ImageSchema
-from pyspark.testing.mlutils import SparkSessionTestCase
from pyspark.sql import Row
+from pyspark.testing.mlutils import SparkSessionTestCase
from pyspark.testing.utils import QuietTest, eventually
diff --git a/python/pyspark/ml/tests/test_linalg.py b/python/pyspark/ml/tests/test_linalg.py
index eea293188ef2..5b6677088ab7 100644
--- a/python/pyspark/ml/tests/test_linalg.py
+++ b/python/pyspark/ml/tests/test_linalg.py
@@ -20,7 +20,6 @@ import array as pyarray
from numpy import arange, array, array_equal, inf, ones, tile, zeros
from pyspark.errors import AnalysisException
-from pyspark.serializers import CPickleSerializer
from pyspark.ml.linalg import (
DenseMatrix,
DenseVector,
@@ -28,13 +27,14 @@ from pyspark.ml.linalg import (
SparseMatrix,
SparseVector,
Vector,
- VectorUDT,
Vectors,
+ VectorUDT,
)
-from pyspark.testing.mllibutils import MLlibTestCase
+from pyspark.serializers import CPickleSerializer
from pyspark.sql import Row
from pyspark.sql.functions import lit, unwrap_udt, wrap_udt
from pyspark.sql.types import StructField, StructType
+from pyspark.testing.mllibutils import MLlibTestCase
class VectorTests(MLlibTestCase):
diff --git a/python/pyspark/ml/tests/test_ovr.py b/python/pyspark/ml/tests/test_ovr.py
index d68ea1d9737b..3b10bfac42c1 100644
--- a/python/pyspark/ml/tests/test_ovr.py
+++ b/python/pyspark/ml/tests/test_ovr.py
@@ -19,13 +19,13 @@ import tempfile
import numpy as np
-from pyspark.ml.linalg import Vectors
from pyspark.ml.classification import (
LinearSVC,
LinearSVCModel,
OneVsRest,
OneVsRestModel,
)
+from pyspark.ml.linalg import Vectors
from pyspark.testing.sqlutils import ReusedSQLTestCase
diff --git a/python/pyspark/ml/tests/test_param.py b/python/pyspark/ml/tests/test_param.py
index fc4fa07863e2..f71048edd659 100644
--- a/python/pyspark/ml/tests/test_param.py
+++ b/python/pyspark/ml/tests/test_param.py
@@ -15,8 +15,8 @@
# limitations under the License.
#
-import inspect
import array as pyarray
+import inspect
import numpy as np
@@ -35,9 +35,9 @@ from pyspark.ml.feature import (
from pyspark.ml.linalg import DenseVector, SparseVector, Vectors
from pyspark.ml.param import Param, Params, TypeConverters
from pyspark.ml.param.shared import HasInputCol, HasMaxIter, HasSeed
-from pyspark.ml.regression import LinearRegressionModel, GeneralizedLinearRegressionModel
+from pyspark.ml.regression import GeneralizedLinearRegressionModel, LinearRegressionModel
from pyspark.ml.wrapper import JavaParams
-from pyspark.testing.mlutils import check_params, PySparkTestCase, SparkSessionTestCase
+from pyspark.testing.mlutils import PySparkTestCase, SparkSessionTestCase, check_params
class ParamTypeConversionTests(PySparkTestCase):
@@ -386,10 +386,10 @@ class DefaultValuesTests(PySparkTestCase):
def test_java_params(self):
import re
- import pyspark.ml.feature
import pyspark.ml.classification
import pyspark.ml.clustering
import pyspark.ml.evaluation
+ import pyspark.ml.feature
import pyspark.ml.pipeline
import pyspark.ml.recommendation
import pyspark.ml.regression
diff --git a/python/pyspark/ml/tests/test_persistence.py b/python/pyspark/ml/tests/test_persistence.py
index ee97ae164a97..76eea039a3f8 100644
--- a/python/pyspark/ml/tests/test_persistence.py
+++ b/python/pyspark/ml/tests/test_persistence.py
@@ -16,22 +16,22 @@
#
import json
-from shutil import rmtree
import tempfile
+from shutil import rmtree
from pyspark.ml import Transformer
from pyspark.ml.classification import (
DecisionTreeClassifier,
- FMClassifier,
FMClassificationModel,
+ FMClassifier,
LogisticRegression,
- MultilayerPerceptronClassifier,
MultilayerPerceptronClassificationModel,
+ MultilayerPerceptronClassifier,
OneVsRest,
OneVsRestModel,
)
from pyspark.ml.clustering import KMeans
-from pyspark.ml.feature import Binarizer, Bucketizer, HashingTF, PCA
+from pyspark.ml.feature import PCA, Binarizer, Bucketizer, HashingTF
from pyspark.ml.linalg import Vectors
from pyspark.ml.param import Params
from pyspark.ml.pipeline import Pipeline, PipelineModel
diff --git a/python/pyspark/ml/tests/test_pipeline.py b/python/pyspark/ml/tests/test_pipeline.py
index ce104b563772..6e72624115b0 100644
--- a/python/pyspark/ml/tests/test_pipeline.py
+++ b/python/pyspark/ml/tests/test_pipeline.py
@@ -18,18 +18,18 @@
import tempfile
import unittest
-from pyspark.sql import Row
-from pyspark.ml.pipeline import Pipeline, PipelineModel
+from pyspark.ml.classification import LogisticRegression, LogisticRegressionModel
+from pyspark.ml.clustering import GaussianMixture, KMeans, KMeansModel
from pyspark.ml.feature import (
- VectorAssembler,
MaxAbsScaler,
MaxAbsScalerModel,
MinMaxScaler,
MinMaxScalerModel,
+ VectorAssembler,
)
from pyspark.ml.linalg import Vectors
-from pyspark.ml.classification import LogisticRegression, LogisticRegressionModel
-from pyspark.ml.clustering import KMeans, KMeansModel, GaussianMixture
+from pyspark.ml.pipeline import Pipeline, PipelineModel
+from pyspark.sql import Row
from pyspark.testing.mlutils import MockDataset, MockEstimator, MockTransformer
from pyspark.testing.sqlutils import ReusedSQLTestCase
diff --git a/python/pyspark/ml/tests/test_regression.py b/python/pyspark/ml/tests/test_regression.py
index 1427f2a45005..929ded5d2761 100644
--- a/python/pyspark/ml/tests/test_regression.py
+++ b/python/pyspark/ml/tests/test_regression.py
@@ -23,24 +23,24 @@ from pyspark.ml.linalg import Vectors
from pyspark.ml.regression import (
AFTSurvivalRegression,
AFTSurvivalRegressionModel,
- IsotonicRegression,
- IsotonicRegressionModel,
- LinearRegression,
- LinearRegressionModel,
+ DecisionTreeRegressionModel,
+ DecisionTreeRegressor,
+ FMRegressionModel,
+ FMRegressor,
+ GBTRegressionModel,
+ GBTRegressor,
GeneralizedLinearRegression,
GeneralizedLinearRegressionModel,
GeneralizedLinearRegressionSummary,
GeneralizedLinearRegressionTrainingSummary,
+ IsotonicRegression,
+ IsotonicRegressionModel,
+ LinearRegression,
+ LinearRegressionModel,
LinearRegressionSummary,
LinearRegressionTrainingSummary,
- FMRegressor,
- FMRegressionModel,
- DecisionTreeRegressor,
- DecisionTreeRegressionModel,
- RandomForestRegressor,
RandomForestRegressionModel,
- GBTRegressor,
- GBTRegressionModel,
+ RandomForestRegressor,
)
from pyspark.testing.sqlutils import ReusedSQLTestCase
diff --git a/python/pyspark/ml/tests/test_stat.py b/python/pyspark/ml/tests/test_stat.py
index 6f3b82e1e12c..60e7f152563b 100644
--- a/python/pyspark/ml/tests/test_stat.py
+++ b/python/pyspark/ml/tests/test_stat.py
@@ -18,7 +18,7 @@
import numpy as np
from pyspark.errors import PySparkException
-from pyspark.ml.linalg import Vectors, DenseVector
+from pyspark.ml.linalg import DenseVector, Vectors
from pyspark.ml.stat import (
ChiSquareTest,
Correlation,
@@ -26,8 +26,8 @@ from pyspark.ml.stat import (
Summarizer,
SummaryBuilder,
)
-from pyspark.sql import functions as F
from pyspark.sql import DataFrame, Row
+from pyspark.sql import functions as F
from pyspark.testing.sqlutils import ReusedSQLTestCase
diff --git a/python/pyspark/ml/tests/test_training_summary.py b/python/pyspark/ml/tests/test_training_summary.py
index ed43d60986b0..33fdb7f735b4 100644
--- a/python/pyspark/ml/tests/test_training_summary.py
+++ b/python/pyspark/ml/tests/test_training_summary.py
@@ -18,12 +18,12 @@
from pyspark.ml.classification import (
BinaryRandomForestClassificationSummary,
- FMClassifier,
FMClassificationSummary,
+ FMClassifier,
LinearSVC,
LinearSVCSummary,
- MultilayerPerceptronClassifier,
MultilayerPerceptronClassificationSummary,
+ MultilayerPerceptronClassifier,
RandomForestClassificationSummary,
RandomForestClassifier,
)
diff --git a/python/pyspark/ml/tests/test_tuning.py b/python/pyspark/ml/tests/test_tuning.py
index 72380f4d68d2..6e2a03a3049f 100644
--- a/python/pyspark/ml/tests/test_tuning.py
+++ b/python/pyspark/ml/tests/test_tuning.py
@@ -19,13 +19,13 @@ import tempfile
import numpy as np
+from pyspark.ml.classification import LogisticRegression, RandomForestClassifier
from pyspark.ml.evaluation import BinaryClassificationEvaluator
from pyspark.ml.linalg import Vectors
-from pyspark.ml.classification import LogisticRegression, RandomForestClassifier
from pyspark.ml.tuning import (
- ParamGridBuilder,
CrossValidator,
CrossValidatorModel,
+ ParamGridBuilder,
TrainValidationSplit,
TrainValidationSplitModel,
)
diff --git a/python/pyspark/ml/tests/test_wrapper.py b/python/pyspark/ml/tests/test_wrapper.py
index 4a58336bce81..1e5d04769f00 100644
--- a/python/pyspark/ml/tests/test_wrapper.py
+++ b/python/pyspark/ml/tests/test_wrapper.py
@@ -21,10 +21,10 @@ import py4j
from pyspark.ml.linalg import DenseVector, Vectors
from pyspark.ml.regression import LinearRegression
from pyspark.ml.wrapper import (
- _java2py,
- _py2java,
JavaParams,
JavaWrapper,
+ _java2py,
+ _py2java,
)
from pyspark.testing.mllibutils import MLlibTestCase
from pyspark.testing.mlutils import SparkSessionTestCase
diff --git a/python/pyspark/ml/tests/tuning/test_cv_io_basic.py b/python/pyspark/ml/tests/tuning/test_cv_io_basic.py
index cd65b3fbb4f7..891f1fc1765b 100644
--- a/python/pyspark/ml/tests/tuning/test_cv_io_basic.py
+++ b/python/pyspark/ml/tests/tuning/test_cv_io_basic.py
@@ -20,6 +20,7 @@ import tempfile
from pyspark.ml.classification import LogisticRegression, LogisticRegressionModel
from pyspark.ml.evaluation import BinaryClassificationEvaluator
from pyspark.ml.linalg import Vectors
+from pyspark.ml.tests.tuning.test_tuning import ValidatorTestUtilsMixin
from pyspark.ml.tuning import (
CrossValidator,
CrossValidatorModel,
@@ -31,7 +32,6 @@ from pyspark.testing.mlutils import (
DummyLogisticRegressionModel,
SparkSessionTestCase,
)
-from pyspark.ml.tests.tuning.test_tuning import ValidatorTestUtilsMixin
class CrossValidatorIOBasicTests(SparkSessionTestCase, ValidatorTestUtilsMixin):
diff --git a/python/pyspark/ml/tests/tuning/test_cv_io_nested.py b/python/pyspark/ml/tests/tuning/test_cv_io_nested.py
index 5f25e6e23ccc..efcd124bb526 100644
--- a/python/pyspark/ml/tests/tuning/test_cv_io_nested.py
+++ b/python/pyspark/ml/tests/tuning/test_cv_io_nested.py
@@ -20,6 +20,7 @@ import tempfile
from pyspark.ml.classification import LogisticRegression, OneVsRest
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
from pyspark.ml.linalg import Vectors
+from pyspark.ml.tests.tuning.test_tuning import ValidatorTestUtilsMixin
from pyspark.ml.tuning import (
CrossValidator,
CrossValidatorModel,
@@ -29,7 +30,6 @@ from pyspark.testing.mlutils import (
DummyLogisticRegression,
SparkSessionTestCase,
)
-from pyspark.ml.tests.tuning.test_tuning import ValidatorTestUtilsMixin
class CrossValidatorIONestedTests(SparkSessionTestCase, ValidatorTestUtilsMixin):
diff --git a/python/pyspark/ml/tests/tuning/test_cv_io_pipeline.py b/python/pyspark/ml/tests/tuning/test_cv_io_pipeline.py
index 32e97ceb4f84..96cb0c482365 100644
--- a/python/pyspark/ml/tests/tuning/test_cv_io_pipeline.py
+++ b/python/pyspark/ml/tests/tuning/test_cv_io_pipeline.py
@@ -17,10 +17,11 @@
import tempfile
-from pyspark.ml.feature import HashingTF, Tokenizer
from pyspark.ml import Pipeline
from pyspark.ml.classification import LogisticRegression, OneVsRest
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
+from pyspark.ml.feature import HashingTF, Tokenizer
+from pyspark.ml.tests.tuning.test_tuning import ValidatorTestUtilsMixin
from pyspark.ml.tuning import (
CrossValidator,
CrossValidatorModel,
@@ -30,7 +31,6 @@ from pyspark.testing.mlutils import (
DummyLogisticRegression,
SparkSessionTestCase,
)
-from pyspark.ml.tests.tuning.test_tuning import ValidatorTestUtilsMixin
class CrossValidatorIOPipelineTests(SparkSessionTestCase, ValidatorTestUtilsMixin):
diff --git a/python/pyspark/ml/tests/tuning/test_tvs_io_basic.py b/python/pyspark/ml/tests/tuning/test_tvs_io_basic.py
index b8ff98c12f33..044e6d38ee1b 100644
--- a/python/pyspark/ml/tests/tuning/test_tvs_io_basic.py
+++ b/python/pyspark/ml/tests/tuning/test_tvs_io_basic.py
@@ -20,6 +20,7 @@ import tempfile
from pyspark.ml.classification import LogisticRegression, LogisticRegressionModel
from pyspark.ml.evaluation import BinaryClassificationEvaluator
from pyspark.ml.linalg import Vectors
+from pyspark.ml.tests.tuning.test_tuning import ValidatorTestUtilsMixin
from pyspark.ml.tuning import (
ParamGridBuilder,
TrainValidationSplit,
@@ -31,7 +32,6 @@ from pyspark.testing.mlutils import (
DummyLogisticRegressionModel,
SparkSessionTestCase,
)
-from pyspark.ml.tests.tuning.test_tuning import ValidatorTestUtilsMixin
class TrainValidationSplitIOBasicTests(SparkSessionTestCase, ValidatorTestUtilsMixin):
diff --git a/python/pyspark/ml/tests/tuning/test_tvs_io_nested.py b/python/pyspark/ml/tests/tuning/test_tvs_io_nested.py
index 252afc344b4e..7176784916a9 100644
--- a/python/pyspark/ml/tests/tuning/test_tvs_io_nested.py
+++ b/python/pyspark/ml/tests/tuning/test_tvs_io_nested.py
@@ -20,6 +20,7 @@ import tempfile
from pyspark.ml.classification import LogisticRegression, OneVsRest
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
from pyspark.ml.linalg import Vectors
+from pyspark.ml.tests.tuning.test_tuning import ValidatorTestUtilsMixin
from pyspark.ml.tuning import (
ParamGridBuilder,
TrainValidationSplit,
@@ -29,7 +30,6 @@ from pyspark.testing.mlutils import (
DummyLogisticRegression,
SparkSessionTestCase,
)
-from pyspark.ml.tests.tuning.test_tuning import ValidatorTestUtilsMixin
class TrainValidationSplitIONestedTests(SparkSessionTestCase, ValidatorTestUtilsMixin):
diff --git a/python/pyspark/ml/tests/tuning/test_tvs_io_pipeline.py b/python/pyspark/ml/tests/tuning/test_tvs_io_pipeline.py
index 029b6990a310..c13de85c5f89 100644
--- a/python/pyspark/ml/tests/tuning/test_tvs_io_pipeline.py
+++ b/python/pyspark/ml/tests/tuning/test_tvs_io_pipeline.py
@@ -17,10 +17,11 @@
import tempfile
-from pyspark.ml.feature import HashingTF, Tokenizer
from pyspark.ml import Pipeline
from pyspark.ml.classification import LogisticRegression, OneVsRest
from pyspark.ml.evaluation import MulticlassClassificationEvaluator
+from pyspark.ml.feature import HashingTF, Tokenizer
+from pyspark.ml.tests.tuning.test_tuning import ValidatorTestUtilsMixin
from pyspark.ml.tuning import (
ParamGridBuilder,
TrainValidationSplit,
@@ -31,8 +32,6 @@ from pyspark.testing.mlutils import (
SparkSessionTestCase,
)
-from pyspark.ml.tests.tuning.test_tuning import ValidatorTestUtilsMixin
-
class TrainValidationSplitIONestedTests(SparkSessionTestCase, ValidatorTestUtilsMixin):
def _run_test_save_load_pipeline_estimator(self, LogisticRegressionCls):
diff --git a/python/pyspark/ml/torch/data.py b/python/pyspark/ml/torch/data.py
index 826526d56c8d..cc49dc1909e6 100644
--- a/python/pyspark/ml/torch/data.py
+++ b/python/pyspark/ml/torch/data.py
@@ -17,8 +17,8 @@
from typing import Any, Callable, Iterator
-import torch
import numpy as np
+import torch
from pyspark.sql.types import StructType
diff --git a/python/pyspark/ml/torch/distributor.py b/python/pyspark/ml/torch/distributor.py
index 67ba6adb03c4..8d5b1a74b9ae 100644
--- a/python/pyspark/ml/torch/distributor.py
+++ b/python/pyspark/ml/torch/distributor.py
@@ -14,9 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-import json
-from contextlib import contextmanager
import collections
+import json
import logging
import os
import random
@@ -27,27 +26,28 @@ import sys
import tempfile
import textwrap
import time
+from contextlib import contextmanager
from typing import (
- Union,
+ Any,
Callable,
- List,
Dict,
- Optional,
- Any,
- Tuple,
Generator,
Iterator,
+ List,
+ Optional,
+ Tuple,
+ Union,
)
from pyspark import cloudpickle
-from pyspark.resource.information import ResourceInformation
-from pyspark.sql import DataFrame, SparkSession
-from pyspark.taskcontext import BarrierTaskContext
+from pyspark.ml.dl_util import FunctionPickler
from pyspark.ml.torch.log_communication import ( # type: ignore
LogStreamingClient,
LogStreamingServer,
)
-from pyspark.ml.dl_util import FunctionPickler
+from pyspark.resource.information import ResourceInformation
+from pyspark.sql import DataFrame, SparkSession
+from pyspark.taskcontext import BarrierTaskContext
def _get_resources(session: SparkSession) -> Dict[str, ResourceInformation]:
@@ -660,8 +660,10 @@ class TorchDistributor(Distributor):
# Spark task program
def wrapped_train_fn(iterator): # type: ignore[no-untyped-def]
import os
+
import pandas as pd
import pyarrow
+
from pyspark import BarrierTaskContext
CUDA_VISIBLE_DEVICES = "CUDA_VISIBLE_DEVICES"
@@ -855,10 +857,11 @@ class TorchDistributor(Distributor):
def _setup_spark_partition_data(
partition_data_iterator: Iterator[Any], input_schema_json: Dict[str, Any]
) -> Iterator[Any]:
- from pyspark.sql.pandas.serializers import ArrowStreamSerializer
- from pyspark.core.files import SparkFiles
import json
+ from pyspark.core.files import SparkFiles
+ from pyspark.sql.pandas.serializers import ArrowStreamSerializer
+
if input_schema_json is None:
yield
return
@@ -1081,10 +1084,11 @@ def _get_spark_partition_data_loader(
prefetch_factor:
Number of batches loaded in advance by each worker
"""
- from pyspark.sql.types import StructType
- from pyspark.ml.torch.data import _SparkPartitionTorchDataset
from torch.utils.data import DataLoader
+ from pyspark.ml.torch.data import _SparkPartitionTorchDataset
+ from pyspark.sql.types import StructType
+
arrow_file = os.environ[SPARK_PARTITION_ARROW_DATA_FILE]
schema_file = os.environ[SPARK_DATAFRAME_SCHEMA_FILE]
diff --git a/python/pyspark/ml/torch/log_communication.py b/python/pyspark/ml/torch/log_communication.py
index ad1fc810f3ef..d6c19af1553e 100644
--- a/python/pyspark/ml/torch/log_communication.py
+++ b/python/pyspark/ml/torch/log_communication.py
@@ -16,16 +16,16 @@
#
# type: ignore
-from contextlib import closing
-import time
import socket
import socketserver
-from struct import pack, unpack
import sys
import threading
+import time
import traceback
-from typing import Generator
import warnings
+from contextlib import closing
+from struct import pack, unpack
+from typing import Generator
# Use b'\x00' as separator instead of b'\n', because the bytes are encoded in utf-8
_SERVER_POLL_INTERVAL = 0.1
diff --git a/python/pyspark/ml/torch/tests/test_data_loader.py b/python/pyspark/ml/torch/tests/test_data_loader.py
index c911188f5807..8b3b97048399 100644
--- a/python/pyspark/ml/torch/tests/test_data_loader.py
+++ b/python/pyspark/ml/torch/tests/test_data_loader.py
@@ -17,12 +17,12 @@
import unittest
+from pyspark.ml.linalg import Vectors
from pyspark.ml.torch.distributor import (
TorchDistributor,
_get_spark_partition_data_loader,
)
from pyspark.sql import SparkSession
-from pyspark.ml.linalg import Vectors
# @unittest.skipIf(not have_torch, torch_requirement_message)
diff --git a/python/pyspark/ml/torch/tests/test_distributor.py b/python/pyspark/ml/torch/tests/test_distributor.py
index 62399e2db68d..0149b869fc53 100644
--- a/python/pyspark/ml/torch/tests/test_distributor.py
+++ b/python/pyspark/ml/torch/tests/test_distributor.py
@@ -18,20 +18,20 @@
import contextlib
import os
import shutil
-from io import StringIO
import stat
import subprocess
import sys
-import time
import tempfile
import threading
-from typing import Callable, Dict, Any
+import time
import unittest
+from io import StringIO
+from typing import Any, Callable, Dict
from unittest.mock import patch
from pyspark import SparkConf, SparkContext
from pyspark.ml.torch.distributor import TorchDistributor, _get_gpus_owned
-from pyspark.ml.torch.torch_run_process_wrapper import clean_and_terminate, check_parent_alive
+from pyspark.ml.torch.torch_run_process_wrapper import check_parent_alive, clean_and_terminate
from pyspark.sql import SparkSession
from pyspark.testing.sqlutils import SPARK_HOME
from pyspark.testing.utils import have_torch, torch_requirement_message
@@ -52,7 +52,7 @@ def patch_stdout() -> StringIO:
def create_training_function(mnist_dir_path: str) -> Callable:
import torch.nn as nn
import torch.nn.functional as F
- from torchvision import transforms, datasets
+ from torchvision import datasets, transforms
batch_size = 100
num_epochs = 1
@@ -87,8 +87,8 @@ def create_training_function(mnist_dir_path: str) -> Callable:
def train_fn(learning_rate: float) -> Any:
import torch
- import torch.optim as optim
import torch.distributed as dist
+ import torch.optim as optim
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.utils.data.distributed import DistributedSampler
diff --git a/python/pyspark/ml/torch/tests/test_log_communication.py b/python/pyspark/ml/torch/tests/test_log_communication.py
index ef8d0c739848..6485bc9c6401 100644
--- a/python/pyspark/ml/torch/tests/test_log_communication.py
+++ b/python/pyspark/ml/torch/tests/test_log_communication.py
@@ -18,18 +18,18 @@
from __future__ import absolute_import, division, print_function
import contextlib
-from io import StringIO
import sys
import time
-from typing import Any, Callable
import unittest
+from io import StringIO
+from typing import Any, Callable
import pyspark.ml.torch.log_communication
from pyspark.ml.torch.log_communication import (
- LogStreamingServer,
+ _SERVER_POLL_INTERVAL,
LogStreamingClient,
LogStreamingClientBase,
- _SERVER_POLL_INTERVAL,
+ LogStreamingServer,
)
diff --git a/python/pyspark/ml/tree.py b/python/pyspark/ml/tree.py
index 92692ec225a7..070a37944f85 100644
--- a/python/pyspark/ml/tree.py
+++ b/python/pyspark/ml/tree.py
@@ -14,23 +14,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from typing import List, Sequence, TypeVar, TYPE_CHECKING
+from typing import TYPE_CHECKING, List, Sequence, TypeVar
from pyspark import since
+from pyspark.ml.common import inherit_doc
from pyspark.ml.linalg import Vector
from pyspark.ml.param import Params
from pyspark.ml.param.shared import (
HasCheckpointInterval,
+ HasMaxIter,
HasSeed,
+ HasStepSize,
+ HasValidationIndicatorCol,
HasWeightCol,
Param,
TypeConverters,
- HasMaxIter,
- HasStepSize,
- HasValidationIndicatorCol,
)
from pyspark.ml.wrapper import JavaPredictionModel
-from pyspark.ml.common import inherit_doc
if TYPE_CHECKING:
from pyspark.ml._typing import P
diff --git a/python/pyspark/ml/tuning.py b/python/pyspark/ml/tuning.py
index 00e317375c40..e4bac8d302c5 100644
--- a/python/pyspark/ml/tuning.py
+++ b/python/pyspark/ml/tuning.py
@@ -15,12 +15,13 @@
# limitations under the License.
#
+import itertools
import json
import os
import sys
-import itertools
from multiprocessing.pool import ThreadPool
from typing import (
+ TYPE_CHECKING,
Any,
Callable,
Dict,
@@ -33,39 +34,39 @@ from typing import (
Union,
cast,
overload,
- TYPE_CHECKING,
)
import numpy as np
-from pyspark import keyword_only, since, inheritable_thread_target
-from pyspark.ml import Estimator, Transformer, Model
-from pyspark.ml.common import inherit_doc, _py2java, _java2py
+from pyspark import inheritable_thread_target, keyword_only, since
+from pyspark.ml.base import Estimator, Model, Transformer
+from pyspark.ml.common import _java2py, _py2java, inherit_doc
from pyspark.ml.evaluation import Evaluator, JavaEvaluator
-from pyspark.ml.param import Params, Param, TypeConverters
+from pyspark.ml.param import Param, Params, TypeConverters
from pyspark.ml.param.shared import HasCollectSubModels, HasParallelism, HasSeed
from pyspark.ml.util import (
DefaultParamsReader,
DefaultParamsWriter,
+ JavaMLWriter,
MetaAlgorithmReadWrite,
MLReadable,
MLReader,
MLWritable,
MLWriter,
- JavaMLWriter,
- try_remote_write,
- try_remote_read,
_cache_spark_dataset,
+ try_remote_read,
+ try_remote_write,
)
-from pyspark.ml.wrapper import JavaParams, JavaEstimator, JavaWrapper
+from pyspark.ml.wrapper import JavaEstimator, JavaParams, JavaWrapper
from pyspark.sql import functions as F
from pyspark.sql.dataframe import DataFrame
if TYPE_CHECKING:
- from pyspark.ml._typing import ParamMap
- from py4j.java_gateway import JavaObject
from py4j.java_collections import JavaArray
+ from py4j.java_gateway import JavaObject
+
from pyspark.core.context import SparkContext
+ from pyspark.ml._typing import ParamMap
__all__ = [
"ParamGridBuilder",
diff --git a/python/pyspark/ml/util.py b/python/pyspark/ml/util.py
index c289078cc43e..447cf44c3c4c 100644
--- a/python/pyspark/ml/util.py
+++ b/python/pyspark/ml/util.py
@@ -15,14 +15,16 @@
# limitations under the License.
#
+import functools
import json
import logging
import os
import threading
import time
import uuid
-import functools
+from contextlib import contextmanager
from typing import (
+ TYPE_CHECKING,
Any,
Callable,
Dict,
@@ -33,11 +35,9 @@ from typing import (
Sequence,
Type,
TypeVar,
- cast,
- TYPE_CHECKING,
Union,
+ cast,
)
-from contextlib import contextmanager
from pyspark import since
from pyspark.ml.common import inherit_doc
@@ -48,13 +48,14 @@ from pyspark.util import VersionUtils
if TYPE_CHECKING:
from py4j.java_gateway import JavaGateway, JavaObject
+
+ from pyspark.core.context import SparkContext
from pyspark.ml._typing import PipelineStage
from pyspark.ml.base import Params
- from pyspark.core.context import SparkContext
+ from pyspark.ml.evaluation import JavaEvaluator
+ from pyspark.ml.wrapper import JavaEstimator, JavaWrapper
from pyspark.sql import DataFrame
from pyspark.sql.connect.dataframe import DataFrame as ConnectDataFrame
- from pyspark.ml.wrapper import JavaWrapper, JavaEstimator
- from pyspark.ml.evaluation import JavaEvaluator
T = TypeVar("T")
RW = TypeVar("RW", bound="BaseReadWrite")
@@ -89,15 +90,15 @@ def invoke_remote_attribute_relation(
instance: "JavaWrapper", method: str, *args: Any
) -> "ConnectDataFrame":
import pyspark.sql.connect.proto as pb2
- from pyspark.ml.connect.util import _extract_id_methods
- from pyspark.ml.connect.serialize import serialize
# The attribute returns a dataframe, we need to wrap it
# in the AttributeRelation
from pyspark.ml.connect.proto import AttributeRelation
- from pyspark.sql.connect.session import SparkSession
- from pyspark.sql.connect.dataframe import DataFrame as ConnectDataFrame
+ from pyspark.ml.connect.serialize import serialize
+ from pyspark.ml.connect.util import _extract_id_methods
from pyspark.ml.wrapper import JavaModel
+ from pyspark.sql.connect.dataframe import DataFrame as ConnectDataFrame
+ from pyspark.sql.connect.session import SparkSession
session = SparkSession.getActiveSession()
assert session is not None
@@ -172,7 +173,7 @@ def try_remote_fit(f: FuncT) -> FuncT:
def wrapped(self: "JavaEstimator", dataset: "ConnectDataFrame") -> Any:
if is_remote() and "PYSPARK_NO_NAMESPACE_SHARE" not in os.environ:
import pyspark.sql.connect.proto as pb2
- from pyspark.ml.connect.serialize import serialize_ml_params, deserialize
+ from pyspark.ml.connect.serialize import deserialize, serialize_ml_params
client = dataset.sparkSession.client
input = dataset._plan.plan(client)
@@ -219,8 +220,8 @@ def try_remote_transform_relation(f: FuncT) -> FuncT:
def wrapped(self: "JavaWrapper", dataset: "ConnectDataFrame") -> Any:
if is_remote() and "PYSPARK_NO_NAMESPACE_SHARE" not in os.environ:
from pyspark.ml import Model, Transformer
- from pyspark.sql.connect.dataframe import DataFrame as ConnectDataFrame
from pyspark.ml.connect.serialize import serialize_ml_params
+ from pyspark.sql.connect.dataframe import DataFrame as ConnectDataFrame
session = dataset.sparkSession
assert session is not None
@@ -278,15 +279,15 @@ def try_remote_call(f: FuncT) -> FuncT:
@functools.wraps(f)
def wrapped(self: "JavaWrapper", name: str, *args: Any) -> Any:
if is_remote() and "PYSPARK_NO_NAMESPACE_SHARE" not in os.environ:
- from pyspark.errors.exceptions.connect import SparkException
import pyspark.sql.connect.proto as pb2
+ from pyspark.errors.exceptions.connect import SparkException
from pyspark.sql.connect.session import SparkSession
session = SparkSession.getActiveSession()
def remote_call() -> Any:
+ from pyspark.ml.connect.serialize import deserialize, serialize
from pyspark.ml.connect.util import _extract_id_methods
- from pyspark.ml.connect.serialize import serialize, deserialize
from pyspark.ml.wrapper import JavaModel
assert session is not None
@@ -460,7 +461,7 @@ def try_remote_evaluate(f: FuncT) -> FuncT:
def wrapped(self: "JavaEvaluator", dataset: "ConnectDataFrame") -> Any:
if is_remote() and "PYSPARK_NO_NAMESPACE_SHARE" not in os.environ:
import pyspark.sql.connect.proto as pb2
- from pyspark.ml.connect.serialize import serialize_ml_params, deserialize
+ from pyspark.ml.connect.serialize import deserialize, serialize_ml_params
client = dataset.sparkSession.client
input = dataset._plan.plan(client)
@@ -1137,8 +1138,8 @@ class MetaAlgorithmReadWrite:
@staticmethod
def isMetaEstimator(pyInstance: Any) -> bool:
from pyspark.ml import Estimator, Pipeline
- from pyspark.ml.tuning import _ValidatorParams
from pyspark.ml.classification import OneVsRest
+ from pyspark.ml.tuning import _ValidatorParams
return (
isinstance(pyInstance, Pipeline)
@@ -1149,8 +1150,8 @@ class MetaAlgorithmReadWrite:
@staticmethod
def getAllNestedStages(pyInstance: Any) -> List["Params"]:
from pyspark.ml import Pipeline, PipelineModel
- from pyspark.ml.tuning import _ValidatorParams
from pyspark.ml.classification import OneVsRest, OneVsRestModel
+ from pyspark.ml.tuning import _ValidatorParams
# TODO: We need to handle `RFormulaModel.pipelineModel` here after Pyspark RFormulaModel
# support pipelineModel property.
diff --git a/python/pyspark/ml/wrapper.py b/python/pyspark/ml/wrapper.py
index cd8df055daa8..09563c3c32da 100644
--- a/python/pyspark/ml/wrapper.py
+++ b/python/pyspark/ml/wrapper.py
@@ -16,27 +16,34 @@
#
from abc import ABCMeta, abstractmethod
-from typing import Any, Generic, Optional, List, Type, TypeVar, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Generic, List, Optional, Type, TypeVar
from pyspark import since
+from pyspark.ml.base import (
+ Estimator,
+ Model,
+ PredictionModel,
+ Predictor,
+ Transformer,
+ _PredictorParams,
+)
+from pyspark.ml.common import _java2py, _py2java, inherit_doc
+from pyspark.ml.param import Param, Params
from pyspark.ml.util import (
- try_remote_transform_relation,
+ _jvm,
try_remote_call,
- try_remote_fit,
try_remote_del,
- try_remote_return_java_class,
+ try_remote_fit,
try_remote_intercept,
+ try_remote_return_java_class,
+ try_remote_transform_relation,
)
from pyspark.sql import DataFrame, is_remote
-from pyspark.ml import Estimator, Predictor, PredictionModel, Transformer, Model
-from pyspark.ml.base import _PredictorParams
-from pyspark.ml.param import Param, Params
-from pyspark.ml.util import _jvm
-from pyspark.ml.common import inherit_doc, _java2py, _py2java
if TYPE_CHECKING:
+ from py4j.java_gateway import JavaClass, JavaObject
+
from pyspark.ml._typing import ParamMap
- from py4j.java_gateway import JavaObject, JavaClass
T = TypeVar("T")
diff --git a/python/pyspark/mllib/_typing.pyi b/python/pyspark/mllib/_typing.pyi
index c5af46eb1d60..2f158aeacadc 100644
--- a/python/pyspark/mllib/_typing.pyi
+++ b/python/pyspark/mllib/_typing.pyi
@@ -16,16 +16,16 @@
# specific language governing permissions and limitations
# under the License.
-from typing import List, Tuple, TYPE_CHECKING, TypeVar, Union
+from typing import TYPE_CHECKING, List, Tuple, TypeVar, Union
-from typing_extensions import Literal
from numpy import ndarray
from py4j.java_gateway import JavaObject
+from typing_extensions import Literal
from pyspark.mllib.linalg import Vector
if TYPE_CHECKING:
- from scipy.sparse import spmatrix, sparray
+ from scipy.sparse import sparray, spmatrix
C = TypeVar("C", bound=type)
JavaObjectOrPickleDump = Union[JavaObject, bytearray, bytes]
diff --git a/python/pyspark/mllib/classification.py b/python/pyspark/mllib/classification.py
index bc661e9c6bb6..0d0efa9fba9b 100644
--- a/python/pyspark/mllib/classification.py
+++ b/python/pyspark/mllib/classification.py
@@ -15,25 +15,24 @@
# limitations under the License.
#
-from math import exp
import sys
import warnings
-from typing import Any, Iterable, Optional, Union, overload, TYPE_CHECKING
+from math import exp
+from typing import TYPE_CHECKING, Any, Iterable, Optional, Union, overload
import numpy
from pyspark import RDD, SparkContext, since
-from pyspark.streaming.dstream import DStream
-from pyspark.mllib.common import callMLlibFunc, _py2java, _java2py
-from pyspark.mllib.linalg import _convert_to_vector
+from pyspark.mllib.common import _java2py, _py2java, callMLlibFunc
+from pyspark.mllib.linalg import Vector, _convert_to_vector
from pyspark.mllib.regression import (
LabeledPoint,
LinearModel,
- _regression_train_wrapper,
StreamingLinearAlgorithm,
+ _regression_train_wrapper,
)
-from pyspark.mllib.util import Saveable, Loader, inherit_doc
-from pyspark.mllib.linalg import Vector
+from pyspark.mllib.util import Loader, Saveable, inherit_doc
+from pyspark.streaming.dstream import DStream
if TYPE_CHECKING:
from pyspark.mllib._typing import VectorLike
@@ -966,8 +965,9 @@ class StreamingLogisticRegressionWithSGD(StreamingLinearAlgorithm):
def _test() -> None:
import doctest
- from pyspark.sql import SparkSession
+
import pyspark.mllib.classification
+ from pyspark.sql import SparkSession
globs = pyspark.mllib.classification.__dict__.copy()
spark = (
diff --git a/python/pyspark/mllib/clustering.py b/python/pyspark/mllib/clustering.py
index c74c51859648..4a0991485f34 100644
--- a/python/pyspark/mllib/clustering.py
+++ b/python/pyspark/mllib/clustering.py
@@ -15,25 +15,26 @@
# limitations under the License.
#
-import sys
import array as pyarray
-from math import exp, log
+import sys
from collections import namedtuple
-from typing import Any, List, Optional, Tuple, TypeVar, Union, overload, TYPE_CHECKING
+from math import exp, log
+from typing import TYPE_CHECKING, Any, List, Optional, Tuple, TypeVar, Union, overload
import numpy as np
from numpy import array, random, tile
from pyspark import SparkContext, since
from pyspark.core.rdd import RDD
-from pyspark.mllib.common import JavaModelWrapper, callMLlibFunc, callJavaFunc, _py2java, _java2py
-from pyspark.mllib.linalg import SparseVector, _convert_to_vector, DenseVector # noqa: F401
+from pyspark.mllib.common import JavaModelWrapper, _java2py, _py2java, callJavaFunc, callMLlibFunc
+from pyspark.mllib.linalg import DenseVector, SparseVector, _convert_to_vector # noqa: F401
from pyspark.mllib.stat.distribution import MultivariateGaussian
-from pyspark.mllib.util import Saveable, Loader, inherit_doc, JavaLoader, JavaSaveable
+from pyspark.mllib.util import JavaLoader, JavaSaveable, Loader, Saveable, inherit_doc
from pyspark.streaming import DStream
if TYPE_CHECKING:
from py4j.java_gateway import JavaObject
+
from pyspark.mllib._typing import VectorLike
T = TypeVar("T")
@@ -1285,7 +1286,9 @@ class LDA:
def _test() -> None:
import doctest
+
import numpy
+
import pyspark.mllib.clustering
try:
diff --git a/python/pyspark/mllib/common.py b/python/pyspark/mllib/common.py
index bfab55b8552c..fc0f9121fd4e 100644
--- a/python/pyspark/mllib/common.py
+++ b/python/pyspark/mllib/common.py
@@ -15,19 +15,19 @@
# limitations under the License.
#
-from typing import Any, Callable, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Callable
if TYPE_CHECKING:
from pyspark.mllib._typing import C, JavaObjectOrPickleDump
import py4j.protocol
-from py4j.protocol import Py4JJavaError
-from py4j.java_gateway import JavaObject
from py4j.java_collections import JavaArray, JavaList
+from py4j.java_gateway import JavaObject
+from py4j.protocol import Py4JJavaError
import pyspark.core.context
from pyspark import RDD, SparkContext
-from pyspark.serializers import CPickleSerializer, AutoBatchedSerializer
+from pyspark.serializers import AutoBatchedSerializer, CPickleSerializer
from pyspark.sql import DataFrame, SparkSession
# Hack for support float('inf') in Py4j
diff --git a/python/pyspark/mllib/evaluation.py b/python/pyspark/mllib/evaluation.py
index 1990668f852a..e873fecc71b6 100644
--- a/python/pyspark/mllib/evaluation.py
+++ b/python/pyspark/mllib/evaluation.py
@@ -15,8 +15,8 @@
# limitations under the License.
#
-from typing import Generic, List, Optional, Tuple, TypeVar, Union
import sys
+from typing import Generic, List, Optional, Tuple, TypeVar, Union
from pyspark import since
from pyspark.core.rdd import RDD
@@ -668,9 +668,11 @@ class MultilabelMetrics(JavaModelWrapper):
def _test() -> None:
import doctest
+
import numpy
- from pyspark.sql import SparkSession
+
import pyspark.mllib.evaluation
+ from pyspark.sql import SparkSession
try:
# Numpy 1.14+ changed it's string format.
diff --git a/python/pyspark/mllib/feature.py b/python/pyspark/mllib/feature.py
index 685a82cf6693..e92c664fd59d 100644
--- a/python/pyspark/mllib/feature.py
+++ b/python/pyspark/mllib/feature.py
@@ -21,19 +21,18 @@ Python package for feature in MLlib.
import sys
import warnings
-from typing import Dict, Hashable, Iterable, List, Optional, Tuple, Union, overload, TYPE_CHECKING
+from typing import TYPE_CHECKING, Dict, Hashable, Iterable, List, Optional, Tuple, Union, overload
+from py4j.java_collections import JavaMap
from py4j.protocol import Py4JJavaError
from pyspark import since
-from pyspark.core.rdd import RDD
-from pyspark.mllib.common import callMLlibFunc, JavaModelWrapper
-from pyspark.mllib.linalg import Vectors, _convert_to_vector
-from pyspark.mllib.util import JavaLoader, JavaSaveable
from pyspark.core.context import SparkContext
-from pyspark.mllib.linalg import Vector
+from pyspark.core.rdd import RDD
+from pyspark.mllib.common import JavaModelWrapper, callMLlibFunc
+from pyspark.mllib.linalg import Vector, Vectors, _convert_to_vector
from pyspark.mllib.regression import LabeledPoint
-from py4j.java_collections import JavaMap
+from pyspark.mllib.util import JavaLoader, JavaSaveable
if TYPE_CHECKING:
from pyspark.mllib._typing import VectorLike
@@ -1052,6 +1051,7 @@ class ElementwiseProduct(VectorTransformer):
def _test() -> None:
import doctest
+
from pyspark.sql import SparkSession
globs = globals().copy()
diff --git a/python/pyspark/mllib/fpm.py b/python/pyspark/mllib/fpm.py
index 3f4d36884d94..cdecff471e8d 100644
--- a/python/pyspark/mllib/fpm.py
+++ b/python/pyspark/mllib/fpm.py
@@ -16,13 +16,12 @@
#
import sys
-
from typing import Any, Generic, List, NamedTuple, TypeVar
-from pyspark import since, SparkContext
-from pyspark.mllib.common import JavaModelWrapper, callMLlibFunc
-from pyspark.mllib.util import JavaSaveable, JavaLoader, inherit_doc
+from pyspark import SparkContext, since
from pyspark.core.rdd import RDD
+from pyspark.mllib.common import JavaModelWrapper, callMLlibFunc
+from pyspark.mllib.util import JavaLoader, JavaSaveable, inherit_doc
__all__ = ["FPGrowth", "FPGrowthModel", "PrefixSpan", "PrefixSpanModel"]
@@ -205,8 +204,9 @@ class PrefixSpan:
def _test() -> None:
import doctest
- from pyspark.sql import SparkSession
+
import pyspark.mllib.fpm
+ from pyspark.sql import SparkSession
globs = pyspark.mllib.fpm.__dict__.copy()
spark = SparkSession.builder.master("local[4]").appName("mllib.fpm tests").getOrCreate()
diff --git a/python/pyspark/mllib/linalg/__init__.py b/python/pyspark/mllib/linalg/__init__.py
index 363f5c9c8ab7..9f19c8fbe2f1 100644
--- a/python/pyspark/mllib/linalg/__init__.py
+++ b/python/pyspark/mllib/linalg/__init__.py
@@ -23,25 +23,25 @@ object from MLlib or pass SciPy `scipy.sparse` column vectors if
SciPy is available in their environment.
"""
-import sys
import array
import struct
+import sys
from typing import (
+ TYPE_CHECKING,
Any,
Callable,
- cast,
Dict,
Generic,
Iterable,
List,
Optional,
- overload,
Sequence,
Tuple,
Type,
TypeVar,
- TYPE_CHECKING,
Union,
+ cast,
+ overload,
)
import numpy as np
@@ -49,21 +49,22 @@ import numpy as np
from pyspark import since
from pyspark.ml import linalg as newlinalg
from pyspark.sql.types import (
- DataTypeSingleton,
- UserDefinedType,
- StructField,
- StructType,
ArrayType,
+ BooleanType,
+ ByteType,
+ DataTypeSingleton,
DoubleType,
IntegerType,
- ByteType,
- BooleanType,
+ StructField,
+ StructType,
+ UserDefinedType,
)
if TYPE_CHECKING:
- from pyspark.mllib._typing import VectorLike, NormType
from numpy.typing import ArrayLike
+ from pyspark.mllib._typing import NormType, VectorLike
+
QT = TypeVar("QT")
RT = TypeVar("RT")
@@ -1623,6 +1624,7 @@ class QRDecomposition(Generic[QT, RT]):
def _test() -> None:
import doctest
+
import numpy
try:
diff --git a/python/pyspark/mllib/linalg/distributed.py b/python/pyspark/mllib/linalg/distributed.py
index e5d27814602a..299c52d07338 100644
--- a/python/pyspark/mllib/linalg/distributed.py
+++ b/python/pyspark/mllib/linalg/distributed.py
@@ -20,13 +20,13 @@ Package for distributed linear algebra.
"""
import sys
-from typing import Any, Generic, Optional, Tuple, TypeVar, Union, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Generic, Optional, Tuple, TypeVar, Union
from py4j.java_gateway import JavaObject
from pyspark import RDD, since
-from pyspark.mllib.common import callMLlibFunc, JavaModelWrapper
-from pyspark.mllib.linalg import _convert_to_vector, DenseMatrix, Matrix, QRDecomposition, Vector
+from pyspark.mllib.common import JavaModelWrapper, callMLlibFunc
+from pyspark.mllib.linalg import DenseMatrix, Matrix, QRDecomposition, Vector, _convert_to_vector
from pyspark.mllib.stat import MultivariateStatisticalSummary
from pyspark.sql import DataFrame
from pyspark.storagelevel import StorageLevel
@@ -1633,10 +1633,12 @@ class BlockMatrix(DistributedMatrix):
def _test() -> None:
import doctest
+
import numpy
- from pyspark.sql import SparkSession
- from pyspark.mllib.linalg import Matrices
+
import pyspark.mllib.linalg.distributed
+ from pyspark.mllib.linalg import Matrices
+ from pyspark.sql import SparkSession
try:
# Numpy 1.14+ changed it's string format.
diff --git a/python/pyspark/mllib/random.py b/python/pyspark/mllib/random.py
index faafb024abb4..815f7c1f5769 100644
--- a/python/pyspark/mllib/random.py
+++ b/python/pyspark/mllib/random.py
@@ -25,9 +25,9 @@ from typing import Any, Callable, Optional
import numpy as np
-from pyspark.mllib.common import callMLlibFunc
from pyspark.core.context import SparkContext
from pyspark.core.rdd import RDD
+from pyspark.mllib.common import callMLlibFunc
from pyspark.mllib.linalg import Vector
__all__ = [
@@ -680,6 +680,7 @@ class RandomRDDs:
def _test() -> None:
import doctest
+
from pyspark.sql import SparkSession
globs = globals().copy()
diff --git a/python/pyspark/mllib/recommendation.py b/python/pyspark/mllib/recommendation.py
index f6a0d39a74c6..f305f7c36adc 100644
--- a/python/pyspark/mllib/recommendation.py
+++ b/python/pyspark/mllib/recommendation.py
@@ -371,6 +371,7 @@ class ALS:
def _test() -> None:
import doctest
+
import pyspark.mllib.recommendation
from pyspark.sql import SQLContext
diff --git a/python/pyspark/mllib/regression.py b/python/pyspark/mllib/regression.py
index a5dde25d32fc..7fb5ff055da6 100644
--- a/python/pyspark/mllib/regression.py
+++ b/python/pyspark/mllib/regression.py
@@ -18,6 +18,7 @@
import sys
import warnings
from typing import (
+ TYPE_CHECKING,
Any,
Callable,
Iterable,
@@ -27,19 +28,17 @@ from typing import (
TypeVar,
Union,
overload,
- TYPE_CHECKING,
)
import numpy as np
from pyspark import since
-from pyspark.streaming.dstream import DStream
-from pyspark.mllib.common import callMLlibFunc, _py2java, _java2py, inherit_doc
-from pyspark.mllib.linalg import _convert_to_vector
-from pyspark.mllib.util import Saveable, Loader
-from pyspark.core.rdd import RDD
from pyspark.core.context import SparkContext
-from pyspark.mllib.linalg import Vector
+from pyspark.core.rdd import RDD
+from pyspark.mllib.common import _java2py, _py2java, callMLlibFunc, inherit_doc
+from pyspark.mllib.linalg import Vector, _convert_to_vector
+from pyspark.mllib.util import Loader, Saveable
+from pyspark.streaming.dstream import DStream
if TYPE_CHECKING:
from pyspark.mllib._typing import VectorLike
@@ -1038,8 +1037,9 @@ class StreamingLinearRegressionWithSGD(StreamingLinearAlgorithm):
def _test() -> None:
import doctest
- from pyspark.sql import SparkSession
+
import pyspark.mllib.regression
+ from pyspark.sql import SparkSession
globs = pyspark.mllib.regression.__dict__.copy()
spark = SparkSession.builder.master("local[2]").appName("mllib.regression tests").getOrCreate()
diff --git a/python/pyspark/mllib/stat/KernelDensity.py b/python/pyspark/mllib/stat/KernelDensity.py
index bb03d0ef1357..ce14a99ddf95 100644
--- a/python/pyspark/mllib/stat/KernelDensity.py
+++ b/python/pyspark/mllib/stat/KernelDensity.py
@@ -20,8 +20,8 @@ from typing import Iterable, Optional
import numpy as np
from numpy import ndarray
-from pyspark.mllib.common import callMLlibFunc
from pyspark.core.rdd import RDD
+from pyspark.mllib.common import callMLlibFunc
class KernelDensity:
diff --git a/python/pyspark/mllib/stat/__init__.py b/python/pyspark/mllib/stat/__init__.py
index dc7a6da7545f..b09d56cc054b 100644
--- a/python/pyspark/mllib/stat/__init__.py
+++ b/python/pyspark/mllib/stat/__init__.py
@@ -19,10 +19,10 @@
Python package for statistical functions in MLlib.
"""
-from pyspark.mllib.stat._statistics import Statistics, MultivariateStatisticalSummary
+from pyspark.mllib.stat._statistics import MultivariateStatisticalSummary, Statistics
from pyspark.mllib.stat.distribution import MultivariateGaussian
-from pyspark.mllib.stat.test import ChiSqTestResult, KolmogorovSmirnovTestResult
from pyspark.mllib.stat.KernelDensity import KernelDensity
+from pyspark.mllib.stat.test import ChiSqTestResult, KolmogorovSmirnovTestResult
__all__ = [
"Statistics",
diff --git a/python/pyspark/mllib/stat/_statistics.py b/python/pyspark/mllib/stat/_statistics.py
index bab36039a610..633c1482adec 100644
--- a/python/pyspark/mllib/stat/_statistics.py
+++ b/python/pyspark/mllib/stat/_statistics.py
@@ -16,13 +16,13 @@
#
import sys
-from typing import cast, overload, List, Optional, TYPE_CHECKING, Union
+from typing import TYPE_CHECKING, List, Optional, Union, cast, overload
from numpy import ndarray
from py4j.java_gateway import JavaObject
from pyspark.core.rdd import RDD
-from pyspark.mllib.common import callMLlibFunc, JavaModelWrapper
+from pyspark.mllib.common import JavaModelWrapper, callMLlibFunc
from pyspark.mllib.linalg import Matrix, Vector, _convert_to_vector
from pyspark.mllib.regression import LabeledPoint
from pyspark.mllib.stat.test import ChiSqTestResult, KolmogorovSmirnovTestResult
@@ -381,7 +381,9 @@ class Statistics:
def _test() -> None:
import doctest
+
import numpy
+
from pyspark.sql import SparkSession
try:
diff --git a/python/pyspark/mllib/stat/test.py b/python/pyspark/mllib/stat/test.py
index 4afd23f8df06..20553f205352 100644
--- a/python/pyspark/mllib/stat/test.py
+++ b/python/pyspark/mllib/stat/test.py
@@ -17,7 +17,7 @@
from typing import Generic, Tuple, TypeVar
-from pyspark.mllib.common import inherit_doc, JavaModelWrapper
+from pyspark.mllib.common import JavaModelWrapper, inherit_doc
__all__ = ["ChiSqTestResult", "KolmogorovSmirnovTestResult"]
diff --git a/python/pyspark/mllib/tests/test_algorithms.py b/python/pyspark/mllib/tests/test_algorithms.py
index 2cd15a5bf8ab..872ff3ce770b 100644
--- a/python/pyspark/mllib/tests/test_algorithms.py
+++ b/python/pyspark/mllib/tests/test_algorithms.py
@@ -141,14 +141,14 @@ class ListTests(MLlibTestCase):
self.assertAlmostEqual((gmm1.weights - gmm2.weights).sum(), 0.0)
def test_classification(self):
- from pyspark.mllib.classification import LogisticRegressionWithSGD, SVMWithSGD, NaiveBayes
+ from pyspark.mllib.classification import LogisticRegressionWithSGD, NaiveBayes, SVMWithSGD
from pyspark.mllib.tree import (
DecisionTree,
DecisionTreeModel,
- RandomForest,
- RandomForestModel,
GradientBoostedTrees,
GradientBoostedTreesModel,
+ RandomForest,
+ RandomForestModel,
)
data = [
@@ -232,11 +232,11 @@ class ListTests(MLlibTestCase):
def test_regression(self):
from pyspark.mllib.regression import (
- LinearRegressionWithSGD,
LassoWithSGD,
+ LinearRegressionWithSGD,
RidgeRegressionWithSGD,
)
- from pyspark.mllib.tree import DecisionTree, RandomForest, GradientBoostedTrees
+ from pyspark.mllib.tree import DecisionTree, GradientBoostedTrees, RandomForest
data = [
LabeledPoint(-1.0, [0, -1]),
diff --git a/python/pyspark/mllib/tests/test_feature.py b/python/pyspark/mllib/tests/test_feature.py
index 1ece1d861d45..f5382cb6d977 100644
--- a/python/pyspark/mllib/tests/test_feature.py
+++ b/python/pyspark/mllib/tests/test_feature.py
@@ -17,11 +17,11 @@
from math import sqrt
-from numpy import array, abs, tile
+from numpy import abs, array, tile
-from pyspark.mllib.linalg import SparseVector, DenseVector, Vectors
+from pyspark.mllib.feature import IDF, ElementwiseProduct, HashingTF, StandardScaler, Word2Vec
+from pyspark.mllib.linalg import DenseVector, SparseVector, Vectors
from pyspark.mllib.linalg.distributed import RowMatrix
-from pyspark.mllib.feature import HashingTF, IDF, StandardScaler, ElementwiseProduct, Word2Vec
from pyspark.testing.mllibutils import MLlibTestCase
diff --git a/python/pyspark/mllib/tests/test_linalg.py b/python/pyspark/mllib/tests/test_linalg.py
index 00f5f1792cf9..6dba3f63d82d 100644
--- a/python/pyspark/mllib/tests/test_linalg.py
+++ b/python/pyspark/mllib/tests/test_linalg.py
@@ -18,24 +18,24 @@
import array as pyarray
import unittest
-from numpy import array, array_equal, zeros, arange, tile, ones, inf
+from numpy import arange, array, array_equal, inf, ones, tile, zeros
import pyspark.ml.linalg as newlinalg
-from pyspark.serializers import CPickleSerializer
from pyspark.mllib.linalg import (
- Vector,
- SparseVector,
- DenseVector,
- VectorUDT,
- _convert_to_vector,
DenseMatrix,
- SparseMatrix,
- Vectors,
+ DenseVector,
Matrices,
MatrixUDT,
+ SparseMatrix,
+ SparseVector,
+ Vector,
+ Vectors,
+ VectorUDT,
+ _convert_to_vector,
)
-from pyspark.mllib.linalg.distributed import RowMatrix, IndexedRowMatrix, IndexedRow
+from pyspark.mllib.linalg.distributed import IndexedRow, IndexedRowMatrix, RowMatrix
from pyspark.mllib.regression import LabeledPoint
+from pyspark.serializers import CPickleSerializer
from pyspark.sql import Row
from pyspark.testing.mllibutils import MLlibTestCase
from pyspark.testing.utils import have_scipy
@@ -591,7 +591,7 @@ class SciPyTests(MLlibTestCase):
self.assertEqual(clusters.predict(data[2]), clusters.predict(data[3]))
def test_classification(self):
- from pyspark.mllib.classification import LogisticRegressionWithSGD, SVMWithSGD, NaiveBayes
+ from pyspark.mllib.classification import LogisticRegressionWithSGD, NaiveBayes, SVMWithSGD
from pyspark.mllib.tree import DecisionTree
data = [
@@ -632,8 +632,8 @@ class SciPyTests(MLlibTestCase):
def test_regression(self):
from pyspark.mllib.regression import (
- LinearRegressionWithSGD,
LassoWithSGD,
+ LinearRegressionWithSGD,
RidgeRegressionWithSGD,
)
from pyspark.mllib.tree import DecisionTree
diff --git a/python/pyspark/mllib/tests/test_stat.py b/python/pyspark/mllib/tests/test_stat.py
index 6d1f4851e897..54cda5342ae2 100644
--- a/python/pyspark/mllib/tests/test_stat.py
+++ b/python/pyspark/mllib/tests/test_stat.py
@@ -19,11 +19,11 @@ import array as pyarray
from numpy import array
-from pyspark.mllib.linalg import Vectors, Matrices
+from pyspark.errors import IllegalArgumentException
+from pyspark.mllib.linalg import Matrices, Vectors
from pyspark.mllib.random import RandomRDDs
from pyspark.mllib.regression import LabeledPoint
from pyspark.mllib.stat import Statistics
-from pyspark.errors import IllegalArgumentException
from pyspark.testing.mllibutils import MLlibTestCase
diff --git a/python/pyspark/mllib/tests/test_streaming_algorithms.py b/python/pyspark/mllib/tests/test_streaming_algorithms.py
index 35e27056a2ff..ad61dcd872cc 100644
--- a/python/pyspark/mllib/tests/test_streaming_algorithms.py
+++ b/python/pyspark/mllib/tests/test_streaming_algorithms.py
@@ -18,12 +18,12 @@
import os
import unittest
-from numpy import array, random, exp, dot, all, mean, abs
+from numpy import abs, all, array, dot, exp, mean, random
from numpy import sum as array_sum
from pyspark import SparkContext
-from pyspark.mllib.clustering import StreamingKMeans, StreamingKMeansModel
from pyspark.mllib.classification import StreamingLogisticRegressionWithSGD
+from pyspark.mllib.clustering import StreamingKMeans, StreamingKMeansModel
from pyspark.mllib.linalg import Vectors
from pyspark.mllib.regression import LabeledPoint, StreamingLinearRegressionWithSGD
from pyspark.mllib.util import LinearDataGenerator
diff --git a/python/pyspark/mllib/tests/test_util.py b/python/pyspark/mllib/tests/test_util.py
index 6ff165e8dcbd..8e68a589684c 100644
--- a/python/pyspark/mllib/tests/test_util.py
+++ b/python/pyspark/mllib/tests/test_util.py
@@ -19,10 +19,9 @@ import os
import tempfile
from pyspark.mllib.common import _to_java_object_rdd
-from pyspark.mllib.util import LinearDataGenerator
-from pyspark.mllib.util import MLUtils
-from pyspark.mllib.linalg import SparseVector, DenseVector, Vectors
+from pyspark.mllib.linalg import DenseVector, SparseVector, Vectors
from pyspark.mllib.random import RandomRDDs
+from pyspark.mllib.util import LinearDataGenerator, MLUtils
from pyspark.testing.mllibutils import MLlibTestCase
diff --git a/python/pyspark/mllib/tree.py b/python/pyspark/mllib/tree.py
index 2f3a85d83ad8..086fa410b18f 100644
--- a/python/pyspark/mllib/tree.py
+++ b/python/pyspark/mllib/tree.py
@@ -15,16 +15,16 @@
# limitations under the License.
#
-import sys
import random
+import sys
+from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union, overload
from pyspark import since
-from pyspark.mllib.common import callMLlibFunc, inherit_doc, JavaModelWrapper
+from pyspark.core.rdd import RDD
+from pyspark.mllib.common import JavaModelWrapper, callMLlibFunc, inherit_doc
from pyspark.mllib.linalg import _convert_to_vector
from pyspark.mllib.regression import LabeledPoint
from pyspark.mllib.util import JavaLoader, JavaSaveable
-from typing import Dict, Optional, Tuple, Union, overload, TYPE_CHECKING
-from pyspark.core.rdd import RDD
if TYPE_CHECKING:
from pyspark.mllib._typing import VectorLike
diff --git a/python/pyspark/mllib/util.py b/python/pyspark/mllib/util.py
index 699909d430ea..5c1bea3e2bf0 100644
--- a/python/pyspark/mllib/util.py
+++ b/python/pyspark/mllib/util.py
@@ -17,25 +17,25 @@
import sys
from functools import reduce
+from typing import TYPE_CHECKING, Generic, Iterable, List, Optional, Tuple, Type, TypeVar, cast
import numpy as np
from pyspark import since
-from pyspark.mllib.common import callMLlibFunc, inherit_doc
-from pyspark.mllib.linalg import Vectors, SparseVector, _convert_to_vector
-from pyspark.sql import DataFrame
-from typing import Generic, Iterable, List, Optional, Tuple, Type, TypeVar, cast, TYPE_CHECKING
from pyspark.core.context import SparkContext
-from pyspark.mllib.linalg import Vector
from pyspark.core.rdd import RDD
+from pyspark.mllib.common import callMLlibFunc, inherit_doc
+from pyspark.mllib.linalg import SparseVector, Vector, Vectors, _convert_to_vector
+from pyspark.sql import DataFrame
T = TypeVar("T")
L = TypeVar("L", bound="Loader")
JL = TypeVar("JL", bound="JavaLoader")
if TYPE_CHECKING:
- from pyspark.mllib._typing import VectorLike
from py4j.java_gateway import JavaObject
+
+ from pyspark.mllib._typing import VectorLike
from pyspark.mllib.regression import LabeledPoint
@@ -636,6 +636,7 @@ class LinearDataGenerator:
def _test() -> None:
import doctest
+
from pyspark.sql import SparkSession
globs = globals().copy()
diff --git a/python/pyspark/pandas/__init__.py b/python/pyspark/pandas/__init__.py
index bb05ec3bf857..aa2b2701acff 100644
--- a/python/pyspark/pandas/__init__.py
+++ b/python/pyspark/pandas/__init__.py
@@ -40,13 +40,13 @@ except ImportError as e:
raise
from pyspark.pandas.frame import DataFrame
+from pyspark.pandas.groupby import NamedAgg
from pyspark.pandas.indexes.base import Index
from pyspark.pandas.indexes.category import CategoricalIndex
from pyspark.pandas.indexes.datetimes import DatetimeIndex
from pyspark.pandas.indexes.multi import MultiIndex
from pyspark.pandas.indexes.timedelta import TimedeltaIndex
from pyspark.pandas.series import Series
-from pyspark.pandas.groupby import NamedAgg
__all__ = [ # noqa: F405
"read_csv",
@@ -80,8 +80,8 @@ __all__ = [ # noqa: F405
def _auto_patch_spark() -> None:
- import os
import logging
+ import os
# Attach a usage logger. 'KOALAS_USAGE_LOGGER' is legacy, and it's for compatibility.
logger_module = os.getenv("PYSPARK_PANDAS_USAGE_LOGGER", os.getenv("KOALAS_USAGE_LOGGER", ""))
@@ -129,7 +129,7 @@ _auto_patch_spark()
_auto_patch_pandas()
# Import after the usage logger is attached.
-from pyspark.pandas.config import get_option, options, option_context, reset_option, set_option
+from pyspark.pandas.config import get_option, option_context, options, reset_option, set_option
from pyspark.pandas.namespace import * # noqa: F403
from pyspark.pandas.sql_formatter import sql
diff --git a/python/pyspark/pandas/_typing.py b/python/pyspark/pandas/_typing.py
index 51d1233fae25..6a18b73acb71 100644
--- a/python/pyspark/pandas/_typing.py
+++ b/python/pyspark/pandas/_typing.py
@@ -16,7 +16,7 @@
#
import datetime
import decimal
-from typing import Any, Tuple, TypeVar, Union, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Tuple, TypeVar, Union
import numpy as np
from pandas.api.extensions import ExtensionDtype
diff --git a/python/pyspark/pandas/accessors.py b/python/pyspark/pandas/accessors.py
index a0edf34da585..86c64a7277e5 100644
--- a/python/pyspark/pandas/accessors.py
+++ b/python/pyspark/pandas/accessors.py
@@ -19,32 +19,32 @@ pandas-on-Spark specific features.
"""
import inspect
-from typing import Any, Callable, Optional, Tuple, Union, TYPE_CHECKING, cast, List
from types import FunctionType
+from typing import TYPE_CHECKING, Any, Callable, List, Optional, Tuple, Union, cast
import numpy as np # noqa: F401
import pandas as pd
-from pyspark.sql import functions as F
-from pyspark.sql.functions import pandas_udf
-from pyspark.sql.types import DataType, LongType, StructField, StructType
from pyspark.pandas._typing import DataFrameOrSeries, Name
from pyspark.pandas.internal import (
- InternalField,
- InternalFrame,
- SPARK_INDEX_NAME_FORMAT,
SPARK_DEFAULT_SERIES_NAME,
+ SPARK_INDEX_NAME_FORMAT,
SPARK_INDEX_NAME_PATTERN,
+ InternalField,
+ InternalFrame,
)
-from pyspark.pandas.typedef import infer_return_type, DataFrameType, ScalarType, SeriesType
+from pyspark.pandas.typedef import DataFrameType, ScalarType, SeriesType, infer_return_type
from pyspark.pandas.utils import (
- is_name_like_value,
is_name_like_tuple,
+ is_name_like_value,
+ log_advice,
name_like_string,
scol_for,
verify_temp_column_name,
- log_advice,
)
+from pyspark.sql import functions as F
+from pyspark.sql.functions import pandas_udf
+from pyspark.sql.types import DataType, LongType, StructField, StructType
if TYPE_CHECKING:
from pyspark.pandas.frame import DataFrame
@@ -328,9 +328,9 @@ class PandasOnSparkFrameMethods:
"""
# TODO: codes here partially duplicate `DataFrame.apply`. Can we deduplicate?
- from pyspark.pandas.groupby import GroupBy
- from pyspark.pandas.frame import DataFrame
from pyspark import pandas as ps
+ from pyspark.pandas.frame import DataFrame
+ from pyspark.pandas.groupby import GroupBy
if not isinstance(func, FunctionType):
assert callable(func), "the first argument should be a callable function."
@@ -564,10 +564,10 @@ class PandasOnSparkFrameMethods:
2 12
Name: B, dtype: int64
"""
- from pyspark.pandas.groupby import GroupBy
+ from pyspark import pandas as ps
from pyspark.pandas.frame import DataFrame
+ from pyspark.pandas.groupby import GroupBy
from pyspark.pandas.series import first_series
- from pyspark import pandas as ps
assert callable(func), "the first argument should be a callable function."
spec = inspect.getfullargspec(func)
@@ -899,9 +899,9 @@ class PandasOnSparkSeriesMethods:
def _transform_batch(
self, func: Callable[..., pd.Series], return_type: Optional[Union[SeriesType, ScalarType]]
) -> "Series":
+ from pyspark import pandas as ps
from pyspark.pandas.groupby import GroupBy
from pyspark.pandas.series import Series, first_series
- from pyspark import pandas as ps
if not isinstance(func, FunctionType):
f = func
@@ -960,11 +960,12 @@ class PandasOnSparkSeriesMethods:
def _test() -> None:
- import os
import doctest
+ import os
import sys
- from pyspark.sql import SparkSession
+
import pyspark.pandas.accessors
+ from pyspark.sql import SparkSession
os.chdir(os.environ["SPARK_HOME"])
diff --git a/python/pyspark/pandas/base.py b/python/pyspark/pandas/base.py
index db3fb53ce855..eca0cf074464 100644
--- a/python/pyspark/pandas/base.py
+++ b/python/pyspark/pandas/base.py
@@ -21,56 +21,56 @@ Base and utility classes for pandas-on-Spark objects.
import warnings
from abc import ABCMeta, abstractmethod
-from functools import wraps, partial
+from functools import partial, wraps
from itertools import chain
-from typing import Any, Callable, ClassVar, Optional, Sequence, Tuple, Union, cast, TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, Sequence, Tuple, Union, cast
import numpy as np
import pandas as pd
-from pandas.api.types import is_list_like, CategoricalDtype
+from pandas.api.types import CategoricalDtype, is_list_like
-from pyspark.sql import functions as F, Column, Window
-from pyspark.sql.types import (
- BinaryType,
- BooleanType,
- CharType,
- DataType,
- DateType,
- DayTimeIntervalType,
- LongType,
- NumericType,
- StringType,
- TimestampNTZType,
- TimestampType,
- TimeType,
- VarcharType,
-)
from pyspark import pandas as ps # For running doctests and reference resolution in PyCharm.
from pyspark.pandas._typing import Axis, Dtype, IndexOpsLike, Label, SeriesOrIndex
from pyspark.pandas.config import get_option, option_context
+from pyspark.pandas.frame import DataFrame
from pyspark.pandas.internal import (
- InternalField,
- InternalFrame,
NATURAL_ORDER_COLUMN_NAME,
SPARK_DEFAULT_INDEX_NAME,
+ InternalField,
+ InternalFrame,
)
from pyspark.pandas.spark.accessors import SparkIndexOpsMethods
from pyspark.pandas.typedef.typehints import handle_dtype_as_extension_dtype
from pyspark.pandas.utils import (
+ ERROR_MESSAGE_CANNOT_COMBINE,
ansi_mode_context,
combine_frames,
same_anchor,
scol_for,
validate_axis,
- ERROR_MESSAGE_CANNOT_COMBINE,
)
-from pyspark.pandas.frame import DataFrame
+from pyspark.sql import Column, Window
+from pyspark.sql import functions as F
+from pyspark.sql.types import (
+ BinaryType,
+ BooleanType,
+ CharType,
+ DataType,
+ DateType,
+ DayTimeIntervalType,
+ LongType,
+ NumericType,
+ StringType,
+ TimestampNTZType,
+ TimestampType,
+ TimeType,
+ VarcharType,
+)
if TYPE_CHECKING:
- from pyspark.sql._typing import ColumnOrName
-
from pyspark.pandas.data_type_ops.base import DataTypeOps
from pyspark.pandas.series import Series
+ from pyspark.sql._typing import ColumnOrName
def should_alignment_for_column_op(self: SeriesOrIndex, other: SeriesOrIndex) -> bool:
@@ -1455,8 +1455,8 @@ class IndexOpsMixin(object, metaclass=ABCMeta):
3 1
Name: count, dtype: int64
"""
- from pyspark.pandas.series import first_series
from pyspark.pandas.indexes.multi import MultiIndex
+ from pyspark.pandas.series import first_series
if bins is not None:
raise NotImplementedError("value_counts currently does not support bins")
@@ -1793,11 +1793,12 @@ class IndexOpsMixin(object, metaclass=ABCMeta):
def _test() -> None:
- import os
import doctest
+ import os
import sys
- from pyspark.sql import SparkSession
+
import pyspark.pandas.base
+ from pyspark.sql import SparkSession
os.chdir(os.environ["SPARK_HOME"])
diff --git a/python/pyspark/pandas/categorical.py b/python/pyspark/pandas/categorical.py
index 36f10f3426a1..90ef691bca0d 100644
--- a/python/pyspark/pandas/categorical.py
+++ b/python/pyspark/pandas/categorical.py
@@ -14,7 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from typing import Any, Callable, List, Optional, Union, TYPE_CHECKING, cast
+from typing import TYPE_CHECKING, Any, Callable, List, Optional, Union, cast
import pandas as pd
from pandas.api.types import (
@@ -23,8 +23,8 @@ from pandas.api.types import (
is_list_like,
)
-from pyspark.pandas.internal import InternalField
from pyspark.pandas.data_type_ops.categorical_ops import _to_cat
+from pyspark.pandas.internal import InternalField
from pyspark.sql import functions as F
from pyspark.sql.types import StructField
@@ -769,11 +769,12 @@ class CategoricalAccessor:
def _test() -> None:
- import os
import doctest
+ import os
import sys
- from pyspark.sql import SparkSession
+
import pyspark.pandas.categorical
+ from pyspark.sql import SparkSession
os.chdir(os.environ["SPARK_HOME"])
diff --git a/python/pyspark/pandas/config.py b/python/pyspark/pandas/config.py
index 8db649997db6..9161810adde1 100644
--- a/python/pyspark/pandas/config.py
+++ b/python/pyspark/pandas/config.py
@@ -19,13 +19,13 @@
Infrastructure of options for pandas-on-Spark.
"""
-from contextlib import contextmanager
import json
-from typing import Any, Callable, Dict, Iterator, List, Tuple, Union, Optional
+from contextlib import contextmanager
+from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple, Union
from pyspark._globals import _NoValue, _NoValueType
-from pyspark.sql.session import SparkSession
from pyspark.pandas.utils import default_session
+from pyspark.sql.session import SparkSession
__all__ = ["get_option", "set_option", "reset_option", "options", "option_context"]
@@ -554,11 +554,12 @@ options = DictWrapper(_options_dict)
def _test() -> None:
- import os
import doctest
+ import os
import sys
- from pyspark.sql import SparkSession
+
import pyspark.pandas.config
+ from pyspark.sql import SparkSession
os.chdir(os.environ["SPARK_HOME"])
diff --git a/python/pyspark/pandas/correlation.py b/python/pyspark/pandas/correlation.py
index cf9d38961216..eb5a172758a4 100644
--- a/python/pyspark/pandas/correlation.py
+++ b/python/pyspark/pandas/correlation.py
@@ -17,9 +17,10 @@
from typing import List
-from pyspark.sql import DataFrame as SparkDataFrame, functions as F
+from pyspark.pandas.utils import is_ansi_mode_enabled, verify_temp_column_name
+from pyspark.sql import DataFrame as SparkDataFrame
+from pyspark.sql import functions as F
from pyspark.sql.window import Window
-from pyspark.pandas.utils import verify_temp_column_name, is_ansi_mode_enabled
CORRELATION_VALUE_1_COLUMN = "__correlation_value_1_input__"
CORRELATION_VALUE_2_COLUMN = "__correlation_value_2_input__"
diff --git a/python/pyspark/pandas/data_type_ops/base.py b/python/pyspark/pandas/data_type_ops/base.py
index 72ce6cf7d930..b7b8d187fc2e 100644
--- a/python/pyspark/pandas/data_type_ops/base.py
+++ b/python/pyspark/pandas/data_type_ops/base.py
@@ -17,15 +17,25 @@
import numbers
from abc import ABCMeta
-from typing import Any, Optional, Union, cast
from itertools import chain
+from typing import Any, Optional, Union, cast
import numpy as np
import pandas as pd
from pandas.api.types import CategoricalDtype
from pandas.core.dtypes.common import is_numeric_dtype
-from pyspark.sql import functions as F, Column as PySparkColumn
+from pyspark.pandas._typing import Dtype, IndexOpsLike, SeriesOrIndex
+from pyspark.pandas.typedef.typehints import (
+ extension_dtypes_available,
+ extension_float_dtypes_available,
+ extension_object_dtypes_available,
+ handle_dtype_as_extension_dtype,
+ is_str_dtype,
+ spark_type_to_pandas_dtype,
+)
+from pyspark.sql import Column as PySparkColumn
+from pyspark.sql import functions as F
from pyspark.sql.types import (
ArrayType,
BinaryType,
@@ -41,19 +51,10 @@ from pyspark.sql.types import (
NumericType,
StringType,
StructType,
- TimestampType,
TimestampNTZType,
+ TimestampType,
UserDefinedType,
)
-from pyspark.pandas._typing import Dtype, IndexOpsLike, SeriesOrIndex
-from pyspark.pandas.typedef.typehints import (
- extension_dtypes_available,
- extension_float_dtypes_available,
- extension_object_dtypes_available,
- handle_dtype_as_extension_dtype,
- is_str_dtype,
- spark_type_to_pandas_dtype,
-)
if extension_dtypes_available:
from pandas import Int8Dtype, Int16Dtype, Int32Dtype, Int64Dtype
@@ -115,9 +116,10 @@ def _should_return_all_false(left: IndexOpsLike, right: Any) -> bool:
Determine if binary comparison should short-circuit to all False,
based on incompatible dtypes: non-numeric vs. numeric (including bools).
"""
- from pyspark.pandas.base import IndexOpsMixin
from pandas.api.types import is_list_like
+ from pyspark.pandas.base import IndexOpsMixin
+
def are_both_numeric(left_dtype: Dtype, right_dtype: Dtype) -> bool:
return is_numeric_dtype(left_dtype) and is_numeric_dtype(right_dtype)
@@ -262,11 +264,11 @@ class DataTypeOps(object, metaclass=ABCMeta):
def __new__(cls, dtype: Dtype, spark_type: DataType) -> "DataTypeOps":
from pyspark.pandas.data_type_ops.binary_ops import BinaryOps
- from pyspark.pandas.data_type_ops.boolean_ops import BooleanOps, BooleanExtensionOps
+ from pyspark.pandas.data_type_ops.boolean_ops import BooleanExtensionOps, BooleanOps
from pyspark.pandas.data_type_ops.categorical_ops import CategoricalOps
from pyspark.pandas.data_type_ops.complex_ops import ArrayOps, MapOps, StructOps
from pyspark.pandas.data_type_ops.date_ops import DateOps
- from pyspark.pandas.data_type_ops.datetime_ops import DatetimeOps, DatetimeNTZOps
+ from pyspark.pandas.data_type_ops.datetime_ops import DatetimeNTZOps, DatetimeOps
from pyspark.pandas.data_type_ops.null_ops import NullOps
from pyspark.pandas.data_type_ops.num_ops import (
DecimalOps,
@@ -275,7 +277,7 @@ class DataTypeOps(object, metaclass=ABCMeta):
IntegralExtensionOps,
IntegralOps,
)
- from pyspark.pandas.data_type_ops.string_ops import StringOps, StringExtensionOps
+ from pyspark.pandas.data_type_ops.string_ops import StringExtensionOps, StringOps
from pyspark.pandas.data_type_ops.timedelta_ops import TimedeltaOps
from pyspark.pandas.data_type_ops.udt_ops import UDTOps
@@ -434,9 +436,9 @@ class DataTypeOps(object, metaclass=ABCMeta):
return cast(SeriesOrIndex, left_scol)
if isinstance(right, (list, tuple)):
- from pyspark.pandas.series import first_series, scol_for
from pyspark.pandas.frame import DataFrame
from pyspark.pandas.internal import NATURAL_ORDER_COLUMN_NAME, InternalField
+ from pyspark.pandas.series import first_series, scol_for
if len(left) != len(right):
raise ValueError("Lengths must be equal")
@@ -526,7 +528,7 @@ class DataTypeOps(object, metaclass=ABCMeta):
return column_op(PySparkColumn.__eq__)(left, right)
def ne(self, left: IndexOpsLike, right: Any) -> SeriesOrIndex:
- from pyspark.pandas.base import column_op, IndexOpsMixin
+ from pyspark.pandas.base import IndexOpsMixin, column_op
_sanitize_list_like(right)
diff --git a/python/pyspark/pandas/data_type_ops/binary_ops.py b/python/pyspark/pandas/data_type_ops/binary_ops.py
index f528d3e9ae2a..26eea9bfe622 100644
--- a/python/pyspark/pandas/data_type_ops/binary_ops.py
+++ b/python/pyspark/pandas/data_type_ops/binary_ops.py
@@ -19,8 +19,8 @@ from typing import Any, Union, cast
from pandas.api.types import CategoricalDtype
-from pyspark.pandas.base import column_op, IndexOpsMixin
from pyspark.pandas._typing import Dtype, IndexOpsLike, SeriesOrIndex
+from pyspark.pandas.base import IndexOpsMixin, column_op
from pyspark.pandas.data_type_ops.base import (
DataTypeOps,
_as_categorical_type,
diff --git a/python/pyspark/pandas/data_type_ops/boolean_ops.py b/python/pyspark/pandas/data_type_ops/boolean_ops.py
index c52bc9b5051c..b34fafa273d5 100644
--- a/python/pyspark/pandas/data_type_ops/boolean_ops.py
+++ b/python/pyspark/pandas/data_type_ops/boolean_ops.py
@@ -22,19 +22,20 @@ import pandas as pd
from pandas.api.types import CategoricalDtype, is_integer_dtype
from pandas.core.dtypes.common import is_numeric_dtype
-from pyspark.pandas.base import column_op, IndexOpsMixin
-from pyspark.pandas.config import get_option
+from pyspark.errors import PySparkValueError
from pyspark.pandas._typing import Dtype, IndexOpsLike, SeriesOrIndex
+from pyspark.pandas.base import IndexOpsMixin, column_op
+from pyspark.pandas.config import get_option
from pyspark.pandas.data_type_ops.base import (
DataTypeOps,
- is_valid_operand_for_numeric_arithmetic,
- transform_boolean_operand_to_numeric,
_as_bool_type,
_as_categorical_type,
_as_other_type,
- _sanitize_list_like,
- _is_valid_for_logical_operator,
_is_boolean_type,
+ _is_valid_for_logical_operator,
+ _sanitize_list_like,
+ is_valid_operand_for_numeric_arithmetic,
+ transform_boolean_operand_to_numeric,
)
from pyspark.pandas.typedef.typehints import (
as_spark_type,
@@ -43,9 +44,9 @@ from pyspark.pandas.typedef.typehints import (
pandas_on_spark_type,
)
from pyspark.pandas.utils import is_ansi_mode_enabled
-from pyspark.sql import functions as F, Column as PySparkColumn
+from pyspark.sql import Column as PySparkColumn
+from pyspark.sql import functions as F
from pyspark.sql.types import BooleanType, StringType
-from pyspark.errors import PySparkValueError
class BooleanOps(DataTypeOps):
diff --git a/python/pyspark/pandas/data_type_ops/categorical_ops.py b/python/pyspark/pandas/data_type_ops/categorical_ops.py
index 3a977f418641..4253cf94c06e 100644
--- a/python/pyspark/pandas/data_type_ops/categorical_ops.py
+++ b/python/pyspark/pandas/data_type_ops/categorical_ops.py
@@ -16,15 +16,15 @@
#
from itertools import chain
-from typing import cast, Any, Sequence, Union
+from typing import Any, Sequence, Union, cast
-import pandas as pd
import numpy as np
-from pandas.api.types import is_list_like, CategoricalDtype
+import pandas as pd
+from pandas.api.types import CategoricalDtype, is_list_like
from pyspark.pandas._typing import Dtype, IndexOpsLike, SeriesOrIndex
from pyspark.pandas.base import IndexOpsMixin
-from pyspark.pandas.data_type_ops.base import _sanitize_list_like, DataTypeOps
+from pyspark.pandas.data_type_ops.base import DataTypeOps, _sanitize_list_like
from pyspark.pandas.typedef import pandas_on_spark_type
from pyspark.sql import functions as F
from pyspark.sql.utils import pyspark_column_op
diff --git a/python/pyspark/pandas/data_type_ops/complex_ops.py b/python/pyspark/pandas/data_type_ops/complex_ops.py
index 415301e400e9..7971516324a6 100644
--- a/python/pyspark/pandas/data_type_ops/complex_ops.py
+++ b/python/pyspark/pandas/data_type_ops/complex_ops.py
@@ -20,7 +20,7 @@ from typing import Any, Union, cast
from pandas.api.types import CategoricalDtype
from pyspark.pandas._typing import Dtype, IndexOpsLike, SeriesOrIndex
-from pyspark.pandas.base import column_op, IndexOpsMixin
+from pyspark.pandas.base import IndexOpsMixin, column_op
from pyspark.pandas.data_type_ops.base import (
DataTypeOps,
_as_bool_type,
@@ -30,7 +30,8 @@ from pyspark.pandas.data_type_ops.base import (
_sanitize_list_like,
)
from pyspark.pandas.typedef import pandas_on_spark_type
-from pyspark.sql import functions as F, Column
+from pyspark.sql import Column
+from pyspark.sql import functions as F
from pyspark.sql.types import ArrayType, BooleanType, NumericType, StringType
diff --git a/python/pyspark/pandas/data_type_ops/date_ops.py b/python/pyspark/pandas/data_type_ops/date_ops.py
index 9a0b82de6ce8..d4e1dce693cd 100644
--- a/python/pyspark/pandas/data_type_ops/date_ops.py
+++ b/python/pyspark/pandas/data_type_ops/date_ops.py
@@ -23,10 +23,8 @@ import numpy as np
import pandas as pd
from pandas.api.types import CategoricalDtype
-from pyspark.sql import functions as F, Column as PySparkColumn
-from pyspark.sql.types import BooleanType, DateType, StringType
from pyspark.pandas._typing import Dtype, IndexOpsLike, SeriesOrIndex
-from pyspark.pandas.base import column_op, IndexOpsMixin
+from pyspark.pandas.base import IndexOpsMixin, column_op
from pyspark.pandas.data_type_ops.base import (
DataTypeOps,
_as_categorical_type,
@@ -35,6 +33,9 @@ from pyspark.pandas.data_type_ops.base import (
_sanitize_list_like,
)
from pyspark.pandas.typedef import pandas_on_spark_type
+from pyspark.sql import Column as PySparkColumn
+from pyspark.sql import functions as F
+from pyspark.sql.types import BooleanType, DateType, StringType
class DateOps(DataTypeOps):
diff --git a/python/pyspark/pandas/data_type_ops/datetime_ops.py b/python/pyspark/pandas/data_type_ops/datetime_ops.py
index d407b8512b4e..d90ce08b0dfa 100644
--- a/python/pyspark/pandas/data_type_ops/datetime_ops.py
+++ b/python/pyspark/pandas/data_type_ops/datetime_ops.py
@@ -24,18 +24,7 @@ import pandas as pd
from pandas.api.types import CategoricalDtype
from pyspark.loose_version import LooseVersion
-from pyspark.sql import Column, functions as F
-from pyspark.sql.types import (
- BooleanType,
- LongType,
- StringType,
- TimestampType,
- TimestampNTZType,
- NumericType,
-)
-from pyspark.sql.utils import pyspark_column_op
from pyspark.pandas._typing import Dtype, IndexOpsLike, SeriesOrIndex
-from pyspark.sql.internal import InternalFunction as SF
from pyspark.pandas.base import IndexOpsMixin
from pyspark.pandas.data_type_ops.base import (
DataTypeOps,
@@ -45,6 +34,18 @@ from pyspark.pandas.data_type_ops.base import (
_sanitize_list_like,
)
from pyspark.pandas.typedef import pandas_on_spark_type
+from pyspark.sql import Column
+from pyspark.sql import functions as F
+from pyspark.sql.internal import InternalFunction as SF
+from pyspark.sql.types import (
+ BooleanType,
+ LongType,
+ NumericType,
+ StringType,
+ TimestampNTZType,
+ TimestampType,
+)
+from pyspark.sql.utils import pyspark_column_op
class DatetimeOps(DataTypeOps):
diff --git a/python/pyspark/pandas/data_type_ops/null_ops.py b/python/pyspark/pandas/data_type_ops/null_ops.py
index 1c3296011b61..955a2422f5d1 100644
--- a/python/pyspark/pandas/data_type_ops/null_ops.py
+++ b/python/pyspark/pandas/data_type_ops/null_ops.py
@@ -19,7 +19,8 @@ from typing import Any, Union
from pandas.api.types import CategoricalDtype, is_list_like
-from pyspark.pandas._typing import Dtype, IndexOpsLike
+from pyspark.pandas._typing import Dtype, IndexOpsLike, SeriesOrIndex
+from pyspark.pandas.base import IndexOpsMixin
from pyspark.pandas.data_type_ops.base import (
DataTypeOps,
_as_bool_type,
@@ -28,11 +29,9 @@ from pyspark.pandas.data_type_ops.base import (
_as_string_type,
_sanitize_list_like,
)
-from pyspark.pandas._typing import SeriesOrIndex
from pyspark.pandas.typedef import pandas_on_spark_type
from pyspark.sql.types import BooleanType, StringType
from pyspark.sql.utils import pyspark_column_op
-from pyspark.pandas.base import IndexOpsMixin
class NullOps(DataTypeOps):
diff --git a/python/pyspark/pandas/data_type_ops/num_ops.py b/python/pyspark/pandas/data_type_ops/num_ops.py
index e2a3180331eb..8baeacf3b018 100644
--- a/python/pyspark/pandas/data_type_ops/num_ops.py
+++ b/python/pyspark/pandas/data_type_ops/num_ops.py
@@ -17,34 +17,35 @@
import decimal
import numbers
-from typing import Any, Union, Callable, cast
+from typing import Any, Callable, Union, cast
import numpy as np
import pandas as pd
from pandas.api.types import (
+ CategoricalDtype,
is_bool_dtype,
- is_integer_dtype,
is_float_dtype,
- is_numeric_dtype,
- CategoricalDtype,
+ is_integer_dtype,
is_list_like,
+ is_numeric_dtype,
)
+from pyspark.errors import PySparkValueError
from pyspark.pandas._typing import Dtype, IndexOpsLike, SeriesOrIndex
-from pyspark.pandas.base import column_op, IndexOpsMixin, numpy_column_op
+from pyspark.pandas.base import IndexOpsMixin, column_op, numpy_column_op
from pyspark.pandas.config import get_option
from pyspark.pandas.data_type_ops.base import (
DataTypeOps,
- is_valid_operand_for_numeric_arithmetic,
- transform_boolean_operand_to_numeric,
_as_bool_type,
_as_categorical_type,
_as_other_type,
_as_string_type,
- _sanitize_list_like,
- _is_valid_for_logical_operator,
_is_boolean_type,
+ _is_valid_for_logical_operator,
+ _sanitize_list_like,
_should_return_all_false,
+ is_valid_operand_for_numeric_arithmetic,
+ transform_boolean_operand_to_numeric,
)
from pyspark.pandas.typedef.typehints import (
as_spark_type,
@@ -52,14 +53,14 @@ from pyspark.pandas.typedef.typehints import (
pandas_on_spark_type,
)
from pyspark.pandas.utils import is_ansi_mode_enabled
-from pyspark.sql import functions as F, Column as PySparkColumn
+from pyspark.sql import Column as PySparkColumn
+from pyspark.sql import functions as F
from pyspark.sql.types import (
BooleanType,
DataType,
DecimalType,
StringType,
)
-from pyspark.errors import PySparkValueError
# For Supporting Spark Connect
from pyspark.sql.utils import pyspark_column_op
diff --git a/python/pyspark/pandas/data_type_ops/string_ops.py b/python/pyspark/pandas/data_type_ops/string_ops.py
index c416d03a9c8f..78d2778038a8 100644
--- a/python/pyspark/pandas/data_type_ops/string_ops.py
+++ b/python/pyspark/pandas/data_type_ops/string_ops.py
@@ -22,11 +22,8 @@ import pandas as pd
from pandas.api.types import CategoricalDtype
from pyspark.loose_version import LooseVersion
-from pyspark.sql import functions as F
-from pyspark.sql.types import IntegralType, StringType
-from pyspark.sql.utils import pyspark_column_op
from pyspark.pandas._typing import Dtype, IndexOpsLike, SeriesOrIndex
-from pyspark.pandas.base import column_op, IndexOpsMixin
+from pyspark.pandas.base import IndexOpsMixin, column_op
from pyspark.pandas.data_type_ops.base import (
DataTypeOps,
_as_categorical_type,
@@ -39,7 +36,9 @@ from pyspark.pandas.typedef import (
is_str_dtype,
pandas_on_spark_type,
)
-from pyspark.sql.types import BooleanType
+from pyspark.sql import functions as F
+from pyspark.sql.types import BooleanType, IntegralType, StringType
+from pyspark.sql.utils import pyspark_column_op
class StringOps(DataTypeOps):
diff --git a/python/pyspark/pandas/data_type_ops/timedelta_ops.py b/python/pyspark/pandas/data_type_ops/timedelta_ops.py
index 6f17474f61d8..bc38d1502ebf 100644
--- a/python/pyspark/pandas/data_type_ops/timedelta_ops.py
+++ b/python/pyspark/pandas/data_type_ops/timedelta_ops.py
@@ -23,11 +23,6 @@ import pandas as pd
from pandas.api.types import CategoricalDtype
from pyspark.loose_version import LooseVersion
-from pyspark.sql.types import (
- BooleanType,
- DayTimeIntervalType,
- StringType,
-)
from pyspark.pandas._typing import Dtype, IndexOpsLike, SeriesOrIndex
from pyspark.pandas.base import IndexOpsMixin
from pyspark.pandas.data_type_ops.base import (
@@ -38,6 +33,11 @@ from pyspark.pandas.data_type_ops.base import (
_sanitize_list_like,
)
from pyspark.pandas.typedef import pandas_on_spark_type
+from pyspark.sql.types import (
+ BooleanType,
+ DayTimeIntervalType,
+ StringType,
+)
from pyspark.sql.utils import pyspark_column_op
diff --git a/python/pyspark/pandas/datetimes.py b/python/pyspark/pandas/datetimes.py
index e0562749b6d9..4a1afca1fae6 100644
--- a/python/pyspark/pandas/datetimes.py
+++ b/python/pyspark/pandas/datetimes.py
@@ -26,12 +26,12 @@ import pandas as pd
from pandas.tseries.offsets import DateOffset
import pyspark.pandas as ps
-from pyspark.loose_version import LooseVersion
import pyspark.sql.functions as F
-from pyspark.sql.types import DateType, TimestampType, TimestampNTZType, IntegerType
+from pyspark.loose_version import LooseVersion
from pyspark.pandas import DataFrame
-from pyspark.pandas.config import option_context
from pyspark.pandas._typing import Dtype
+from pyspark.pandas.config import option_context
+from pyspark.sql.types import DateType, IntegerType, TimestampNTZType, TimestampType
class DatetimeMethods:
@@ -913,11 +913,12 @@ class DatetimeMethods:
def _test() -> None:
- import os
import doctest
+ import os
import sys
- from pyspark.sql import SparkSession
+
import pyspark.pandas.datetimes
+ from pyspark.sql import SparkSession
os.chdir(os.environ["SPARK_HOME"])
diff --git a/python/pyspark/pandas/exceptions.py b/python/pyspark/pandas/exceptions.py
index fce91deec2b2..3289cd32f380 100644
--- a/python/pyspark/pandas/exceptions.py
+++ b/python/pyspark/pandas/exceptions.py
@@ -113,11 +113,12 @@ class PandasNotImplementedError(NotImplementedError):
def _test() -> None:
- import os
import doctest
+ import os
import sys
- from pyspark.sql import SparkSession
+
import pyspark.pandas.exceptions
+ from pyspark.sql import SparkSession
os.chdir(os.environ["SPARK_HOME"])
diff --git a/python/pyspark/pandas/extensions.py b/python/pyspark/pandas/extensions.py
index 1764f2477065..c36f61f6c0be 100644
--- a/python/pyspark/pandas/extensions.py
+++ b/python/pyspark/pandas/extensions.py
@@ -14,8 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-from typing import Callable, Generic, Optional, Type, Union, TYPE_CHECKING
import warnings
+from typing import TYPE_CHECKING, Callable, Generic, Optional, Type, Union
from pyspark.pandas._typing import T
@@ -357,12 +357,14 @@ def register_index_accessor(name: str) -> Callable[[Type[T]], Type[T]]:
def _test() -> None:
- import os
import doctest
+ import os
import sys
+
import numpy
- from pyspark.sql import SparkSession
+
import pyspark.pandas.extensions
+ from pyspark.sql import SparkSession
os.chdir(os.environ["SPARK_HOME"])
diff --git a/python/pyspark/pandas/frame.py b/python/pyspark/pandas/frame.py
index 11ea453685a6..73919101f996 100644
--- a/python/pyspark/pandas/frame.py
+++ b/python/pyspark/pandas/frame.py
@@ -19,24 +19,26 @@
A wrapper class for Spark DataFrame to behave like pandas DataFrame.
"""
-from collections import defaultdict, namedtuple
-from collections.abc import Mapping
-import re
-import warnings
+import datetime
import inspect
import json
+import re
+import sys
import types
+import warnings
+from collections import defaultdict, namedtuple
+from collections.abc import Mapping
from functools import partial, reduce, wraps
-import sys
-from itertools import zip_longest, chain
+from itertools import chain, zip_longest
from types import TracebackType
from typing import (
+ IO,
+ TYPE_CHECKING,
Any,
Callable,
ClassVar,
Dict,
Generic,
- IO,
Iterable,
Iterator,
List,
@@ -49,16 +51,14 @@ from typing import (
Union,
cast,
no_type_check,
- TYPE_CHECKING,
)
-import datetime
import numpy as np
import pandas as pd
from pandas.api.types import (
is_bool_dtype,
- is_list_like,
is_dict_like,
+ is_list_like,
is_scalar,
)
from pandas.tseries.frequencies import DateOffset, to_offset # type: ignore[attr-defined]
@@ -66,34 +66,15 @@ from pandas.tseries.frequencies import DateOffset, to_offset # type: ignore[att
if TYPE_CHECKING:
from pandas.io.formats.style import Styler
-from pandas.core.dtypes.common import infer_dtype_from_object # type: ignore[attr-defined]
from pandas.core.accessor import CachedAccessor # type: ignore[attr-defined]
+from pandas.core.dtypes.common import infer_dtype_from_object # type: ignore[attr-defined]
from pandas.core.dtypes.inference import is_sequence # type: ignore[attr-defined]
-from pyspark._globals import _NoValue, _NoValueType
-from pyspark.loose_version import LooseVersion
-from pyspark.errors import PySparkValueError
from pyspark import StorageLevel
-from pyspark.sql import Column as PySparkColumn, DataFrame as PySparkDataFrame, functions as F
-from pyspark.sql.functions import pandas_udf
-from pyspark.sql.internal import InternalFunction as SF
-from pyspark.sql.types import (
- ArrayType,
- BooleanType,
- DataType,
- DoubleType,
- NumericType,
- Row,
- StringType,
- StructField,
- StructType,
- DecimalType,
- TimestampType,
- TimestampNTZType,
- NullType,
-)
-from pyspark.sql.window import Window
from pyspark import pandas as ps # For running doctests and reference resolution in PyCharm.
+from pyspark._globals import _NoValue, _NoValueType
+from pyspark.errors import PySparkValueError
+from pyspark.loose_version import LooseVersion
from pyspark.pandas._typing import (
Axis,
DataFrameOrSeries,
@@ -104,15 +85,38 @@ from pyspark.pandas._typing import (
T,
)
from pyspark.pandas.accessors import PandasOnSparkFrameMethods
-from pyspark.pandas.config import option_context, get_option
+from pyspark.pandas.config import get_option, option_context
from pyspark.pandas.correlation import (
- compute,
- CORRELATION_VALUE_1_COLUMN,
- CORRELATION_VALUE_2_COLUMN,
CORRELATION_CORR_OUTPUT_COLUMN,
CORRELATION_COUNT_OUTPUT_COLUMN,
+ CORRELATION_VALUE_1_COLUMN,
+ CORRELATION_VALUE_2_COLUMN,
+ compute,
+)
+from pyspark.pandas.generic import Frame
+from pyspark.pandas.internal import (
+ HIDDEN_COLUMNS,
+ NATURAL_ORDER_COLUMN_NAME,
+ SPARK_DEFAULT_INDEX_NAME,
+ SPARK_DEFAULT_SERIES_NAME,
+ SPARK_INDEX_NAME_FORMAT,
+ SPARK_INDEX_NAME_PATTERN,
+ InternalField,
+ InternalFrame,
+)
+from pyspark.pandas.missing.frame import MissingPandasLikeDataFrame
+from pyspark.pandas.plot import PandasOnSparkPlotAccessor
+from pyspark.pandas.spark.accessors import CachedSparkFrameMethods, SparkFrameMethods
+from pyspark.pandas.typedef.typehints import (
+ DataFrameType,
+ ScalarType,
+ SeriesType,
+ as_spark_type,
+ create_tuple_for_frame_type,
+ infer_return_type,
+ pandas_on_spark_type,
+ spark_type_to_pandas_dtype,
)
-from pyspark.pandas.spark.accessors import SparkFrameMethods, CachedSparkFrameMethods
from pyspark.pandas.utils import (
align_diff_frames,
ansi_mode_context,
@@ -123,6 +127,7 @@ from pyspark.pandas.utils import (
is_name_like_tuple,
is_name_like_value,
is_testing,
+ log_advice,
name_like_string,
same_anchor,
scol_for,
@@ -132,39 +137,35 @@ from pyspark.pandas.utils import (
validate_how,
validate_mode,
verify_temp_column_name,
- log_advice,
)
-from pyspark.pandas.generic import Frame
-from pyspark.pandas.internal import (
- InternalField,
- InternalFrame,
- HIDDEN_COLUMNS,
- NATURAL_ORDER_COLUMN_NAME,
- SPARK_INDEX_NAME_FORMAT,
- SPARK_DEFAULT_INDEX_NAME,
- SPARK_DEFAULT_SERIES_NAME,
- SPARK_INDEX_NAME_PATTERN,
-)
-from pyspark.pandas.missing.frame import MissingPandasLikeDataFrame
-from pyspark.pandas.typedef.typehints import (
- as_spark_type,
- infer_return_type,
- pandas_on_spark_type,
- spark_type_to_pandas_dtype,
- DataFrameType,
- SeriesType,
- ScalarType,
- create_tuple_for_frame_type,
+from pyspark.sql import Column as PySparkColumn
+from pyspark.sql import DataFrame as PySparkDataFrame
+from pyspark.sql import functions as F
+from pyspark.sql.functions import pandas_udf
+from pyspark.sql.internal import InternalFunction as SF
+from pyspark.sql.types import (
+ ArrayType,
+ BooleanType,
+ DataType,
+ DecimalType,
+ DoubleType,
+ NullType,
+ NumericType,
+ Row,
+ StringType,
+ StructField,
+ StructType,
+ TimestampNTZType,
+ TimestampType,
)
-from pyspark.pandas.plot import PandasOnSparkPlotAccessor
+from pyspark.sql.window import Window
if TYPE_CHECKING:
- from pyspark.sql._typing import OptionalPrimitiveType
-
from pyspark.pandas.groupby import DataFrameGroupBy
- from pyspark.pandas.resample import DataFrameResampler
from pyspark.pandas.indexes import Index
+ from pyspark.pandas.resample import DataFrameResampler
from pyspark.pandas.series import Series
+ from pyspark.sql._typing import OptionalPrimitiveType
# These regular expression patterns are compiled and defined here to avoid compiling the same
@@ -2718,6 +2719,7 @@ defaultdict(<class 'list'>, {'col..., 'col...})]
# but PlanMetrics/PlanObservedMetrics objects from Spark Connect are not
# JSON serializable. We filter these internal attrs only for affected versions.
import pyarrow as pa
+
from pyspark.loose_version import LooseVersion
if LooseVersion(pa.__version__) >= LooseVersion("22.0.0"):
@@ -14520,14 +14522,15 @@ class CachedDataFrame(DataFrame):
def _test() -> None:
- import os
import doctest
+ import os
import shutil
import sys
import tempfile
import uuid
- from pyspark.sql import SparkSession
+
import pyspark.pandas.frame
+ from pyspark.sql import SparkSession
os.chdir(os.environ["SPARK_HOME"])
diff --git a/python/pyspark/pandas/generic.py b/python/pyspark/pandas/generic.py
index ab0dd63b0f4e..004cd580645e 100644
--- a/python/pyspark/pandas/generic.py
+++ b/python/pyspark/pandas/generic.py
@@ -19,38 +19,30 @@
A base class of DataFrame/Column to behave like pandas DataFrame/Series.
"""
+import warnings
from abc import ABCMeta, abstractmethod
from functools import reduce
from typing import (
+ IO,
+ TYPE_CHECKING,
Any,
Callable,
Dict,
- IO,
List,
- Optional,
NoReturn,
+ Optional,
Tuple,
Union,
- TYPE_CHECKING,
cast,
)
-import warnings
import numpy as np
import pandas as pd
from pandas.api.types import is_list_like
+from pyspark import pandas as ps # For running doctests and reference resolution in PyCharm.
from pyspark._globals import _NoValue, _NoValueType
from pyspark.loose_version import LooseVersion
-from pyspark.sql import Column, functions as F
-from pyspark.sql.internal import InternalFunction as SF
-from pyspark.sql.types import (
- BooleanType,
- DoubleType,
- LongType,
- NumericType,
-)
-from pyspark import pandas as ps # For running doctests and reference resolution in PyCharm.
from pyspark.pandas._typing import (
Axis,
DataFrameOrSeries,
@@ -60,28 +52,37 @@ from pyspark.pandas._typing import (
Name,
Scalar,
)
-from pyspark.pandas.indexing import AtIndexer, iAtIndexer, iLocIndexer, LocIndexer
+from pyspark.pandas.indexing import AtIndexer, LocIndexer, iAtIndexer, iLocIndexer
from pyspark.pandas.internal import InternalFrame
from pyspark.pandas.typedef import spark_type_to_pandas_dtype
from pyspark.pandas.utils import (
+ SPARK_CONF_ARROW_ENABLED,
is_name_like_tuple,
is_name_like_value,
+ log_advice,
name_like_string,
scol_for,
sql_conf,
validate_arguments_and_invoke_function,
validate_axis,
validate_mode,
- SPARK_CONF_ARROW_ENABLED,
- log_advice,
+)
+from pyspark.sql import Column
+from pyspark.sql import functions as F
+from pyspark.sql.internal import InternalFunction as SF
+from pyspark.sql.types import (
+ BooleanType,
+ DoubleType,
+ LongType,
+ NumericType,
)
if TYPE_CHECKING:
from pyspark.pandas.frame import DataFrame
- from pyspark.pandas.indexes.base import Index
from pyspark.pandas.groupby import GroupBy
+ from pyspark.pandas.indexes.base import Index
from pyspark.pandas.series import Series
- from pyspark.pandas.window import Rolling, Expanding, ExponentialMoving
+ from pyspark.pandas.window import Expanding, ExponentialMoving, Rolling
bool_type = bool
@@ -3659,13 +3660,14 @@ class Frame(object, metaclass=ABCMeta):
def _test() -> None:
- import os
import doctest
+ import os
import shutil
import sys
import tempfile
- from pyspark.sql import SparkSession
+
import pyspark.pandas.generic
+ from pyspark.sql import SparkSession
os.chdir(os.environ["SPARK_HOME"])
diff --git a/python/pyspark/pandas/groupby.py b/python/pyspark/pandas/groupby.py
index 9907d9553245..667e272a0baf 100644
--- a/python/pyspark/pandas/groupby.py
+++ b/python/pyspark/pandas/groupby.py
@@ -19,19 +19,21 @@
A wrapper for GroupedData to behave like pandas GroupBy.
"""
-from abc import ABCMeta, abstractmethod
import inspect
+import warnings
+from abc import ABCMeta, abstractmethod
from collections import defaultdict, namedtuple
from functools import partial, wraps
from itertools import product
from typing import (
+ TYPE_CHECKING,
Any,
Callable,
Dict,
Generic,
Iterator,
- Mapping,
List,
+ Mapping,
Optional,
Sequence,
Set,
@@ -40,68 +42,68 @@ from typing import (
TypeVar,
Union,
cast,
- TYPE_CHECKING,
)
-import warnings
import pandas as pd
-from pandas.api.types import is_number, is_hashable, is_list_like
+from pandas.api.types import is_hashable, is_list_like, is_number
-from pyspark.sql import Column, DataFrame as SparkDataFrame, Window, functions as F
-from pyspark.sql.internal import InternalFunction as SF
-from pyspark.sql.types import (
- BooleanType,
- DataType,
- DoubleType,
- NumericType,
- StructField,
- StructType,
- StringType,
-)
from pyspark import pandas as ps # For running doctests and reference resolution in PyCharm.
from pyspark._globals import _NoValue, _NoValueType
from pyspark.loose_version import LooseVersion
from pyspark.pandas._typing import Axis, FrameLike, Label, Name
-from pyspark.pandas.typedef import infer_return_type, DataFrameType, ScalarType, SeriesType
+from pyspark.pandas.config import get_option
+from pyspark.pandas.correlation import (
+ CORRELATION_CORR_OUTPUT_COLUMN,
+ CORRELATION_COUNT_OUTPUT_COLUMN,
+ CORRELATION_VALUE_1_COLUMN,
+ CORRELATION_VALUE_2_COLUMN,
+ compute,
+)
+from pyspark.pandas.exceptions import DataError
from pyspark.pandas.frame import DataFrame
from pyspark.pandas.internal import (
- InternalField,
- InternalFrame,
HIDDEN_COLUMNS,
NATURAL_ORDER_COLUMN_NAME,
- SPARK_INDEX_NAME_FORMAT,
SPARK_DEFAULT_SERIES_NAME,
+ SPARK_INDEX_NAME_FORMAT,
SPARK_INDEX_NAME_PATTERN,
+ InternalField,
+ InternalFrame,
)
from pyspark.pandas.missing.groupby import (
MissingPandasLikeDataFrameGroupBy,
MissingPandasLikeSeriesGroupBy,
)
from pyspark.pandas.series import Series, first_series
-from pyspark.pandas.config import get_option
-from pyspark.pandas.correlation import (
- compute,
- CORRELATION_VALUE_1_COLUMN,
- CORRELATION_VALUE_2_COLUMN,
- CORRELATION_CORR_OUTPUT_COLUMN,
- CORRELATION_COUNT_OUTPUT_COLUMN,
-)
+from pyspark.pandas.spark.utils import as_nullable_spark_type, force_decimal_precision_scale
+from pyspark.pandas.typedef import DataFrameType, ScalarType, SeriesType, infer_return_type
from pyspark.pandas.utils import (
align_diff_frames,
ansi_mode_context,
is_name_like_tuple,
is_name_like_value,
+ log_advice,
name_like_string,
same_anchor,
scol_for,
verify_temp_column_name,
- log_advice,
)
-from pyspark.pandas.spark.utils import as_nullable_spark_type, force_decimal_precision_scale
-from pyspark.pandas.exceptions import DataError
+from pyspark.sql import Column, Window
+from pyspark.sql import DataFrame as SparkDataFrame
+from pyspark.sql import functions as F
+from pyspark.sql.internal import InternalFunction as SF
+from pyspark.sql.types import (
+ BooleanType,
+ DataType,
+ DoubleType,
+ NumericType,
+ StringType,
+ StructField,
+ StructType,
+)
if TYPE_CHECKING:
- from pyspark.pandas.window import RollingGroupby, ExpandingGroupby, ExponentialMovingGroupby
+ from pyspark.pandas.window import ExpandingGroupby, ExponentialMovingGroupby, RollingGroupby
FuncT = TypeVar("FuncT", bound=Callable[..., Any])
@@ -5029,12 +5031,14 @@ def normalize_keyword_aggregation(
def _test() -> None:
- import os
import doctest
+ import os
import sys
+
import numpy
- from pyspark.sql import SparkSession
+
import pyspark.pandas.groupby
+ from pyspark.sql import SparkSession
os.chdir(os.environ["SPARK_HOME"])
diff --git a/python/pyspark/pandas/indexes/base.py b/python/pyspark/pandas/indexes/base.py
index 97bed387864d..07fb897e3a69 100644
--- a/python/pyspark/pandas/indexes/base.py
+++ b/python/pyspark/pandas/indexes/base.py
@@ -15,8 +15,10 @@
# limitations under the License.
#
+import warnings
from functools import partial
from typing import (
+ TYPE_CHECKING,
Any,
Callable,
Iterator,
@@ -26,62 +28,61 @@ from typing import (
Union,
cast,
no_type_check,
- TYPE_CHECKING,
)
-import warnings
-import pandas as pd
import numpy as np
+import pandas as pd
+from pandas._libs import lib
from pandas.api.types import (
- is_list_like,
+ CategoricalDtype,
is_bool_dtype,
- is_integer_dtype,
is_float_dtype,
+ is_hashable,
+ is_integer_dtype,
+ is_list_like,
is_numeric_dtype,
is_object_dtype,
)
from pandas.core.accessor import CachedAccessor # type: ignore[attr-defined]
from pandas.io.formats.printing import pprint_thing # type: ignore[import-not-found]
-from pandas.api.types import CategoricalDtype, is_hashable
-from pandas._libs import lib
-from pyspark.loose_version import LooseVersion
-from pyspark.sql.column import Column
-from pyspark.sql import functions as F
-from pyspark.sql.types import (
- DayTimeIntervalType,
- IntegralType,
- TimestampType,
- TimestampNTZType,
-)
from pyspark import pandas as ps # For running doctests and reference resolution in PyCharm.
+from pyspark.loose_version import LooseVersion
from pyspark.pandas._typing import Dtype, Label, Name, Scalar
-from pyspark.pandas.config import get_option, option_context
from pyspark.pandas.base import IndexOpsMixin
+from pyspark.pandas.config import get_option, option_context
from pyspark.pandas.frame import DataFrame
+from pyspark.pandas.internal import (
+ DEFAULT_SERIES_NAME,
+ SPARK_DEFAULT_INDEX_NAME,
+ SPARK_INDEX_NAME_FORMAT,
+ InternalField,
+ InternalFrame,
+)
from pyspark.pandas.missing.indexes import MissingPandasLikeIndex
from pyspark.pandas.series import Series, first_series
from pyspark.pandas.spark.accessors import SparkIndexMethods
from pyspark.pandas.utils import (
+ ERROR_MESSAGE_CANNOT_COMBINE,
is_ansi_mode_enabled,
is_name_like_tuple,
is_name_like_value,
+ log_advice,
name_like_string,
same_anchor,
scol_for,
- verify_temp_column_name,
validate_bool_kwarg,
validate_index_loc,
- ERROR_MESSAGE_CANNOT_COMBINE,
- log_advice,
+ verify_temp_column_name,
xor,
)
-from pyspark.pandas.internal import (
- InternalField,
- InternalFrame,
- DEFAULT_SERIES_NAME,
- SPARK_DEFAULT_INDEX_NAME,
- SPARK_INDEX_NAME_FORMAT,
+from pyspark.sql import functions as F
+from pyspark.sql.column import Column
+from pyspark.sql.types import (
+ DayTimeIntervalType,
+ IntegralType,
+ TimestampNTZType,
+ TimestampType,
)
if TYPE_CHECKING:
@@ -1847,8 +1848,8 @@ class Index(IndexOpsMixin):
('b', 'y')],
)
"""
- from pyspark.pandas.indexes.multi import MultiIndex
from pyspark.pandas.indexes.category import CategoricalIndex
+ from pyspark.pandas.indexes.multi import MultiIndex
if isinstance(self, MultiIndex) != isinstance(other, MultiIndex):
raise NotImplementedError(
@@ -2652,13 +2653,15 @@ class Index(IndexOpsMixin):
def _test() -> None:
- import os
import doctest
+ import os
import sys
- from pyspark.sql import SparkSession
- import pyspark.pandas.indexes.base
+
from pandas.util.version import Version
+ import pyspark.pandas.indexes.base
+ from pyspark.sql import SparkSession
+
os.chdir(os.environ["SPARK_HOME"])
if Version(np.__version__) >= Version("2"):
diff --git a/python/pyspark/pandas/indexes/category.py b/python/pyspark/pandas/indexes/category.py
index d4c709fe6284..96830c6fe8a7 100644
--- a/python/pyspark/pandas/indexes/category.py
+++ b/python/pyspark/pandas/indexes/category.py
@@ -17,7 +17,7 @@
from typing import Any, Callable, List, Optional, Union, cast, no_type_check
import pandas as pd
-from pandas.api.types import is_hashable, CategoricalDtype
+from pandas.api.types import CategoricalDtype, is_hashable
from pyspark import pandas as ps
from pyspark.pandas.indexes.base import Index
@@ -639,11 +639,12 @@ class CategoricalIndex(Index):
def _test() -> None:
- import os
import doctest
+ import os
import sys
- from pyspark.sql import SparkSession
+
import pyspark.pandas.indexes.category
+ from pyspark.sql import SparkSession
os.chdir(os.environ["SPARK_HOME"])
... 16889 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org