You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ue...@apache.org on 2023/10/24 18:25:18 UTC
[spark] branch master updated (840306d867e2 -> 7ef96ee5b888)
This is an automated email from the ASF dual-hosted git repository.
ueshin pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
from 840306d867e2 [SPARK-45643][CORE][SQL] Replace `s.c.mutable.MapOps#transform` with `s.c.mutable.MapOps#mapValuesInPlace`
add 7ef96ee5b888 [SPARK-45524][PYTHON][SQL] Initial support for Python data source read API
No new revisions were added by this update.
Summary of changes:
.../src/main/resources/error/error-classes.json | 6 +
dev/sparktestsupport/modules.py | 1 +
docs/sql-error-conditions.md | 6 +
python/pyspark/errors/error_classes.py | 15 ++
python/pyspark/sql/datasource.py | 214 +++++++++++++++++++++
python/pyspark/sql/tests/test_python_datasource.py | 60 ++++++
python/pyspark/sql/worker/plan_data_source_read.py | 155 +++++++++++++++
.../plans/logical/pythonLogicalOperators.scala | 40 +++-
.../spark/sql/catalyst/trees/TreePatterns.scala | 1 +
.../spark/sql/errors/QueryCompilationErrors.scala | 7 +
.../spark/sql/execution/SparkOptimizer.scala | 8 +-
.../spark/sql/execution/SparkStrategies.scala | 2 +
.../datasources/PlanPythonDataSourceScan.scala | 90 +++++++++
.../PythonDataSourcePartitionsExec.scala} | 66 ++-----
.../python/UserDefinedPythonDataSource.scala | 87 +++++++++
.../apache/spark/sql/IntegratedUDFTestUtils.scala | 45 ++++-
.../execution/python/PythonDataSourceSuite.scala | 111 +++++++++++
17 files changed, 864 insertions(+), 50 deletions(-)
create mode 100644 python/pyspark/sql/datasource.py
create mode 100644 python/pyspark/sql/tests/test_python_datasource.py
create mode 100644 python/pyspark/sql/worker/plan_data_source_read.py
create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/PlanPythonDataSourceScan.scala
copy sql/core/src/main/scala/org/apache/spark/sql/execution/{LocalTableScanExec.scala => python/PythonDataSourcePartitionsExec.scala} (58%)
create mode 100644 sql/core/src/main/scala/org/apache/spark/sql/execution/python/UserDefinedPythonDataSource.scala
create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/python/PythonDataSourceSuite.scala
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org