You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@wayang.apache.org by al...@apache.org on 2026/01/07 08:52:00 UTC
(wayang) branch main updated (07cc9fd2d -> e6ce5a933)
This is an automated email from the ASF dual-hosted git repository.
aloalt pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/wayang.git
from 07cc9fd2d Merge pull request #651 from aleudework/modified-rest-server
add fabc62862 Spark DataFrames support / Optimizer load profiles
add 5f42f3c52 Update readme / add documentation
add 599508d25 add license header
add 7b5d3b191 Add Dataset flag to read/write Parquet APIs and update docs to reflect the unified interface.
add a738c3fb6 Unify parquet defaults in JavaPlanBuilder [INFO] BUILD SUCCESS, this solves the CR
add fbc4da06d Add Java overloads for readParquet [INFO] BUILD SUCCESS
add 0af3ec8a9 Import JvmOverloads for readParquet [INFO] BUILD SUCCESS
add 67f5dcc12 Restore Java parquet overload [INFO] BUILD SUCCESS
add a47315251 Fix parquet overload defaults [INFO] BUILD SUCCESS
new e6ce5a933 Merge pull request #646 from novatechflow/feature/spark-dataframes
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
README.md | 14 ++
guides/spark-datasets.md | 61 ++++++++
.../scala/org/apache/wayang/api/DataQuanta.scala | 17 ++-
.../org/apache/wayang/api/JavaPlanBuilder.scala | 13 +-
.../scala/org/apache/wayang/api/PlanBuilder.scala | 6 +-
.../apache/wayang/basic/operators/ParquetSink.java | 58 ++++++++
.../wayang/basic/operators/ParquetSource.java | 12 ++
.../wayang/spark/channels/ChannelConversions.java | 51 ++++++-
.../{RddChannel.java => DatasetChannel.java} | 93 +++++-------
.../org/apache/wayang/spark/mapping/Mappings.java | 3 +-
.../{SortMapping.java => ParquetSinkMapping.java} | 18 +--
...perator.java => SparkDatasetToRddOperator.java} | 66 ++++-----
.../wayang/spark/operators/SparkParquetSink.java | 90 ++++++++++++
.../wayang/spark/operators/SparkParquetSource.java | 36 +++--
...ileSink.java => SparkRddToDatasetOperator.java} | 66 ++++-----
.../wayang/spark/util/DatasetConverters.java | 157 +++++++++++++++++++++
.../resources/wayang-spark-defaults.properties | 22 +++
.../wayang/spark/operators/DatasetChannelTest.java | 98 +++++++++++++
.../wayang/spark/operators/DatasetTestUtils.java | 84 +++++++++++
...est.java => SparkDatasetToRddOperatorTest.java} | 39 ++---
.../spark/operators/SparkOperatorTestBase.java | 47 ++++--
.../spark/operators/SparkParquetSinkTest.java | 70 +++++++++
.../SparkParquetSourceDatasetOutputTest.java | 57 ++++++++
.../operators/SparkRddToDatasetOperatorTest.java | 44 +++---
.../apache/wayang/spark/test/ChannelFactory.java | 17 +++
25 files changed, 1022 insertions(+), 217 deletions(-)
create mode 100644 guides/spark-datasets.md
create mode 100644 wayang-commons/wayang-basic/src/main/java/org/apache/wayang/basic/operators/ParquetSink.java
copy wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/channels/{RddChannel.java => DatasetChannel.java} (53%)
copy wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/mapping/{SortMapping.java => ParquetSinkMapping.java} (74%)
copy wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/operators/{SparkCacheOperator.java => SparkDatasetToRddOperator.java} (54%)
create mode 100644 wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/operators/SparkParquetSink.java
copy wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/operators/{SparkTextFileSink.java => SparkRddToDatasetOperator.java} (52%)
create mode 100644 wayang-platforms/wayang-spark/src/main/java/org/apache/wayang/spark/util/DatasetConverters.java
create mode 100644 wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/DatasetChannelTest.java
create mode 100644 wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/DatasetTestUtils.java
copy wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/{SparkCollectionSourceTest.java => SparkDatasetToRddOperatorTest.java} (52%)
create mode 100644 wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/SparkParquetSinkTest.java
create mode 100644 wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/SparkParquetSourceDatasetOutputTest.java
copy wayang-commons/wayang-core/src/test/java/org/apache/wayang/core/util/LimitedInputStreamTest.java => wayang-platforms/wayang-spark/src/test/java/org/apache/wayang/spark/operators/SparkRddToDatasetOperatorTest.java (51%)
(wayang) 01/01: Merge pull request #646 from novatechflow/feature/spark-dataframes
Posted by al...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
aloalt pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/wayang.git
commit e6ce5a9337c2f4fed6d2730a85df203a57e4954f
Merge: 07cc9fd2d a47315251
Author: Alexander Alten <al...@apache.org>
AuthorDate: Wed Jan 7 09:51:55 2026 +0100
Merge pull request #646 from novatechflow/feature/spark-dataframes
Feature/spark dataframes
README.md | 14 ++
guides/spark-datasets.md | 61 ++++++++
.../scala/org/apache/wayang/api/DataQuanta.scala | 17 ++-
.../org/apache/wayang/api/JavaPlanBuilder.scala | 13 +-
.../scala/org/apache/wayang/api/PlanBuilder.scala | 6 +-
.../apache/wayang/basic/operators/ParquetSink.java | 58 ++++++++
.../wayang/basic/operators/ParquetSource.java | 12 ++
.../wayang/spark/channels/ChannelConversions.java | 51 ++++++-
.../wayang/spark/channels/DatasetChannel.java | 123 ++++++++++++++++
.../org/apache/wayang/spark/mapping/Mappings.java | 3 +-
.../wayang/spark/mapping/ParquetSinkMapping.java | 56 ++++++++
.../spark/operators/SparkDatasetToRddOperator.java | 96 +++++++++++++
.../wayang/spark/operators/SparkParquetSink.java | 90 ++++++++++++
.../wayang/spark/operators/SparkParquetSource.java | 36 +++--
.../spark/operators/SparkRddToDatasetOperator.java | 104 ++++++++++++++
.../wayang/spark/util/DatasetConverters.java | 157 +++++++++++++++++++++
.../resources/wayang-spark-defaults.properties | 22 +++
.../wayang/spark/operators/DatasetChannelTest.java | 98 +++++++++++++
.../wayang/spark/operators/DatasetTestUtils.java | 84 +++++++++++
.../operators/SparkDatasetToRddOperatorTest.java | 46 ++++++
.../spark/operators/SparkOperatorTestBase.java | 47 ++++--
.../spark/operators/SparkParquetSinkTest.java | 70 +++++++++
.../SparkParquetSourceDatasetOutputTest.java | 57 ++++++++
.../operators/SparkRddToDatasetOperatorTest.java | 43 ++++++
.../apache/wayang/spark/test/ChannelFactory.java | 17 +++
25 files changed, 1348 insertions(+), 33 deletions(-)