You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2025/05/14 02:57:46 UTC
(spark-connect-swift) branch gh-pages updated (65afad8 -> 631133f)
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a change to branch gh-pages
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
discard 65afad8 Setup GitHub Pages
new 631133f Setup GitHub Pages
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (65afad8)
\
N -- N -- N refs/heads/gh-pages (631133f)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
index.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org
(spark-connect-swift) 01/01: Setup GitHub Pages
Posted by do...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch gh-pages
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
commit 631133ffb32196a6a595b8d5c80ec182b8b83e31
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Tue May 13 19:26:10 2025 -0700
Setup GitHub Pages
---
.asf.yaml | 36 -
.github/.licenserc.yaml | 21 -
.github/PULL_REQUEST_TEMPLATE | 54 -
.github/workflows/build_and_test.yml | 156 -
.gitignore | 71 +-
.spi.yml | 20 -
Examples/app/Dockerfile | 46 -
Examples/app/Package.swift | 37 -
Examples/app/README.md | 76 -
Examples/app/Sources/main.swift | 41 -
Examples/pi/Dockerfile | 46 -
Examples/pi/Package.swift | 37 -
Examples/pi/README.md | 34 -
Examples/pi/Sources/main.swift | 35 -
Examples/stream/Dockerfile | 46 -
Examples/stream/Package.swift | 37 -
Examples/stream/README.md | 85 -
Examples/stream/Sources/main.swift | 52 -
Examples/web/Dockerfile | 48 -
Examples/web/Package.swift | 52 -
Examples/web/README.md | 113 -
.../Sources/SparkConnectSwiftWeb/configure.swift | 27 -
.../Sources/SparkConnectSwiftWeb/entrypoint.swift | 50 -
.../web/Sources/SparkConnectSwiftWeb/routes.swift | 40 -
LICENSE | 201 -
NOTICE | 5 -
Package.swift | 67 -
README.md | 210 -
Sources/SparkConnect/ArrowArray.swift | 343 -
Sources/SparkConnect/ArrowArrayBuilder.swift | 341 -
Sources/SparkConnect/ArrowBuffer.swift | 91 -
Sources/SparkConnect/ArrowBufferBuilder.swift | 385 -
Sources/SparkConnect/ArrowData.swift | 72 -
Sources/SparkConnect/ArrowDecoder.swift | 373 -
Sources/SparkConnect/ArrowEncoder.swift | 460 -
Sources/SparkConnect/ArrowReader.swift | 372 -
Sources/SparkConnect/ArrowReaderHelper.swift | 314 -
Sources/SparkConnect/ArrowSchema.swift | 76 -
Sources/SparkConnect/ArrowTable.swift | 205 -
Sources/SparkConnect/ArrowType.swift | 417 -
Sources/SparkConnect/ArrowWriter.swift | 366 -
Sources/SparkConnect/ArrowWriterHelper.swift | 142 -
Sources/SparkConnect/BearerTokenInterceptor.swift | 44 -
Sources/SparkConnect/BitUtility.swift | 40 -
.../SparkConnect/CaseInsensitiveDictionary.swift | 63 -
Sources/SparkConnect/Catalog.swift | 444 -
Sources/SparkConnect/ChunkedArray.swift | 153 -
Sources/SparkConnect/DataFrame.swift | 1404 ---
Sources/SparkConnect/DataFrameReader.swift | 278 -
Sources/SparkConnect/DataFrameWriter.swift | 248 -
Sources/SparkConnect/DataFrameWriterV2.swift | 152 -
Sources/SparkConnect/DataStreamReader.swift | 227 -
Sources/SparkConnect/DataStreamWriter.swift | 208 -
.../SparkConnect/Documentation.docc/Examples.md | 137 -
.../Documentation.docc/GettingStarted.md | 100 -
Sources/SparkConnect/Documentation.docc/Info.plist | 32 -
.../Documentation.docc/SparkConnect.md | 25 -
.../Documentation.docc/SparkSession.md | 64 -
Sources/SparkConnect/ErrorUtils.swift | 36 -
Sources/SparkConnect/Extension.swift | 324 -
Sources/SparkConnect/File_generated.swift | 261 -
Sources/SparkConnect/Flight.pb.swift | 1366 ---
Sources/SparkConnect/FlightData.swift | 50 -
Sources/SparkConnect/FlightDescriptor.swift | 57 -
Sources/SparkConnect/GroupedData.swift | 51 -
Sources/SparkConnect/MemoryAllocator.swift | 32 -
Sources/SparkConnect/Message_generated.swift | 637 --
Sources/SparkConnect/ProtoUtil.swift | 74 -
Sources/SparkConnect/ProtoUtils.swift | 39 -
Sources/SparkConnect/Row.swift | 86 -
Sources/SparkConnect/RuntimeConf.swift | 103 -
Sources/SparkConnect/Schema_generated.swift | 2151 ----
Sources/SparkConnect/SparkConnectClient.swift | 1021 --
Sources/SparkConnect/SparkConnectError.swift | 26 -
Sources/SparkConnect/SparkFileUtils.swift | 112 -
Sources/SparkConnect/SparkSession.swift | 600 --
Sources/SparkConnect/SparseTensor_generated.swift | 868 --
Sources/SparkConnect/StorageLevel.swift | 106 -
Sources/SparkConnect/StreamingQuery.swift | 176 -
Sources/SparkConnect/StreamingQueryManager.swift | 173 -
Sources/SparkConnect/Tensor_generated.swift | 311 -
Sources/SparkConnect/TypeAliases.swift | 69 -
Sources/SparkConnect/base.grpc.swift | 2438 -----
Sources/SparkConnect/base.pb.swift | 7606 -------------
Sources/SparkConnect/catalog.pb.swift | 2309 ----
Sources/SparkConnect/commands.pb.swift | 4588 --------
Sources/SparkConnect/common.pb.swift | 1115 --
Sources/SparkConnect/example_plugins.pb.swift | 215 -
Sources/SparkConnect/expressions.pb.swift | 4922 ---------
Sources/SparkConnect/ml.pb.swift | 991 --
Sources/SparkConnect/ml_common.pb.swift | 263 -
Sources/SparkConnect/relations.pb.swift | 10559 -------------------
Sources/SparkConnect/types.pb.swift | 2457 -----
Sources/SparkSQLRepl/main.swift | 57 -
Tests/SparkConnectTests/BuilderTests.swift | 54 -
.../CaseInsensitiveDictionaryTests.swift | 88 -
Tests/SparkConnectTests/CatalogTests.swift | 408 -
.../SparkConnectTests/DataFrameInternalTests.swift | 87 -
Tests/SparkConnectTests/DataFrameReaderTests.swift | 124 -
Tests/SparkConnectTests/DataFrameTests.swift | 855 --
Tests/SparkConnectTests/DataFrameWriterTests.swift | 175 -
.../SparkConnectTests/DataFrameWriterV2Tests.swift | 90 -
Tests/SparkConnectTests/DataStreamTests.swift | 69 -
.../SparkConnectTests/Resources/queries/array.sql | 1 -
.../Resources/queries/array.sql.answer | 5 -
.../SparkConnectTests/Resources/queries/binary.sql | 1 -
.../Resources/queries/binary.sql.answer | 5 -
.../SparkConnectTests/Resources/queries/cache.sql | 1 -
.../Resources/queries/cache.sql.answer | 4 -
.../Resources/queries/clear_cache.sql | 1 -
.../Resources/queries/clear_cache.sql.answer | 4 -
.../Resources/queries/create_scala_function.sql | 3 -
.../queries/create_scala_function.sql.answer | 4 -
.../Resources/queries/create_table_function.sql | 3 -
.../queries/create_table_function.sql.answer | 4 -
Tests/SparkConnectTests/Resources/queries/date.sql | 1 -
.../Resources/queries/date.sql.answer | 5 -
.../Resources/queries/describe_database.sql | 1 -
.../Resources/queries/describe_database.sql.answer | 9 -
.../Resources/queries/describe_function.sql | 1 -
.../Resources/queries/describe_function.sql.answer | 7 -
.../Resources/queries/describe_query.sql | 1 -
.../Resources/queries/describe_query.sql.answer | 7 -
.../Resources/queries/describe_table.sql | 1 -
.../Resources/queries/describe_table.sql.answer | 5 -
.../Resources/queries/drop_scala_function.sql | 1 -
.../queries/drop_scala_function.sql.answer | 4 -
.../Resources/queries/drop_table_function.sql | 1 -
.../queries/drop_table_function.sql.answer | 4 -
.../Resources/queries/explain.sql | 1 -
.../Resources/queries/explain.sql.answer | 5 -
.../Resources/queries/floating.sql | 5 -
.../Resources/queries/floating.sql.answer | 5 -
.../Resources/queries/function.sql | 1 -
.../Resources/queries/function.sql.answer | 5 -
.../Resources/queries/integral.sql | 5 -
.../Resources/queries/integral.sql.answer | 5 -
Tests/SparkConnectTests/Resources/queries/map.sql | 1 -
.../Resources/queries/map.sql.answer | 5 -
.../Resources/queries/pipesyntax.sql | 2 -
.../Resources/queries/pipesyntax.sql.answer | 6 -
.../Resources/queries/posexplode.sql | 1 -
.../Resources/queries/posexplode.sql.answer | 6 -
.../SparkConnectTests/Resources/queries/select.sql | 1 -
.../Resources/queries/select.sql.answer | 5 -
.../Resources/queries/show_databases.sql | 1 -
.../Resources/queries/show_databases.sql.answer | 5 -
.../Resources/queries/show_tables.sql | 1 -
.../Resources/queries/show_tables.sql.answer | 5 -
.../SparkConnectTests/Resources/queries/string.sql | 1 -
.../Resources/queries/string.sql.answer | 5 -
.../SparkConnectTests/Resources/queries/struct.sql | 1 -
.../Resources/queries/struct.sql.answer | 5 -
.../Resources/queries/timestamp.sql | 1 -
.../Resources/queries/timestamp.sql.answer | 5 -
.../Resources/queries/uncache.sql | 1 -
.../Resources/queries/uncache.sql.answer | 4 -
Tests/SparkConnectTests/RowTests.swift | 95 -
Tests/SparkConnectTests/RuntimeConfTests.swift | 120 -
Tests/SparkConnectTests/SQLHelper.swift | 86 -
Tests/SparkConnectTests/SQLTests.swift | 121 -
.../SparkConnectClientTests.swift | 100 -
Tests/SparkConnectTests/SparkFileUtilsTests.swift | 70 -
Tests/SparkConnectTests/SparkSessionTests.swift | 191 -
.../StreamingQueryManagerTests.swift | 106 -
Tests/SparkConnectTests/StreamingQueryTests.swift | 57 -
_config.yml | 21 +
dev/merge_spark_pr.py | 740 --
index.md | 38 +
169 files changed, 64 insertions(+), 59377 deletions(-)
diff --git a/.asf.yaml b/.asf.yaml
deleted file mode 100644
index b8e8b93..0000000
--- a/.asf.yaml
+++ /dev/null
@@ -1,36 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# https://cwiki.apache.org/confluence/display/INFRA/git+-+.asf.yaml+features
----
-github:
- description: "Apache Spark Connect Client for Swift"
- homepage: https://spark.apache.org/
- labels:
- - swift
- - big-data
- - sql
- - spark
- enabled_merge_buttons:
- merge: false
- squash: true
- rebase: true
- autolink_jira: SPARK
-
-notifications:
- pullrequests: reviews@spark.apache.org
- issues: reviews@spark.apache.org
- commits: commits@spark.apache.org
- jira_options: link label
diff --git a/.github/.licenserc.yaml b/.github/.licenserc.yaml
deleted file mode 100644
index 5f08a50..0000000
--- a/.github/.licenserc.yaml
+++ /dev/null
@@ -1,21 +0,0 @@
-header:
- license:
- spdx-id: Apache-2.0
- copyright-owner: Apache Software Foundation
-
- paths:
- - '**'
-
- paths-ignore:
- - '**/*.md'
- - '**/.gitignore'
- - '.github/**'
- - 'LICENSE'
- - 'NOTICE'
- - '.asf.yaml'
- - '.nojekyll'
- - 'Package.swift'
- - '**/*pb.swift'
- - 'Tests/SparkConnectTests/Resources/queries/**'
-
- comment: on-failure
diff --git a/.github/PULL_REQUEST_TEMPLATE b/.github/PULL_REQUEST_TEMPLATE
deleted file mode 100644
index af7bbe2..0000000
--- a/.github/PULL_REQUEST_TEMPLATE
+++ /dev/null
@@ -1,54 +0,0 @@
-<!--
-Thanks for sending a pull request! Here are some tips for you:
- 1. If this is your first time, please read our contributor guidelines: https://spark.apache.org/contributing.html
- 2. Ensure you have added or run the appropriate tests for your PR: https://spark.apache.org/developer-tools.html
- 3. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP][SPARK-XXXX] Your PR title ...'.
- 4. Be sure to keep the PR description updated to reflect all changes.
- 5. Please write your PR title to summarize what this PR proposes.
- 6. If possible, provide a concise example to reproduce the issue for a faster review.
--->
-
-### What changes were proposed in this pull request?
-<!--
-Please clarify what changes you are proposing. The purpose of this section is to outline the changes and how this PR fixes the issue.
-If possible, please consider writing useful notes for better and faster reviews in your PR. See the examples below.
- 1. If you refactor some codes with changing classes, showing the class hierarchy will help reviewers.
- 2. If you fix some SQL features, you can provide some references of other DBMSes.
- 3. If there is design documentation, please add the link.
- 4. If there is a discussion in the mailing list, please add the link.
--->
-
-
-### Why are the changes needed?
-<!--
-Please clarify why the changes are needed. For instance,
- 1. If you propose a new API, clarify the use case for a new API.
- 2. If you fix a bug, you can clarify why it is a bug.
--->
-
-
-### Does this PR introduce _any_ user-facing change?
-<!--
-Note that it means *any* user-facing change including all aspects such as new features, bug fixes, or other behavior changes. Documentation-only updates are not considered user-facing changes.
-
-If yes, please clarify the previous behavior and the change this PR proposes - provide the console output, description and/or an example to show the behavior difference if possible.
-If possible, please also clarify if this is a user-facing change compared to the released Spark versions or within the unreleased branches such as master.
-If no, write 'No'.
--->
-
-
-### How was this patch tested?
-<!--
-If tests were added, say they were added here. Please make sure to add some test cases that check the changes thoroughly including negative and positive cases if possible.
-If it was tested in a way different from regular unit tests, please clarify how you tested step by step, ideally copy and paste-able, so that other reviewers can test and check, and descendants can verify in the future.
-If tests were not added, please describe why they were not added and/or why it was difficult to add.
--->
-
-
-### Was this patch authored or co-authored using generative AI tooling?
-<!--
-If generative AI tooling has been used in the process of authoring this patch, please include the
-phrase: 'Generated-by: ' followed by the name of the tool and its version.
-If no, write 'No'.
-Please refer to the [ASF Generative Tooling Guidance](https://www.apache.org/legal/generative-tooling.html) for details.
--->
diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml
deleted file mode 100644
index 1b0a9ea..0000000
--- a/.github/workflows/build_and_test.yml
+++ /dev/null
@@ -1,156 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-
-name: Build and test
-
-on:
- push:
- branches: [ "main" ]
- pull_request:
- branches: [ "main" ]
-
-jobs:
- license-check:
- name: "License Check"
- runs-on: ubuntu-latest
- steps:
- - name: Checkout repository
- uses: actions/checkout@v3
- - name: Check license header
- uses: apache/skywalking-eyes@main
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- with:
- config: .github/.licenserc.yaml
-
- build-macos-15-swift60:
- runs-on: macos-15
- steps:
- - uses: actions/checkout@v4
- - uses: swift-actions/setup-swift@d10500c1ac8822132eebbd74c48c3372c71d7ff5
- with:
- swift-version: "6.0"
- - name: Build
- run: swift build -c release
-
- build-macos-15-swift61:
- runs-on: macos-15
- steps:
- - uses: actions/checkout@v4
- - uses: swift-actions/setup-swift@d10500c1ac8822132eebbd74c48c3372c71d7ff5
- with:
- swift-version: "6.1"
- - name: Build
- run: swift build -c release
-
- build-ubuntu-latest:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
- - name: Build
- run: |
- docker run swift:6.1 uname -a
- docker run -v $PWD:/spark -w /spark swift:6.1 swift build -c release
-
- # setup-swift doesn't support ARM linux yet.
- build-ubuntu-arm:
- runs-on: ubuntu-24.04-arm
- steps:
- - uses: actions/checkout@v4
- - name: Build
- run: |
- docker run swift:6.1 uname -a
- docker run -v $PWD:/spark -w /spark swift:6.1 swift build -c release
-
- integration-test-linux:
- runs-on: ubuntu-latest
- env:
- SPARK_REMOTE: "sc://localhost:15003"
- services:
- spark:
- image: apache/spark:4.0.0-preview2
- env:
- SPARK_NO_DAEMONIZE: 1
- ports:
- - 15003:15002
- options: --entrypoint /opt/spark/sbin/start-connect-server.sh
- steps:
- - uses: actions/checkout@v4
- - name: Build
- run: |
- docker run swift:6.1 uname -a
- docker run --add-host=host.docker.internal:host-gateway -v $PWD:/spark -w /spark -e SPARK_REMOTE='sc://host.docker.internal:15003' swift:6.1 swift test --no-parallel -c release
-
- integration-test-mac:
- runs-on: macos-15
- steps:
- - uses: actions/checkout@v4
- - uses: swift-actions/setup-swift@d10500c1ac8822132eebbd74c48c3372c71d7ff5
- with:
- swift-version: "6.1"
- - name: Test
- run: |
- curl -LO https://dist.apache.org/repos/dist/dev/spark/v4.0.0-rc5-bin/spark-4.0.0-bin-hadoop3.tgz
- tar xvfz spark-4.0.0-bin-hadoop3.tgz
- mv spark-4.0.0-bin-hadoop3 /tmp/spark
- cd /tmp/spark/sbin
- ./start-connect-server.sh
- cd -
- swift test --no-parallel -c release
-
- integration-test-token:
- runs-on: macos-15
- env:
- SPARK_CONNECT_AUTHENTICATE_TOKEN: ${{ github.run_id }}-${{ github.run_attempt }}
- steps:
- - uses: actions/checkout@v4
- - uses: swift-actions/setup-swift@d10500c1ac8822132eebbd74c48c3372c71d7ff5
- with:
- swift-version: "6.1"
- - name: Test
- run: |
- curl -LO https://dist.apache.org/repos/dist/dev/spark/v4.0.0-rc5-bin/spark-4.0.0-bin-hadoop3.tgz
- tar xvfz spark-4.0.0-bin-hadoop3.tgz
- mv spark-4.0.0-bin-hadoop3 /tmp/spark
- cd /tmp/spark/sbin
- ./start-connect-server.sh
- cd -
- swift test --no-parallel -c release
-
- integration-test-mac-spark3:
- runs-on: macos-15
- steps:
- - uses: actions/checkout@v4
- - uses: swift-actions/setup-swift@d10500c1ac8822132eebbd74c48c3372c71d7ff5
- with:
- swift-version: "6.1"
- - name: Install Java
- uses: actions/setup-java@v4
- with:
- distribution: zulu
- java-version: 17
- - name: Test
- run: |
- curl -LO https://downloads.apache.org/spark/spark-3.5.5/spark-3.5.5-bin-hadoop3.tgz
- tar xvfz spark-3.5.5-bin-hadoop3.tgz
- mv spark-3.5.5-bin-hadoop3 /tmp/spark
- cd /tmp/spark/sbin
- ./start-connect-server.sh --packages org.apache.spark:spark-connect_2.12:3.5.5
- cd -
- swift test --no-parallel -c release
diff --git a/.gitignore b/.gitignore
index 5ed8585..1e73773 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,66 +1,5 @@
-# Xcode
-#
-# gitignore contributors: remember to update Global/Xcode.gitignore, Objective-C.gitignore & Swift.gitignore
-
-## Swift Package Manager
-Package.resolved
-.swiftpm/
-
-## User settings
-xcuserdata/
-
-## Obj-C/Swift specific
-*.hmap
-
-## App packaging
-*.ipa
-*.dSYM.zip
-*.dSYM
-
-## Playgrounds
-timeline.xctimeline
-playground.xcworkspace
-
-# Swift Package Manager
-#
-# Add this line if you want to avoid checking in source code from Swift Package Manager dependencies.
-# Packages/
-# Package.pins
-# Package.resolved
-# *.xcodeproj
-#
-# Xcode automatically generates this directory with a .xcworkspacedata file and xcuserdata
-# hence it is not needed unless you have added a package configuration file to your project
-# .swiftpm
-
-.build/
-
-# CocoaPods
-#
-# We recommend against adding the Pods directory to your .gitignore. However
-# you should judge for yourself, the pros and cons are mentioned at:
-# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
-#
-# Pods/
-#
-# Add this line if you want to avoid checking in source code from the Xcode workspace
-# *.xcworkspace
-
-# Carthage
-#
-# Add this line if you want to avoid checking in source code from Carthage dependencies.
-# Carthage/Checkouts
-
-Carthage/Build/
-
-# fastlane
-#
-# It is recommended to not store the screenshots in the git repo.
-# Instead, use fastlane to re-generate the screenshots whenever they are needed.
-# For more information about the recommended setup visit:
-# https://docs.fastlane.tools/best-practices/source-control/#source-control
-
-fastlane/report.xml
-fastlane/Preview.html
-fastlane/screenshots/**/*.png
-fastlane/test_output
+*.swp
+*~
+.DS_Store
+.idea/
+.vscode
diff --git a/.spi.yml b/.spi.yml
deleted file mode 100644
index 20c68bd..0000000
--- a/.spi.yml
+++ /dev/null
@@ -1,20 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-version: 1
-metadata:
- authors: "Apache Spark project"
-builder:
- configs:
- - documentation_targets: [SparkConnect]
diff --git a/Examples/app/Dockerfile b/Examples/app/Dockerfile
deleted file mode 100644
index 1bbacd2..0000000
--- a/Examples/app/Dockerfile
+++ /dev/null
@@ -1,46 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM swift:6.1 AS builder
-
-WORKDIR /app
-
-COPY . .
-
-RUN swift build -c release
-
-FROM swift:6.1-slim
-
-ARG SPARK_UID=185
-
-LABEL org.opencontainers.image.authors="Apache Spark project <de...@spark.apache.org>"
-LABEL org.opencontainers.image.licenses="Apache-2.0"
-LABEL org.opencontainers.image.ref.name="Apache Spark Connect for Swift"
-
-ENV SPARK_SWIFT_HOME=/opt/spark-swift
-ENV SPARK_SWIFT_APP=SparkConnectSwiftApp
-
-WORKDIR $SPARK_SWIFT_HOME
-
-RUN groupadd --system --gid=$SPARK_UID spark && \
- useradd --system --home-dir $SPARK_SWIFT_HOME --uid=$SPARK_UID --gid=spark spark && \
- chown -R spark:spark $SPARK_SWIFT_HOME
-
-COPY --from=builder --chown=spark:spark /app/.build/*-unknown-linux-gnu/release/$SPARK_SWIFT_APP .
-
-USER spark
-
-ENTRYPOINT ["/bin/sh", "-c", "$SPARK_SWIFT_HOME/$SPARK_SWIFT_APP"]
diff --git a/Examples/app/Package.swift b/Examples/app/Package.swift
deleted file mode 100644
index 8afa325..0000000
--- a/Examples/app/Package.swift
+++ /dev/null
@@ -1,37 +0,0 @@
-// swift-tools-version: 6.0
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-
-import PackageDescription
-
-let package = Package(
- name: "SparkConnectSwiftApp",
- platforms: [
- .macOS(.v15)
- ],
- dependencies: [
- .package(url: "https://github.com/apache/spark-connect-swift.git", branch: "v0.1.0")
- ],
- targets: [
- .executableTarget(
- name: "SparkConnectSwiftApp",
- dependencies: [.product(name: "SparkConnect", package: "spark-connect-swift")]
- )
- ]
-)
diff --git a/Examples/app/README.md b/Examples/app/README.md
deleted file mode 100644
index bede7c2..0000000
--- a/Examples/app/README.md
+++ /dev/null
@@ -1,76 +0,0 @@
-# A Swift Application with Apache Spark Connect Swift Client
-
-This is an example Swift application to show how to use Apache Spark Connect Swift Client library.
-
-## How to run
-
-Prepare `Spark Connect Server` via running Docker image.
-
-```
-docker run --rm -p 15002:15002 apache/spark:4.0.0-preview2 bash -c "/opt/spark/sbin/start-connect-server.sh --wait"
-```
-
-Build an application Docker image.
-
-```
-$ docker build -t apache/spark-connect-swift:app .
-$ docker images apache/spark-connect-swift:app
-REPOSITORY TAG IMAGE ID CREATED SIZE
-apache/spark-connect-swift app e132e1b38348 5 seconds ago 368MB
-```
-
-Run `app` docker image.
-
-```
-$ docker run --rm -e SPARK_REMOTE=sc://host.docker.internal:15002 apache/spark-connect-swift:app
-Connected to Apache Spark 4.0.0-preview2 Server
-EXECUTE: DROP TABLE IF EXISTS t
-EXECUTE: CREATE TABLE IF NOT EXISTS t(a INT) USING ORC
-EXECUTE: INSERT INTO t VALUES (1), (2), (3)
-SELECT * FROM t
-+---+
-| a|
-+---+
-| 2|
-| 1|
-| 3|
-+---+
-
-+---+
-| id|
-+---+
-| 0|
-| 8|
-| 6|
-| 2|
-| 4|
-+---+
-```
-
-Run from source code.
-
-```
-$ swift run
-...
-Connected to Apache Spark 4.0.0-preview2 Server
-EXECUTE: DROP TABLE IF EXISTS t
-EXECUTE: CREATE TABLE IF NOT EXISTS t(a INT) USING ORC
-EXECUTE: INSERT INTO t VALUES (1), (2), (3)
-SELECT * FROM t
-+---+
-| a |
-+---+
-| 2 |
-| 1 |
-| 3 |
-+---+
-+----+
-| id |
-+----+
-| 2 |
-| 6 |
-| 0 |
-| 8 |
-| 4 |
-+----+
-```
diff --git a/Examples/app/Sources/main.swift b/Examples/app/Sources/main.swift
deleted file mode 100644
index c9ae2e2..0000000
--- a/Examples/app/Sources/main.swift
+++ /dev/null
@@ -1,41 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-
-import SparkConnect
-
-let spark = try await SparkSession.builder.getOrCreate()
-print("Connected to Apache Spark \(await spark.version) Server")
-
-let statements = [
- "DROP TABLE IF EXISTS t",
- "CREATE TABLE IF NOT EXISTS t(a INT) USING ORC",
- "INSERT INTO t VALUES (1), (2), (3)",
-]
-
-for s in statements {
- print("EXECUTE: \(s)")
- _ = try await spark.sql(s).count()
-}
-print("SELECT * FROM t")
-try await spark.sql("SELECT * FROM t").cache().show()
-
-try await spark.range(10).filter("id % 2 == 0").write.mode("overwrite").orc("/tmp/orc")
-try await spark.read.orc("/tmp/orc").show()
-
-await spark.stop()
diff --git a/Examples/pi/Dockerfile b/Examples/pi/Dockerfile
deleted file mode 100644
index 49ee0de..0000000
--- a/Examples/pi/Dockerfile
+++ /dev/null
@@ -1,46 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM swift:6.1 AS builder
-
-WORKDIR /app
-
-COPY . .
-
-RUN swift build -c release
-
-FROM swift:6.1-slim
-
-ARG SPARK_UID=185
-
-LABEL org.opencontainers.image.authors="Apache Spark project <de...@spark.apache.org>"
-LABEL org.opencontainers.image.licenses="Apache-2.0"
-LABEL org.opencontainers.image.ref.name="Apache Spark Connect for Swift"
-
-ENV SPARK_SWIFT_HOME=/opt/spark-swift
-ENV SPARK_SWIFT_APP=SparkConnectSwiftPi
-
-WORKDIR $SPARK_SWIFT_HOME
-
-RUN groupadd --system --gid=$SPARK_UID spark && \
- useradd --system --home-dir $SPARK_SWIFT_HOME --uid=$SPARK_UID --gid=spark spark && \
- chown -R spark:spark $SPARK_SWIFT_HOME
-
-COPY --from=builder --chown=spark:spark /app/.build/*-unknown-linux-gnu/release/$SPARK_SWIFT_APP .
-
-USER spark
-
-ENTRYPOINT ["/bin/sh", "-c", "$SPARK_SWIFT_HOME/$SPARK_SWIFT_APP"]
diff --git a/Examples/pi/Package.swift b/Examples/pi/Package.swift
deleted file mode 100644
index 7b2400e..0000000
--- a/Examples/pi/Package.swift
+++ /dev/null
@@ -1,37 +0,0 @@
-// swift-tools-version: 6.0
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-
-import PackageDescription
-
-let package = Package(
- name: "SparkConnectSwiftPi",
- platforms: [
- .macOS(.v15)
- ],
- dependencies: [
- .package(url: "https://github.com/apache/spark-connect-swift.git", branch: "main")
- ],
- targets: [
- .executableTarget(
- name: "SparkConnectSwiftPi",
- dependencies: [.product(name: "SparkConnect", package: "spark-connect-swift")]
- )
- ]
-)
diff --git a/Examples/pi/README.md b/Examples/pi/README.md
deleted file mode 100644
index 715c381..0000000
--- a/Examples/pi/README.md
+++ /dev/null
@@ -1,34 +0,0 @@
-# A Swift Application computing an approximation to pi with Apache Spark Connect Swift Client
-
-This is an example Swift application to show how to use Apache Spark Connect Swift Client library.
-
-## How to run
-
-Prepare `Spark Connect Server` via running Docker image.
-```
-docker run --rm -p 15002:15002 apache/spark:4.0.0-preview2 bash -c "/opt/spark/sbin/start-connect-server.sh --wait"
-```
-
-Build an application Docker image.
-
-```
-$ docker build -t apache/spark-connect-swift:pi .
-$ docker images apache/spark-connect-swift:pi
-REPOSITORY TAG IMAGE ID CREATED SIZE
-apache/spark-connect-swift pi d03952577564 4 seconds ago 369MB
-```
-
-Run `pi` docker image.
-
-```
-$ docker run --rm -e SPARK_REMOTE=sc://host.docker.internal:15002 apache/spark-connect-swift:pi
-Pi is roughly 3.1412831412831412
-```
-
-Run from source code.
-
-```
-$ swift run
-...
-Pi is roughly 3.1423711423711422
-```
diff --git a/Examples/pi/Sources/main.swift b/Examples/pi/Sources/main.swift
deleted file mode 100644
index f894a54..0000000
--- a/Examples/pi/Sources/main.swift
+++ /dev/null
@@ -1,35 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-
-import SparkConnect
-
-let spark = try await SparkSession.builder.getOrCreate()
-
-let n: Int64 = CommandLine.arguments.count > 1 ? Int64(CommandLine.arguments[1])! : 1_000_000
-
-let count =
- try await spark
- .range(n)
- .selectExpr("(pow(rand() * 2 - 1, 2) + pow(rand() * 2 - 1, 2)) as v")
- .where("v <= 1")
- .count()
-
-print("Pi is roughly \(4.0 * Double(count) / (Double(n) - 1))")
-
-await spark.stop()
diff --git a/Examples/stream/Dockerfile b/Examples/stream/Dockerfile
deleted file mode 100644
index 16583cd..0000000
--- a/Examples/stream/Dockerfile
+++ /dev/null
@@ -1,46 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM swift:6.1 AS builder
-
-WORKDIR /app
-
-COPY . .
-
-RUN swift build -c release
-
-FROM swift:6.1-slim
-
-ARG SPARK_UID=185
-
-LABEL org.opencontainers.image.authors="Apache Spark project <de...@spark.apache.org>"
-LABEL org.opencontainers.image.licenses="Apache-2.0"
-LABEL org.opencontainers.image.ref.name="Apache Spark Connect for Swift"
-
-ENV SPARK_SWIFT_HOME=/opt/spark-swift
-ENV SPARK_SWIFT_APP=SparkConnectSwiftNetworkWordCount
-
-WORKDIR $SPARK_SWIFT_HOME
-
-RUN groupadd --system --gid=$SPARK_UID spark && \
- useradd --system --home-dir $SPARK_SWIFT_HOME --uid=$SPARK_UID --gid=spark spark && \
- chown -R spark:spark $SPARK_SWIFT_HOME
-
-COPY --from=builder --chown=spark:spark /app/.build/*-unknown-linux-gnu/release/$SPARK_SWIFT_APP .
-
-USER spark
-
-ENTRYPOINT ["/bin/sh", "-c", "$SPARK_SWIFT_HOME/$SPARK_SWIFT_APP"]
diff --git a/Examples/stream/Package.swift b/Examples/stream/Package.swift
deleted file mode 100644
index 8c8ef7a..0000000
--- a/Examples/stream/Package.swift
+++ /dev/null
@@ -1,37 +0,0 @@
-// swift-tools-version: 6.0
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-
-import PackageDescription
-
-let package = Package(
- name: "SparkConnectSwiftNetworkWordCount",
- platforms: [
- .macOS(.v15)
- ],
- dependencies: [
- .package(url: "https://github.com/apache/spark-connect-swift.git", branch: "main")
- ],
- targets: [
- .executableTarget(
- name: "SparkConnectSwiftNetworkWordCount",
- dependencies: [.product(name: "SparkConnect", package: "spark-connect-swift")]
- )
- ]
-)
diff --git a/Examples/stream/README.md b/Examples/stream/README.md
deleted file mode 100644
index 3d0221a..0000000
--- a/Examples/stream/README.md
+++ /dev/null
@@ -1,85 +0,0 @@
-# A Swift Network Word Count Application with Apache Spark Connect Swift Client
-
-This is an example Swift stream processing application to show how to count words with Apache Spark Connect Swift Client library.
-
-## Run `Spark Connect Server`
-
-```bash
-docker run --rm -p 15002:15002 apache/spark:4.0.0-preview2 bash -c "/opt/spark/sbin/start-connect-server.sh --wait -c spark.log.level=ERROR"
-```
-
-## Run `Netcat` as a streaming input server
-
-You will first need to run Netcat (a small utility found in most Unix-like systems) as a data server by using
-
-```bash
-nc -lk 9999
-```
-
-## Build and run from docker image
-
-Build an application Docker image.
-
-```
-$ docker build -t apache/spark-connect-swift:stream .
-$ docker images apache/spark-connect-swift:stream
-REPOSITORY TAG IMAGE ID CREATED SIZE
-apache/spark-connect-swift stream a4daa10ad9c5 7 seconds ago 369MB
-```
-
-Run `stream` docker image.
-
-```
-$ docker run --rm -e SPARK_REMOTE=sc://host.docker.internal:15002 -e TARGET_HOST=host.docker.internal apache/spark-connect-swift:stream
-```
-
-## Send input and check output
-
-Then, any lines typed in the terminal running the `Netcat` server will be counted and printed on screen every second.
-
-```bash
-$ nc -lk 9999
-apache spark
-apache hadoop
-```
-
-`Spark Connect Server` output will look something like the following.
-
-```bash
--------------------------------------------
-Batch: 0
--------------------------------------------
-+----+--------+
-|word|count(1)|
-+----+--------+
-+----+--------+
-
--------------------------------------------
-Batch: 1
--------------------------------------------
-+------+--------+
-| word|count(1)|
-+------+--------+
-|apache| 1|
-| spark| 1|
-+------+--------+
-
--------------------------------------------
-Batch: 2
--------------------------------------------
-+------+--------+
-| word|count(1)|
-+------+--------+
-|apache| 2|
-| spark| 1|
-|hadoop| 1|
-+------+--------+
-```
-
-## Run from source code
-
-```bash
-$ swift run
-...
-Connected to Apache Spark 4.0.0 Server
-```
diff --git a/Examples/stream/Sources/main.swift b/Examples/stream/Sources/main.swift
deleted file mode 100644
index 5c73356..0000000
--- a/Examples/stream/Sources/main.swift
+++ /dev/null
@@ -1,52 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-
-import Foundation
-import SparkConnect
-
-let spark = try await SparkSession.builder.getOrCreate()
-print("Connected to Apache Spark \(await spark.version) Server")
-
-let host = ProcessInfo.processInfo.environment["TARGET_HOST"] ?? "localhost"
-
-let lines =
- await spark
- .readStream
- .format("socket")
- .option("host", host)
- .option("port", 9999)
- .load()
-
-let word =
- await lines
- .selectExpr("explode(split(value, ' ')) as word")
-
-let wordCounts =
- await word
- .groupBy("word")
- .agg("count(*)")
-
-let query =
- try await wordCounts
- .writeStream
- .outputMode("complete")
- .format("console")
- .start()
-
-_ = try await query.awaitTermination()
diff --git a/Examples/web/Dockerfile b/Examples/web/Dockerfile
deleted file mode 100644
index 681f81f..0000000
--- a/Examples/web/Dockerfile
+++ /dev/null
@@ -1,48 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-FROM swift:6.1 AS builder
-
-WORKDIR /app
-
-COPY . .
-
-RUN swift build -c release
-
-FROM swift:6.1-slim
-
-ARG SPARK_UID=185
-
-LABEL org.opencontainers.image.authors="Apache Spark project <de...@spark.apache.org>"
-LABEL org.opencontainers.image.licenses="Apache-2.0"
-LABEL org.opencontainers.image.ref.name="Apache Spark Connect for Swift"
-
-ENV SPARK_SWIFT_HOME=/opt/spark-swift
-ENV SPARK_SWIFT_APP=SparkConnectSwiftWeb
-
-WORKDIR $SPARK_SWIFT_HOME
-
-RUN groupadd --system --gid=$SPARK_UID spark && \
- useradd --system --home-dir $SPARK_SWIFT_HOME --uid=$SPARK_UID --gid=spark spark && \
- chown -R spark:spark $SPARK_SWIFT_HOME
-
-COPY --from=builder --chown=spark:spark /app/.build/*-unknown-linux-gnu/release/$SPARK_SWIFT_APP .
-
-USER spark
-
-EXPOSE 8080
-
-ENTRYPOINT ["/bin/sh", "-c", "$SPARK_SWIFT_HOME/$SPARK_SWIFT_APP"]
diff --git a/Examples/web/Package.swift b/Examples/web/Package.swift
deleted file mode 100644
index f17d697..0000000
--- a/Examples/web/Package.swift
+++ /dev/null
@@ -1,52 +0,0 @@
-// swift-tools-version:6.0
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-import PackageDescription
-
-let package = Package(
- name: "SparkConnectSwiftWeb",
- platforms: [
- .macOS(.v15)
- ],
- dependencies: [
- // 💧 A server-side Swift web framework.
- .package(url: "https://github.com/vapor/vapor.git", from: "4.110.1"),
- // 🔵 Non-blocking, event-driven networking for Swift. Used for custom executors
- .package(url: "https://github.com/apple/swift-nio.git", from: "2.65.0"),
- .package(url: "https://github.com/apache/spark-connect-swift.git", branch: "v0.1.0"),
- ],
- targets: [
- .executableTarget(
- name: "SparkConnectSwiftWeb",
- dependencies: [
- .product(name: "Vapor", package: "vapor"),
- .product(name: "NIOCore", package: "swift-nio"),
- .product(name: "NIOPosix", package: "swift-nio"),
- .product(name: "SparkConnect", package: "spark-connect-swift"),
- ],
- swiftSettings: swiftSettings
- )
- ]
-)
-
-var swiftSettings: [SwiftSetting] {
- [
- .enableUpcomingFeature("ExistentialAny")
- ]
-}
diff --git a/Examples/web/README.md b/Examples/web/README.md
deleted file mode 100644
index c744ef2..0000000
--- a/Examples/web/README.md
+++ /dev/null
@@ -1,113 +0,0 @@
-# A Swift Application with Apache Spark Connect Swift Client
-
-This project is designed to illustrate a Swift-based HTTP WebServer with Apache Spark Connect.
-
-- https://swiftpackageindex.com/vapor/vapor
-
-## Create a Swift project
-
-```
-brew install vapor
-vapor new spark-connect-swift-web -n
-```
-
-## Use `Apache Spark Connect Swift Client` package.
-
-```
-$ git diff HEAD
-diff --git a/Package.swift b/Package.swift
-index 477bcbd..3e7bb06 100644
---- a/Package.swift
-+++ b/Package.swift
-@@ -4,13 +4,14 @@ import PackageDescription
- let package = Package(
- name: "SparkConnectSwiftWebapp",
- platforms: [
-- .macOS(.v13)
-+ .macOS(.v15)
- ],
- dependencies: [
- // 💧 A server-side Swift web framework.
- .package(url: "https://github.com/vapor/vapor.git", from: "4.110.1"),
- // 🔵 Non-blocking, event-driven networking for Swift. Used for custom executors
- .package(url: "https://github.com/apple/swift-nio.git", from: "2.65.0"),
-+ .package(url: "https://github.com/apache/spark-connect-swift.git", branch: "main"),
- ],
- targets: [
- .executableTarget(
-@@ -19,6 +20,7 @@ let package = Package(
- .product(name: "Vapor", package: "vapor"),
- .product(name: "NIOCore", package: "swift-nio"),
- .product(name: "NIOPosix", package: "swift-nio"),
-+ .product(name: "SparkConnect", package: "spark-connect-swift"),
- ],
- swiftSettings: swiftSettings
- ),
-diff --git a/Sources/SparkConnectSwiftWebapp/routes.swift b/Sources/SparkConnectSwiftWebapp/routes.swift
-index 2edcc8f..22313c8 100644
---- a/Sources/SparkConnectSwiftWebapp/routes.swift
-+++ b/Sources/SparkConnectSwiftWebapp/routes.swift
-@@ -1,4 +1,5 @@
- import Vapor
-+import SparkConnect
-
- func routes(_ app: Application) throws {
- app.get { req async in
-@@ -6,6 +7,15 @@ func routes(_ app: Application) throws {
- }
-
- app.get("hello") { req async -> String in
-- "Hello, world!"
-+ return await Task {
-+ do {
-+ let spark = try await SparkSession.builder.getOrCreate()
-+ let response = "Hi, this is powered by the Apache Spark \(await spark.version)."
-+ await spark.stop()
-+ return response
-+ } catch {
-+ return "Fail to connect: \(error)"
-+ }
-+ }.value
- }
- }
-```
-
-## How to run
-
-Prepare `Spark Connect Server` via running Docker image.
-
-```
-docker run --rm -p 15002:15002 apache/spark:4.0.0-preview2 bash -c "/opt/spark/sbin/start-connect-server.sh --wait"
-```
-
-Build an application Docker image.
-
-```
-$ docker build -t apache/spark-connect-swift:web .
-$ docker images apache/spark-connect-swift:web
-REPOSITORY TAG IMAGE ID CREATED SIZE
-apache/spark-connect-swift web 3fd2422fdbee 27 seconds ago 417MB
-```
-
-Run `web` docker image
-
-```
-$ docker run -it --rm -p 8080:8080 -e SPARK_REMOTE=sc://host.docker.internal:15002 apache/spark-connect-swift:web
-[ NOTICE ] Server started on http://127.0.0.1:8080
-```
-
-Connect to the Swift Web Server to talk with `Apache Spark`.
-
-```
-$ curl http://127.0.0.1:8080/
-Welcome to the Swift world. Say hello!%
-
-$ curl http://127.0.0.1:8080/hello
-Hi, this is powered by the Apache Spark 4.0.0.%
-```
-
-Run from source code.
-
-```
-$ swift run
-```
diff --git a/Examples/web/Sources/SparkConnectSwiftWeb/configure.swift b/Examples/web/Sources/SparkConnectSwiftWeb/configure.swift
deleted file mode 100644
index f70192a..0000000
--- a/Examples/web/Sources/SparkConnectSwiftWeb/configure.swift
+++ /dev/null
@@ -1,27 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-
-import Vapor
-
-// configures your application
-public func configure(_ app: Application) async throws {
- app.http.server.configuration.hostname = "0.0.0.0"
- // register routes
- try routes(app)
-}
diff --git a/Examples/web/Sources/SparkConnectSwiftWeb/entrypoint.swift b/Examples/web/Sources/SparkConnectSwiftWeb/entrypoint.swift
deleted file mode 100644
index 0be116f..0000000
--- a/Examples/web/Sources/SparkConnectSwiftWeb/entrypoint.swift
+++ /dev/null
@@ -1,50 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-
-import Logging
-import NIOCore
-import NIOPosix
-import Vapor
-
-@main
-enum Entrypoint {
- static func main() async throws {
- var env = try Environment.detect()
- try LoggingSystem.bootstrap(from: &env)
-
- let app = try await Application.make(env)
-
- // This attempts to install NIO as the Swift Concurrency global executor.
- // You can enable it if you'd like to reduce the amount of context switching between NIO and Swift Concurrency.
- // Note: this has caused issues with some libraries that use `.wait()` and cleanly shutting down.
- // If enabled, you should be careful about calling async functions before this point as it can cause assertion failures.
- // let executorTakeoverSuccess = NIOSingletons.unsafeTryInstallSingletonPosixEventLoopGroupAsConcurrencyGlobalExecutor()
- // app.logger.debug("Tried to install SwiftNIO's EventLoopGroup as Swift's global concurrency executor", metadata: ["success": .stringConvertible(executorTakeoverSuccess)])
-
- do {
- try await configure(app)
- try await app.execute()
- } catch {
- app.logger.report(error: error)
- try? await app.asyncShutdown()
- throw error
- }
- try await app.asyncShutdown()
- }
-}
diff --git a/Examples/web/Sources/SparkConnectSwiftWeb/routes.swift b/Examples/web/Sources/SparkConnectSwiftWeb/routes.swift
deleted file mode 100644
index c6a1d91..0000000
--- a/Examples/web/Sources/SparkConnectSwiftWeb/routes.swift
+++ /dev/null
@@ -1,40 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-
-import SparkConnect
-import Vapor
-
-func routes(_ app: Application) throws {
- app.get { req async in
- "Welcome to the Swift world. Say hello!"
- }
-
- app.get("hello") { req async -> String in
- return await Task {
- do {
- let spark = try await SparkSession.builder.getOrCreate()
- let response = "Hi, this is powered by the Apache Spark \(await spark.version)."
- await spark.stop()
- return response
- } catch {
- return "Fail to connect: \(error)"
- }
- }.value
- }
-}
diff --git a/LICENSE b/LICENSE
deleted file mode 100644
index 261eeb9..0000000
--- a/LICENSE
+++ /dev/null
@@ -1,201 +0,0 @@
- Apache License
- Version 2.0, January 2004
- http://www.apache.org/licenses/
-
- TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
- 1. Definitions.
-
- "License" shall mean the terms and conditions for use, reproduction,
- and distribution as defined by Sections 1 through 9 of this document.
-
- "Licensor" shall mean the copyright owner or entity authorized by
- the copyright owner that is granting the License.
-
- "Legal Entity" shall mean the union of the acting entity and all
- other entities that control, are controlled by, or are under common
- control with that entity. For the purposes of this definition,
- "control" means (i) the power, direct or indirect, to cause the
- direction or management of such entity, whether by contract or
- otherwise, or (ii) ownership of fifty percent (50%) or more of the
- outstanding shares, or (iii) beneficial ownership of such entity.
-
- "You" (or "Your") shall mean an individual or Legal Entity
- exercising permissions granted by this License.
-
- "Source" form shall mean the preferred form for making modifications,
- including but not limited to software source code, documentation
- source, and configuration files.
-
- "Object" form shall mean any form resulting from mechanical
- transformation or translation of a Source form, including but
- not limited to compiled object code, generated documentation,
- and conversions to other media types.
-
- "Work" shall mean the work of authorship, whether in Source or
- Object form, made available under the License, as indicated by a
- copyright notice that is included in or attached to the work
- (an example is provided in the Appendix below).
-
- "Derivative Works" shall mean any work, whether in Source or Object
- form, that is based on (or derived from) the Work and for which the
- editorial revisions, annotations, elaborations, or other modifications
- represent, as a whole, an original work of authorship. For the purposes
- of this License, Derivative Works shall not include works that remain
- separable from, or merely link (or bind by name) to the interfaces of,
- the Work and Derivative Works thereof.
-
- "Contribution" shall mean any work of authorship, including
- the original version of the Work and any modifications or additions
- to that Work or Derivative Works thereof, that is intentionally
- submitted to Licensor for inclusion in the Work by the copyright owner
- or by an individual or Legal Entity authorized to submit on behalf of
- the copyright owner. For the purposes of this definition, "submitted"
- means any form of electronic, verbal, or written communication sent
- to the Licensor or its representatives, including but not limited to
- communication on electronic mailing lists, source code control systems,
- and issue tracking systems that are managed by, or on behalf of, the
- Licensor for the purpose of discussing and improving the Work, but
- excluding communication that is conspicuously marked or otherwise
- designated in writing by the copyright owner as "Not a Contribution."
-
- "Contributor" shall mean Licensor and any individual or Legal Entity
- on behalf of whom a Contribution has been received by Licensor and
- subsequently incorporated within the Work.
-
- 2. Grant of Copyright License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- copyright license to reproduce, prepare Derivative Works of,
- publicly display, publicly perform, sublicense, and distribute the
- Work and such Derivative Works in Source or Object form.
-
- 3. Grant of Patent License. Subject to the terms and conditions of
- this License, each Contributor hereby grants to You a perpetual,
- worldwide, non-exclusive, no-charge, royalty-free, irrevocable
- (except as stated in this section) patent license to make, have made,
- use, offer to sell, sell, import, and otherwise transfer the Work,
- where such license applies only to those patent claims licensable
- by such Contributor that are necessarily infringed by their
- Contribution(s) alone or by combination of their Contribution(s)
- with the Work to which such Contribution(s) was submitted. If You
- institute patent litigation against any entity (including a
- cross-claim or counterclaim in a lawsuit) alleging that the Work
- or a Contribution incorporated within the Work constitutes direct
- or contributory patent infringement, then any patent licenses
- granted to You under this License for that Work shall terminate
- as of the date such litigation is filed.
-
- 4. Redistribution. You may reproduce and distribute copies of the
- Work or Derivative Works thereof in any medium, with or without
- modifications, and in Source or Object form, provided that You
- meet the following conditions:
-
- (a) You must give any other recipients of the Work or
- Derivative Works a copy of this License; and
-
- (b) You must cause any modified files to carry prominent notices
- stating that You changed the files; and
-
- (c) You must retain, in the Source form of any Derivative Works
- that You distribute, all copyright, patent, trademark, and
- attribution notices from the Source form of the Work,
- excluding those notices that do not pertain to any part of
- the Derivative Works; and
-
- (d) If the Work includes a "NOTICE" text file as part of its
- distribution, then any Derivative Works that You distribute must
- include a readable copy of the attribution notices contained
- within such NOTICE file, excluding those notices that do not
- pertain to any part of the Derivative Works, in at least one
- of the following places: within a NOTICE text file distributed
- as part of the Derivative Works; within the Source form or
- documentation, if provided along with the Derivative Works; or,
- within a display generated by the Derivative Works, if and
- wherever such third-party notices normally appear. The contents
- of the NOTICE file are for informational purposes only and
- do not modify the License. You may add Your own attribution
- notices within Derivative Works that You distribute, alongside
- or as an addendum to the NOTICE text from the Work, provided
- that such additional attribution notices cannot be construed
- as modifying the License.
-
- You may add Your own copyright statement to Your modifications and
- may provide additional or different license terms and conditions
- for use, reproduction, or distribution of Your modifications, or
- for any such Derivative Works as a whole, provided Your use,
- reproduction, and distribution of the Work otherwise complies with
- the conditions stated in this License.
-
- 5. Submission of Contributions. Unless You explicitly state otherwise,
- any Contribution intentionally submitted for inclusion in the Work
- by You to the Licensor shall be under the terms and conditions of
- this License, without any additional terms or conditions.
- Notwithstanding the above, nothing herein shall supersede or modify
- the terms of any separate license agreement you may have executed
- with Licensor regarding such Contributions.
-
- 6. Trademarks. This License does not grant permission to use the trade
- names, trademarks, service marks, or product names of the Licensor,
- except as required for reasonable and customary use in describing the
- origin of the Work and reproducing the content of the NOTICE file.
-
- 7. Disclaimer of Warranty. Unless required by applicable law or
- agreed to in writing, Licensor provides the Work (and each
- Contributor provides its Contributions) on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
- implied, including, without limitation, any warranties or conditions
- of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
- PARTICULAR PURPOSE. You are solely responsible for determining the
- appropriateness of using or redistributing the Work and assume any
- risks associated with Your exercise of permissions under this License.
-
- 8. Limitation of Liability. In no event and under no legal theory,
- whether in tort (including negligence), contract, or otherwise,
- unless required by applicable law (such as deliberate and grossly
- negligent acts) or agreed to in writing, shall any Contributor be
- liable to You for damages, including any direct, indirect, special,
- incidental, or consequential damages of any character arising as a
- result of this License or out of the use or inability to use the
- Work (including but not limited to damages for loss of goodwill,
- work stoppage, computer failure or malfunction, or any and all
- other commercial damages or losses), even if such Contributor
- has been advised of the possibility of such damages.
-
- 9. Accepting Warranty or Additional Liability. While redistributing
- the Work or Derivative Works thereof, You may choose to offer,
- and charge a fee for, acceptance of support, warranty, indemnity,
- or other liability obligations and/or rights consistent with this
- License. However, in accepting such obligations, You may act only
- on Your own behalf and on Your sole responsibility, not on behalf
- of any other Contributor, and only if You agree to indemnify,
- defend, and hold each Contributor harmless for any liability
- incurred by, or claims asserted against, such Contributor by reason
- of your accepting any such warranty or additional liability.
-
- END OF TERMS AND CONDITIONS
-
- APPENDIX: How to apply the Apache License to your work.
-
- To apply the Apache License to your work, attach the following
- boilerplate notice, with the fields enclosed by brackets "[]"
- replaced with your own identifying information. (Don't include
- the brackets!) The text should be enclosed in the appropriate
- comment syntax for the file format. We also recommend that a
- file or class name and description of purpose be included on the
- same "printed page" as the copyright notice for easier
- identification within third-party archives.
-
- Copyright [yyyy] [name of copyright owner]
-
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
diff --git a/NOTICE b/NOTICE
deleted file mode 100644
index b110e07..0000000
--- a/NOTICE
+++ /dev/null
@@ -1,5 +0,0 @@
-Apache Spark
-Copyright 2014 and onwards The Apache Software Foundation.
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
diff --git a/Package.swift b/Package.swift
deleted file mode 100644
index ff0150d..0000000
--- a/Package.swift
+++ /dev/null
@@ -1,67 +0,0 @@
-// swift-tools-version: 6.0
-// The swift-tools-version declares the minimum version of Swift required to build this package.
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-import PackageDescription
-
-let package = Package(
- name: "SparkConnect",
- platforms: [
- .macOS(.v15),
- .iOS(.v18),
- .watchOS(.v11),
- .tvOS(.v18),
- ],
- products: [
- .library(
- name: "SparkConnect",
- targets: ["SparkConnect"])
- ],
- dependencies: [
- .package(url: "https://github.com/grpc/grpc-swift.git", from: "2.2.0"),
- .package(url: "https://github.com/grpc/grpc-swift-protobuf.git", from: "1.2.0"),
- .package(url: "https://github.com/grpc/grpc-swift-nio-transport.git", from: "1.0.3"),
- .package(url: "https://github.com/google/flatbuffers.git", branch: "v25.2.10"),
- ],
- targets: [
- .target(
- name: "SparkConnect",
- dependencies: [
- .product(name: "GRPCCore", package: "grpc-swift"),
- .product(name: "GRPCProtobuf", package: "grpc-swift-protobuf"),
- .product(name: "GRPCNIOTransportHTTP2", package: "grpc-swift-nio-transport"),
- .product(name: "FlatBuffers", package: "flatbuffers"),
- ],
- resources: [
- .process("Documentation.docc")
- ]
- ),
- .executableTarget(
- name: "SparkSQLRepl",
- dependencies: ["SparkConnect"]
- ),
- .testTarget(
- name: "SparkConnectTests",
- dependencies: ["SparkConnect"],
- resources: [
- .copy("Resources/queries")
- ]
- ),
- ]
-)
diff --git a/README.md b/README.md
deleted file mode 100644
index 00cb058..0000000
--- a/README.md
+++ /dev/null
@@ -1,210 +0,0 @@
-# Apache Spark Connect Client for Swift
-
-[](https://github.com/apache/spark-connect-swift/blob/main/.github/workflows/build_and_test.yml)
-[](https://swiftpackageindex.com/apache/spark-connect-swift)
-[](https://swiftpackageindex.com/apache/spark-connect-swift)
-
-This is an experimental Swift library to show how to connect to a remote Apache Spark Connect Server and run SQL statements to manipulate remote data.
-
-So far, this library project is tracking the upstream changes like the [Apache Spark](https://spark.apache.org) 4.0.0 RC5 release and [Apache Arrow](https://arrow.apache.org) project's Swift-support.
-
-## Requirement
-
-- [Apache Spark 4.0.0 RC5 (April 2025)](https://dist.apache.org/repos/dist/dev/spark/v4.0.0-rc5-bin/)
-- [Swift 6.0 (2024) or 6.1 (2025)](https://swift.org)
-- [gRPC Swift 2.2 (May 2025)](https://github.com/grpc/grpc-swift/releases/tag/2.2.0)
-- [gRPC Swift Protobuf 1.2 (April 2025)](https://github.com/grpc/grpc-swift-protobuf/releases/tag/1.2.0)
-- [gRPC Swift NIO Transport 1.0 (March 2025)](https://github.com/grpc/grpc-swift-nio-transport/releases/tag/1.0.3)
-- [FlatBuffers v25.2.10 (February 2025)](https://github.com/google/flatbuffers/releases/tag/v25.2.10)
-- [Apache Arrow Swift](https://github.com/apache/arrow/tree/main/swift)
-
-## How to use in your apps
-
-Create a Swift project.
-
-```bash
-mkdir SparkConnectSwiftApp
-cd SparkConnectSwiftApp
-swift package init --name SparkConnectSwiftApp --type executable
-```
-
-Add `SparkConnect` package to the dependency like the following
-
-```bash
-$ cat Package.swift
-import PackageDescription
-
-let package = Package(
- name: "SparkConnectSwiftApp",
- platforms: [
- .macOS(.v15)
- ],
- dependencies: [
- .package(url: "https://github.com/apache/spark-connect-swift.git", branch: "main")
- ],
- targets: [
- .executableTarget(
- name: "SparkConnectSwiftApp",
- dependencies: [.product(name: "SparkConnect", package: "spark-connect-swift")]
- )
- ]
-)
-```
-
-Use `SparkSession` of `SparkConnect` module in Swift.
-
-```bash
-$ cat Sources/main.swift
-
-import SparkConnect
-
-let spark = try await SparkSession.builder.getOrCreate()
-print("Connected to Apache Spark \(await spark.version) Server")
-
-let statements = [
- "DROP TABLE IF EXISTS t",
- "CREATE TABLE IF NOT EXISTS t(a INT) USING ORC",
- "INSERT INTO t VALUES (1), (2), (3)",
-]
-
-for s in statements {
- print("EXECUTE: \(s)")
- _ = try await spark.sql(s).count()
-}
-print("SELECT * FROM t")
-try await spark.sql("SELECT * FROM t").cache().show()
-
-try await spark.range(10).filter("id % 2 == 0").write.mode("overwrite").orc("/tmp/orc")
-try await spark.read.orc("/tmp/orc").show()
-
-await spark.stop()
-```
-
-Run your Swift application.
-
-```bash
-$ swift run
-...
-Connected to Apache Spark 4.0.0 Server
-EXECUTE: DROP TABLE IF EXISTS t
-EXECUTE: CREATE TABLE IF NOT EXISTS t(a INT)
-EXECUTE: INSERT INTO t VALUES (1), (2), (3)
-SELECT * FROM t
-+---+
-| a |
-+---+
-| 2 |
-| 1 |
-| 3 |
-+---+
-+----+
-| id |
-+----+
-| 2 |
-| 6 |
-| 0 |
-| 8 |
-| 4 |
-+----+
-```
-
-You can find more complete examples including Web Server and Streaming applications in the `Examples` directory.
-
-## How to use `Spark SQL REPL` via `Spark Connect for Swift`
-
-This project also provides `Spark SQL REPL`. You can run it directly from this repository.
-
-```bash
-$ swift run
-...
-Build of product 'SparkSQLRepl' complete! (2.33s)
-Connected to Apache Spark 4.0.0 Server
-spark-sql (default)> SHOW DATABASES;
-+---------+
-|namespace|
-+---------+
-| default|
-+---------+
-
-Time taken: 30 ms
-spark-sql (default)> CREATE DATABASE db1;
-++
-||
-++
-++
-
-Time taken: 31 ms
-spark-sql (default)> USE db1;
-++
-||
-++
-++
-
-Time taken: 27 ms
-spark-sql (db1)> CREATE TABLE t1 AS SELECT * FROM RANGE(10);
-++
-||
-++
-++
-
-Time taken: 99 ms
-spark-sql (db1)> SELECT * FROM t1;
-+---+
-| id|
-+---+
-| 1|
-| 5|
-| 3|
-| 0|
-| 6|
-| 9|
-| 4|
-| 8|
-| 7|
-| 2|
-+---+
-
-Time taken: 80 ms
-spark-sql (db1)> USE default;
-++
-||
-++
-++
-
-Time taken: 26 ms
-spark-sql (default)> DROP DATABASE db1 CASCADE;
-++
-||
-++
-++
-spark-sql (default)> exit;
-```
-
-Apache Spark 4 supports [SQL Pipe Syntax](https://dist.apache.org/repos/dist/dev/spark/v4.0.0-rc5-docs/_site/sql-pipe-syntax.html).
-
-```
-$ swift run
-...
-Build of product 'SparkSQLRepl' complete! (2.33s)
-Connected to Apache Spark 4.0.0 Server
-spark-sql (default)>
-FROM ORC.`/opt/spark/examples/src/main/resources/users.orc`
-|> AGGREGATE COUNT(*) cnt
- GROUP BY name
-|> ORDER BY cnt DESC, name ASC
-;
-+------+---+
-| name|cnt|
-+------+---+
-|Alyssa| 1|
-| Ben| 1|
-+------+---+
-
-Time taken: 159 ms
-```
-
-You can use `SPARK_REMOTE` to specify the [Spark Connect connection string](https://spark.apache.org/docs/latest/spark-connect-overview.html#set-sparkremote-environment-variable) in order to provide more options.
-
-```bash
-SPARK_REMOTE=sc://localhost swift run
-```
diff --git a/Sources/SparkConnect/ArrowArray.swift b/Sources/SparkConnect/ArrowArray.swift
deleted file mode 100644
index e3d61bb..0000000
--- a/Sources/SparkConnect/ArrowArray.swift
+++ /dev/null
@@ -1,343 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import Foundation
-
-/// @nodoc
-public protocol ArrowArrayHolder {
- var type: ArrowType { get }
- var length: UInt { get }
- var nullCount: UInt { get }
- var array: AnyArray { get }
- var data: ArrowData { get }
- var getBufferData: () -> [Data] { get }
- var getBufferDataSizes: () -> [Int] { get }
- var getArrowColumn: (ArrowField, [ArrowArrayHolder]) throws -> ArrowColumn { get }
-}
-
-/// @nodoc
-public class ArrowArrayHolderImpl: ArrowArrayHolder {
- public let data: ArrowData
- public let type: ArrowType
- public let length: UInt
- public let nullCount: UInt
- public let array: AnyArray
- public let getBufferData: () -> [Data]
- public let getBufferDataSizes: () -> [Int]
- public let getArrowColumn: (ArrowField, [ArrowArrayHolder]) throws -> ArrowColumn
- public init<T>(_ arrowArray: ArrowArray<T>) {
- self.array = arrowArray
- self.data = arrowArray.arrowData
- self.length = arrowArray.length
- self.type = arrowArray.arrowData.type
- self.nullCount = arrowArray.nullCount
- self.getBufferData = { () -> [Data] in
- var bufferData = [Data]()
- for buffer in arrowArray.arrowData.buffers {
- bufferData.append(Data())
- buffer.append(to: &bufferData[bufferData.count - 1])
- }
-
- return bufferData
- }
-
- self.getBufferDataSizes = { () -> [Int] in
- var bufferDataSizes = [Int]()
- for buffer in arrowArray.arrowData.buffers {
- bufferDataSizes.append(Int(buffer.capacity))
- }
-
- return bufferDataSizes
- }
-
- self.getArrowColumn = {
- (field: ArrowField, arrayHolders: [ArrowArrayHolder]) throws -> ArrowColumn in
- var arrays = [ArrowArray<T>]()
- for arrayHolder in arrayHolders {
- if let array = arrayHolder.array as? ArrowArray<T> {
- arrays.append(array)
- }
- }
-
- return ArrowColumn(field, chunked: ChunkedArrayHolder(try ChunkedArray<T>(arrays)))
- }
- }
-
- public static func loadArray( // swiftlint:disable:this cyclomatic_complexity
- _ arrowType: ArrowType, with: ArrowData
- ) throws -> ArrowArrayHolder {
- switch arrowType.id {
- case .int8:
- return try ArrowArrayHolderImpl(FixedArray<Int8>(with))
- case .int16:
- return try ArrowArrayHolderImpl(FixedArray<Int16>(with))
- case .int32:
- return try ArrowArrayHolderImpl(FixedArray<Int32>(with))
- case .int64:
- return try ArrowArrayHolderImpl(FixedArray<Int64>(with))
- case .uint8:
- return try ArrowArrayHolderImpl(FixedArray<UInt8>(with))
- case .uint16:
- return try ArrowArrayHolderImpl(FixedArray<UInt16>(with))
- case .uint32:
- return try ArrowArrayHolderImpl(FixedArray<UInt32>(with))
- case .uint64:
- return try ArrowArrayHolderImpl(FixedArray<UInt64>(with))
- case .double:
- return try ArrowArrayHolderImpl(FixedArray<Double>(with))
- case .float:
- return try ArrowArrayHolderImpl(FixedArray<Float>(with))
- case .date32:
- return try ArrowArrayHolderImpl(Date32Array(with))
- case .date64:
- return try ArrowArrayHolderImpl(Date64Array(with))
- case .time32:
- return try ArrowArrayHolderImpl(Time32Array(with))
- case .time64:
- return try ArrowArrayHolderImpl(Time64Array(with))
- case .string:
- return try ArrowArrayHolderImpl(StringArray(with))
- case .boolean:
- return try ArrowArrayHolderImpl(BoolArray(with))
- case .binary:
- return try ArrowArrayHolderImpl(BinaryArray(with))
- case .strct:
- return try ArrowArrayHolderImpl(StructArray(with))
- default:
- throw ArrowError.invalid("Array not found for type: \(arrowType)")
- }
- }
-}
-
-/// @nodoc
-public class ArrowArray<T>: AsString, AnyArray {
- public typealias ItemType = T
- public let arrowData: ArrowData
- public var nullCount: UInt { return self.arrowData.nullCount }
- public var length: UInt { return self.arrowData.length }
-
- public required init(_ arrowData: ArrowData) throws {
- self.arrowData = arrowData
- }
-
- public func isNull(_ at: UInt) throws -> Bool {
- if at >= self.length {
- throw ArrowError.outOfBounds(index: Int64(at))
- }
-
- return self.arrowData.isNull(at)
- }
-
- public subscript(_ index: UInt) -> T? {
- fatalError("subscript() has not been implemented")
- }
-
- public func asString(_ index: UInt) -> String {
- if self[index] == nil {
- return ""
- }
-
- return "\(self[index]!)"
- }
-
- public func asAny(_ index: UInt) -> Any? {
- if self[index] == nil {
- return nil
- }
-
- return self[index]!
- }
-}
-
-/// @nodoc
-public class FixedArray<T>: ArrowArray<T> {
- public override subscript(_ index: UInt) -> T? {
- if self.arrowData.isNull(index) {
- return nil
- }
-
- let byteOffset = self.arrowData.stride * Int(index)
- return self.arrowData.buffers[1].rawPointer.advanced(by: byteOffset).load(as: T.self)
- }
-}
-
-/// @nodoc
-public class StringArray: ArrowArray<String> {
- public override subscript(_ index: UInt) -> String? {
- let offsetIndex = MemoryLayout<Int32>.stride * Int(index)
- if self.arrowData.isNull(index) {
- return nil
- }
-
- let offsets = self.arrowData.buffers[1]
- let values = self.arrowData.buffers[2]
-
- var startIndex: Int32 = 0
- if index > 0 {
- startIndex = offsets.rawPointer.advanced(by: offsetIndex).load(as: Int32.self)
- }
-
- let endIndex = offsets.rawPointer.advanced(by: offsetIndex + MemoryLayout<Int32>.stride)
- .load(as: Int32.self)
- let arrayLength = Int(endIndex - startIndex)
- let rawPointer = values.rawPointer.advanced(by: Int(startIndex))
- .bindMemory(to: UInt8.self, capacity: arrayLength)
- let buffer = UnsafeBufferPointer<UInt8>(start: rawPointer, count: arrayLength)
- let byteArray = Array(buffer)
- return String(data: Data(byteArray), encoding: .utf8)
- }
-}
-
-/// @nodoc
-public class BoolArray: ArrowArray<Bool> {
- public override subscript(_ index: UInt) -> Bool? {
- if self.arrowData.isNull(index) {
- return nil
- }
-
- let valueBuffer = self.arrowData.buffers[1]
- return BitUtility.isSet(index, buffer: valueBuffer)
- }
-}
-
-/// @nodoc
-public class Date32Array: ArrowArray<Date> {
- public override subscript(_ index: UInt) -> Date? {
- if self.arrowData.isNull(index) {
- return nil
- }
-
- let byteOffset = self.arrowData.stride * Int(index)
- let milliseconds = self.arrowData.buffers[1].rawPointer.advanced(by: byteOffset).load(
- as: UInt32.self)
- return Date(timeIntervalSince1970: TimeInterval(milliseconds * 86400))
- }
-}
-
-/// @nodoc
-public class Date64Array: ArrowArray<Date> {
- public override subscript(_ index: UInt) -> Date? {
- if self.arrowData.isNull(index) {
- return nil
- }
-
- let byteOffset = self.arrowData.stride * Int(index)
- let milliseconds = self.arrowData.buffers[1].rawPointer.advanced(by: byteOffset).load(
- as: UInt64.self)
- return Date(timeIntervalSince1970: TimeInterval(milliseconds / 1000))
- }
-}
-
-/// @nodoc
-public class Time32Array: FixedArray<Time32> {}
-/// @nodoc
-public class Time64Array: FixedArray<Time64> {}
-
-/// @nodoc
-public class BinaryArray: ArrowArray<Data> {
- public struct Options {
- public var printAsHex = false
- public var printEncoding: String.Encoding = .utf8
- }
-
- public var options = Options()
-
- public override subscript(_ index: UInt) -> Data? {
- let offsetIndex = MemoryLayout<Int32>.stride * Int(index)
- if self.arrowData.isNull(index) {
- return nil
- }
-
- let offsets = self.arrowData.buffers[1]
- let values = self.arrowData.buffers[2]
- var startIndex: Int32 = 0
- if index > 0 {
- startIndex = offsets.rawPointer.advanced(by: offsetIndex).load(as: Int32.self)
- }
-
- let endIndex = offsets.rawPointer.advanced(by: offsetIndex + MemoryLayout<Int32>.stride)
- .load(as: Int32.self)
- let arrayLength = Int(endIndex - startIndex)
- let rawPointer = values.rawPointer.advanced(by: Int(startIndex))
- .bindMemory(to: UInt8.self, capacity: arrayLength)
- let buffer = UnsafeBufferPointer<UInt8>(start: rawPointer, count: arrayLength)
- let byteArray = Array(buffer)
- return Data(byteArray)
- }
-
- public override func asString(_ index: UInt) -> String {
- if self[index] == nil { return "" }
- let data = self[index]!
- if options.printAsHex {
- return data.hexEncodedString()
- } else {
- return String(data: data, encoding: .utf8)!
- }
- }
-}
-
-/// @nodoc
-public class StructArray: ArrowArray<[Any?]> {
- public private(set) var arrowFields: [ArrowArrayHolder]?
- public required init(_ arrowData: ArrowData) throws {
- try super.init(arrowData)
- var fields = [ArrowArrayHolder]()
- for child in arrowData.children {
- fields.append(try ArrowArrayHolderImpl.loadArray(child.type, with: child))
- }
-
- self.arrowFields = fields
- }
-
- public override subscript(_ index: UInt) -> [Any?]? {
- if self.arrowData.isNull(index) {
- return nil
- }
-
- if let fields = arrowFields {
- var result = [Any?]()
- for field in fields {
- result.append(field.array.asAny(index))
- }
-
- return result
- }
-
- return nil
- }
-
- public override func asString(_ index: UInt) -> String {
- if self.arrowData.isNull(index) {
- return ""
- }
-
- var output = "{"
- if let fields = arrowFields {
- for fieldIndex in 0..<fields.count {
- let asStr = fields[fieldIndex].array as? AsString
- if fieldIndex == 0 {
- output.append("\(asStr!.asString(index))")
- } else {
- output.append(",\(asStr!.asString(index))")
- }
- }
- }
-
- output += "}"
- return output
- }
-}
diff --git a/Sources/SparkConnect/ArrowArrayBuilder.swift b/Sources/SparkConnect/ArrowArrayBuilder.swift
deleted file mode 100644
index 5062127..0000000
--- a/Sources/SparkConnect/ArrowArrayBuilder.swift
+++ /dev/null
@@ -1,341 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import Foundation
-
-/// @nodoc
-public protocol ArrowArrayHolderBuilder {
- func toHolder() throws -> ArrowArrayHolder
- func appendAny(_ val: Any?)
-}
-
-public class ArrowArrayBuilder<T: ArrowBufferBuilder, U: ArrowArray<T.ItemType>>:
- ArrowArrayHolderBuilder
-{
- let type: ArrowType
- let bufferBuilder: T
- public var length: UInt { return self.bufferBuilder.length }
- public var capacity: UInt { return self.bufferBuilder.capacity }
- public var nullCount: UInt { return self.bufferBuilder.nullCount }
- public var offset: UInt { return self.bufferBuilder.offset }
-
- fileprivate init(_ type: ArrowType) throws {
- self.type = type
- self.bufferBuilder = try T()
- }
-
- public func append(_ vals: T.ItemType?...) {
- for val in vals {
- self.bufferBuilder.append(val)
- }
- }
-
- public func append(_ vals: [T.ItemType?]) {
- for val in vals {
- self.bufferBuilder.append(val)
- }
- }
-
- public func append(_ val: T.ItemType?) {
- self.bufferBuilder.append(val)
- }
-
- public func appendAny(_ val: Any?) {
- self.bufferBuilder.append(val as? T.ItemType)
- }
-
- public func finish() throws -> ArrowArray<T.ItemType> {
- let buffers = self.bufferBuilder.finish()
- let arrowData = try ArrowData(self.type, buffers: buffers, nullCount: self.nullCount)
- let array = try U(arrowData)
- return array
- }
-
- public func getStride() -> Int {
- return self.type.getStride()
- }
-
- public func toHolder() throws -> ArrowArrayHolder {
- return try ArrowArrayHolderImpl(self.finish())
- }
-}
-
-public class NumberArrayBuilder<T>: ArrowArrayBuilder<FixedBufferBuilder<T>, FixedArray<T>> {
- fileprivate convenience init() throws {
- try self.init(ArrowType(ArrowType.infoForNumericType(T.self)))
- }
-}
-
-public class StringArrayBuilder: ArrowArrayBuilder<VariableBufferBuilder<String>, StringArray> {
- fileprivate convenience init() throws {
- try self.init(ArrowType(ArrowType.ArrowString))
- }
-}
-
-public class BinaryArrayBuilder: ArrowArrayBuilder<VariableBufferBuilder<Data>, BinaryArray> {
- fileprivate convenience init() throws {
- try self.init(ArrowType(ArrowType.ArrowBinary))
- }
-}
-
-public class BoolArrayBuilder: ArrowArrayBuilder<BoolBufferBuilder, BoolArray> {
- fileprivate convenience init() throws {
- try self.init(ArrowType(ArrowType.ArrowBool))
- }
-}
-
-public class Date32ArrayBuilder: ArrowArrayBuilder<Date32BufferBuilder, Date32Array> {
- fileprivate convenience init() throws {
- try self.init(ArrowType(ArrowType.ArrowDate32))
- }
-}
-
-public class Date64ArrayBuilder: ArrowArrayBuilder<Date64BufferBuilder, Date64Array> {
- fileprivate convenience init() throws {
- try self.init(ArrowType(ArrowType.ArrowDate64))
- }
-}
-
-public class Time32ArrayBuilder: ArrowArrayBuilder<FixedBufferBuilder<Time32>, Time32Array> {
- fileprivate convenience init(_ unit: ArrowTime32Unit) throws {
- try self.init(ArrowTypeTime32(unit))
- }
-}
-
-public class Time64ArrayBuilder: ArrowArrayBuilder<FixedBufferBuilder<Time64>, Time64Array> {
- fileprivate convenience init(_ unit: ArrowTime64Unit) throws {
- try self.init(ArrowTypeTime64(unit))
- }
-}
-
-public class StructArrayBuilder: ArrowArrayBuilder<StructBufferBuilder, StructArray> {
- let builders: [any ArrowArrayHolderBuilder]
- let fields: [ArrowField]
- public init(_ fields: [ArrowField], builders: [any ArrowArrayHolderBuilder]) throws {
- self.fields = fields
- self.builders = builders
- try super.init(ArrowNestedType(ArrowType.ArrowStruct, fields: fields))
- self.bufferBuilder.initializeTypeInfo(fields)
- }
-
- public init(_ fields: [ArrowField]) throws {
- self.fields = fields
- var builders = [any ArrowArrayHolderBuilder]()
- for field in fields {
- builders.append(try ArrowArrayBuilders.loadBuilder(arrowType: field.type))
- }
-
- self.builders = builders
- try super.init(ArrowNestedType(ArrowType.ArrowStruct, fields: fields))
- }
-
- public override func append(_ values: [Any?]?) {
- self.bufferBuilder.append(values)
- if let anyValues = values {
- for index in 0..<builders.count {
- self.builders[index].appendAny(anyValues[index])
- }
- } else {
- for index in 0..<builders.count {
- self.builders[index].appendAny(nil)
- }
- }
- }
-
- public override func finish() throws -> StructArray {
- let buffers = self.bufferBuilder.finish()
- var childData = [ArrowData]()
- for builder in self.builders {
- childData.append(try builder.toHolder().array.arrowData)
- }
-
- let arrowData = try ArrowData(
- self.type, buffers: buffers,
- children: childData, nullCount: self.nullCount,
- length: self.length)
- let structArray = try StructArray(arrowData)
- return structArray
- }
-}
-
-public class ArrowArrayBuilders {
- public static func loadBuilder( // swiftlint:disable:this cyclomatic_complexity
- _ builderType: Any.Type
- ) throws -> ArrowArrayHolderBuilder {
- if builderType == Int8.self || builderType == Int8?.self {
- return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<Int8>
- } else if builderType == Int16.self || builderType == Int16?.self {
- return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<Int16>
- } else if builderType == Int32.self || builderType == Int32?.self {
- return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<Int32>
- } else if builderType == Int64.self || builderType == Int64?.self {
- return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<Int64>
- } else if builderType == Float.self || builderType == Float?.self {
- return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<Float>
- } else if builderType == UInt8.self || builderType == UInt8?.self {
- return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<UInt8>
- } else if builderType == UInt16.self || builderType == UInt16?.self {
- return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<UInt16>
- } else if builderType == UInt32.self || builderType == UInt32?.self {
- return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<UInt32>
- } else if builderType == UInt64.self || builderType == UInt64?.self {
- return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<UInt64>
- } else if builderType == Double.self || builderType == Double?.self {
- return try ArrowArrayBuilders.loadNumberArrayBuilder() as NumberArrayBuilder<Double>
- } else if builderType == String.self || builderType == String?.self {
- return try ArrowArrayBuilders.loadStringArrayBuilder()
- } else if builderType == Bool.self || builderType == Bool?.self {
- return try ArrowArrayBuilders.loadBoolArrayBuilder()
- } else if builderType == Date.self || builderType == Date?.self {
- return try ArrowArrayBuilders.loadDate64ArrayBuilder()
- } else {
- throw ArrowError.invalid("Invalid type for builder: \(builderType)")
- }
- }
-
- public static func isValidBuilderType<T>(_ type: T.Type) -> Bool {
- return type == Int8?.self || type == Int16?.self || type == Int32?.self || type == Int64?.self
- || type == UInt8?.self || type == UInt16?.self || type == UInt32?.self || type == UInt64?.self
- || type == String?.self || type == Double?.self || type == Float?.self || type == Date?.self
- || type == Bool?.self || type == Bool.self || type == Int8.self || type == Int16.self
- || type == Int32.self || type == Int64.self || type == UInt8.self || type == UInt16.self
- || type == UInt32.self || type == UInt64.self || type == String.self || type == Double.self
- || type == Float.self || type == Date.self
- }
-
- public static func loadStructArrayBuilderForType<T>(_ obj: T) throws -> StructArrayBuilder {
- let mirror = Mirror(reflecting: obj)
- var builders = [ArrowArrayHolderBuilder]()
- var fields = [ArrowField]()
- for (property, value) in mirror.children {
- guard let propertyName = property else {
- continue
- }
-
- let builderType = type(of: value)
- let arrowType = ArrowType(ArrowType.infoForType(builderType))
- fields.append(ArrowField(propertyName, type: arrowType, isNullable: true))
- builders.append(try loadBuilder(arrowType: arrowType))
- }
-
- return try StructArrayBuilder(fields, builders: builders)
- }
-
- public static func loadBuilder( // swiftlint:disable:this cyclomatic_complexity
- arrowType: ArrowType
- ) throws -> ArrowArrayHolderBuilder {
- switch arrowType.id {
- case .uint8:
- return try loadNumberArrayBuilder() as NumberArrayBuilder<UInt8>
- case .uint16:
- return try loadNumberArrayBuilder() as NumberArrayBuilder<UInt16>
- case .uint32:
- return try loadNumberArrayBuilder() as NumberArrayBuilder<UInt32>
- case .uint64:
- return try loadNumberArrayBuilder() as NumberArrayBuilder<UInt64>
- case .int8:
- return try loadNumberArrayBuilder() as NumberArrayBuilder<Int8>
- case .int16:
- return try loadNumberArrayBuilder() as NumberArrayBuilder<Int16>
- case .int32:
- return try loadNumberArrayBuilder() as NumberArrayBuilder<Int32>
- case .int64:
- return try loadNumberArrayBuilder() as NumberArrayBuilder<Int64>
- case .double:
- return try loadNumberArrayBuilder() as NumberArrayBuilder<Double>
- case .float:
- return try loadNumberArrayBuilder() as NumberArrayBuilder<Float>
- case .string:
- return try StringArrayBuilder()
- case .boolean:
- return try BoolArrayBuilder()
- case .binary:
- return try BinaryArrayBuilder()
- case .date32:
- return try Date32ArrayBuilder()
- case .date64:
- return try Date64ArrayBuilder()
- case .time32:
- guard let timeType = arrowType as? ArrowTypeTime32 else {
- throw ArrowError.invalid("Expected arrow type for \(arrowType.id) not found")
- }
- return try Time32ArrayBuilder(timeType.unit)
- case .time64:
- guard let timeType = arrowType as? ArrowTypeTime64 else {
- throw ArrowError.invalid("Expected arrow type for \(arrowType.id) not found")
- }
- return try Time64ArrayBuilder(timeType.unit)
- default:
- throw ArrowError.unknownType("Builder not found for arrow type: \(arrowType.id)")
- }
- }
-
- public static func loadNumberArrayBuilder<T>() throws -> NumberArrayBuilder<T> {
- let type = T.self
- if type == Int8.self {
- return try NumberArrayBuilder<T>()
- } else if type == Int16.self {
- return try NumberArrayBuilder<T>()
- } else if type == Int32.self {
- return try NumberArrayBuilder<T>()
- } else if type == Int64.self {
- return try NumberArrayBuilder<T>()
- } else if type == UInt8.self {
- return try NumberArrayBuilder<T>()
- } else if type == UInt16.self {
- return try NumberArrayBuilder<T>()
- } else if type == UInt32.self {
- return try NumberArrayBuilder<T>()
- } else if type == UInt64.self {
- return try NumberArrayBuilder<T>()
- } else if type == Float.self {
- return try NumberArrayBuilder<T>()
- } else if type == Double.self {
- return try NumberArrayBuilder<T>()
- } else {
- throw ArrowError.unknownType("Type is invalid for NumberArrayBuilder")
- }
- }
-
- public static func loadStringArrayBuilder() throws -> StringArrayBuilder {
- return try StringArrayBuilder()
- }
-
- public static func loadBoolArrayBuilder() throws -> BoolArrayBuilder {
- return try BoolArrayBuilder()
- }
-
- public static func loadDate32ArrayBuilder() throws -> Date32ArrayBuilder {
- return try Date32ArrayBuilder()
- }
-
- public static func loadDate64ArrayBuilder() throws -> Date64ArrayBuilder {
- return try Date64ArrayBuilder()
- }
-
- public static func loadBinaryArrayBuilder() throws -> BinaryArrayBuilder {
- return try BinaryArrayBuilder()
- }
-
- public static func loadTime32ArrayBuilder(_ unit: ArrowTime32Unit) throws -> Time32ArrayBuilder {
- return try Time32ArrayBuilder(unit)
- }
-
- public static func loadTime64ArrayBuilder(_ unit: ArrowTime64Unit) throws -> Time64ArrayBuilder {
- return try Time64ArrayBuilder(unit)
- }
-}
diff --git a/Sources/SparkConnect/ArrowBuffer.swift b/Sources/SparkConnect/ArrowBuffer.swift
deleted file mode 100644
index 8b8a375..0000000
--- a/Sources/SparkConnect/ArrowBuffer.swift
+++ /dev/null
@@ -1,91 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import Foundation
-
-/// @nodoc
-public class ArrowBuffer {
- static let minLength: UInt = 1 << 5
- static let maxLength = UInt.max
- fileprivate(set) var length: UInt
- let capacity: UInt
- public let rawPointer: UnsafeMutableRawPointer
- let isMemoryOwner: Bool
-
- init(
- length: UInt, capacity: UInt, rawPointer: UnsafeMutableRawPointer, isMemoryOwner: Bool = true
- ) {
- self.length = length
- self.capacity = capacity
- self.rawPointer = rawPointer
- self.isMemoryOwner = isMemoryOwner
- }
-
- deinit {
- if isMemoryOwner {
- self.rawPointer.deallocate()
- }
- }
-
- func append(to data: inout Data) {
- let ptr = UnsafePointer(rawPointer.assumingMemoryBound(to: UInt8.self))
- data.append(ptr, count: Int(capacity))
- }
-
- static func createEmptyBuffer() -> ArrowBuffer {
- return ArrowBuffer(
- length: 0,
- capacity: 0,
- rawPointer: UnsafeMutableRawPointer.allocate(byteCount: 0, alignment: .zero))
- }
-
- static func createBuffer(_ data: [UInt8], length: UInt) -> ArrowBuffer {
- let byteCount = UInt(data.count)
- let capacity = alignTo64(byteCount)
- let memory = MemoryAllocator(64)
- let rawPointer = memory.allocateArray(Int(capacity))
- rawPointer.copyMemory(from: data, byteCount: data.count)
- return ArrowBuffer(length: length, capacity: capacity, rawPointer: rawPointer)
- }
-
- static func createBuffer(_ length: UInt, size: UInt, doAlign: Bool = true) -> ArrowBuffer {
- let actualLen = max(length, ArrowBuffer.minLength)
- let byteCount = size * actualLen
- var capacity = byteCount
- if doAlign {
- capacity = alignTo64(byteCount)
- }
-
- let memory = MemoryAllocator(64)
- let rawPointer = memory.allocateArray(Int(capacity))
- rawPointer.initializeMemory(as: UInt8.self, repeating: 0, count: Int(capacity))
- return ArrowBuffer(length: length, capacity: capacity, rawPointer: rawPointer)
- }
-
- static func copyCurrent(_ from: ArrowBuffer, to: inout ArrowBuffer, len: UInt) {
- to.rawPointer.copyMemory(from: from.rawPointer, byteCount: Int(len))
- }
-
- private static func alignTo64(_ length: UInt) -> UInt {
- let bufAlignment = length % 64
- if bufAlignment != 0 {
- return length + (64 - bufAlignment) + 8
- }
-
- return length + 8
- }
-}
diff --git a/Sources/SparkConnect/ArrowBufferBuilder.swift b/Sources/SparkConnect/ArrowBufferBuilder.swift
deleted file mode 100644
index 3c38c1d..0000000
--- a/Sources/SparkConnect/ArrowBufferBuilder.swift
+++ /dev/null
@@ -1,385 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import Foundation
-
-/// @nodoc
-public protocol ArrowBufferBuilder {
- associatedtype ItemType
- var capacity: UInt { get }
- var length: UInt { get }
- var nullCount: UInt { get }
- var offset: UInt { get }
- init() throws
- func append(_ newValue: ItemType?)
- func isNull(_ index: UInt) -> Bool
- func resize(_ length: UInt)
- func finish() -> [ArrowBuffer]
-}
-
-public class BaseBufferBuilder {
- var nulls: ArrowBuffer
- public var offset: UInt = 0
- public var capacity: UInt { return self.nulls.capacity }
- public var length: UInt = 0
- public var nullCount: UInt = 0
-
- init(_ nulls: ArrowBuffer) {
- self.nulls = nulls
- }
-
- public func isNull(_ index: UInt) -> Bool {
- return self.nulls.length == 0 || BitUtility.isSet(index + self.offset, buffer: self.nulls)
- }
-
- func resizeLength(_ data: ArrowBuffer, len: UInt = 0) -> UInt {
- if len == 0 || len < data.length * 2 {
- if data.length == 0 || data.length * 2 < ArrowBuffer.minLength {
- return ArrowBuffer.minLength
- }
- return UInt(data.length * 2)
- }
-
- return UInt(len * 2)
- }
-}
-
-public class ValuesBufferBuilder<T>: BaseBufferBuilder {
- var values: ArrowBuffer
- var stride: Int
- public override var capacity: UInt { return self.values.capacity }
-
- init(values: ArrowBuffer, nulls: ArrowBuffer, stride: Int = MemoryLayout<T>.stride) {
- self.stride = stride
- self.values = values
- super.init(nulls)
- }
-}
-
-public class FixedBufferBuilder<T>: ValuesBufferBuilder<T>, ArrowBufferBuilder {
- public typealias ItemType = T
- private let defaultVal: ItemType
- public required init() throws {
- self.defaultVal = try FixedBufferBuilder<T>.defaultValueForType()
- let values = ArrowBuffer.createBuffer(0, size: UInt(MemoryLayout<T>.stride))
- let nulls = ArrowBuffer.createBuffer(0, size: UInt(MemoryLayout<UInt8>.stride))
- super.init(values: values, nulls: nulls)
- }
-
- public func append(_ newValue: ItemType?) {
- let index = UInt(self.length)
- let byteIndex = self.stride * Int(index)
- self.length += 1
- if length > self.values.length {
- self.resize(length)
- }
-
- if let val = newValue {
- BitUtility.setBit(index + self.offset, buffer: self.nulls)
- self.values.rawPointer.advanced(by: byteIndex).storeBytes(of: val, as: T.self)
- } else {
- self.nullCount += 1
- BitUtility.clearBit(index + self.offset, buffer: self.nulls)
- self.values.rawPointer.advanced(by: byteIndex).storeBytes(of: defaultVal, as: T.self)
- }
- }
-
- public func resize(_ length: UInt) {
- if length > self.values.length {
- let resizeLength = resizeLength(self.values)
- var values = ArrowBuffer.createBuffer(resizeLength, size: UInt(MemoryLayout<T>.size))
- var nulls = ArrowBuffer.createBuffer(
- resizeLength / 8 + 1, size: UInt(MemoryLayout<UInt8>.size))
- ArrowBuffer.copyCurrent(self.values, to: &values, len: self.values.capacity)
- ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: self.nulls.capacity)
- self.values = values
- self.nulls = nulls
- }
- }
-
- public func finish() -> [ArrowBuffer] {
- let length = self.length
- var values = ArrowBuffer.createBuffer(length, size: UInt(MemoryLayout<T>.size))
- var nulls = ArrowBuffer.createBuffer(length / 8 + 1, size: UInt(MemoryLayout<UInt8>.size))
- ArrowBuffer.copyCurrent(self.values, to: &values, len: values.capacity)
- ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: nulls.capacity)
- return [nulls, values]
- }
-
- fileprivate static func defaultValueForType() throws -> T {
- let type = T.self
- if type == Int8.self {
- return Int8(0) as! T // swiftlint:disable:this force_cast
- } else if type == Int16.self {
- return Int16(0) as! T // swiftlint:disable:this force_cast
- } else if type == Int32.self {
- return Int32(0) as! T // swiftlint:disable:this force_cast
- } else if type == Int64.self {
- return Int64(0) as! T // swiftlint:disable:this force_cast
- } else if type == UInt8.self {
- return UInt8(0) as! T // swiftlint:disable:this force_cast
- } else if type == UInt16.self {
- return UInt16(0) as! T // swiftlint:disable:this force_cast
- } else if type == UInt32.self {
- return UInt32(0) as! T // swiftlint:disable:this force_cast
- } else if type == UInt64.self {
- return UInt64(0) as! T // swiftlint:disable:this force_cast
- } else if type == Float.self {
- return Float(0) as! T // swiftlint:disable:this force_cast
- } else if type == Double.self {
- return Double(0) as! T // swiftlint:disable:this force_cast
- }
-
- throw ArrowError.unknownType("Unable to determine default value")
- }
-}
-
-public class BoolBufferBuilder: ValuesBufferBuilder<Bool>, ArrowBufferBuilder {
- public typealias ItemType = Bool
- public required init() throws {
- let values = ArrowBuffer.createBuffer(0, size: UInt(MemoryLayout<UInt8>.stride))
- let nulls = ArrowBuffer.createBuffer(0, size: UInt(MemoryLayout<UInt8>.stride))
- super.init(values: values, nulls: nulls)
- }
-
- public func append(_ newValue: ItemType?) {
- let index = UInt(self.length)
- self.length += 1
- if (length / 8) > self.values.length {
- self.resize(length)
- }
-
- if newValue != nil {
- BitUtility.setBit(index + self.offset, buffer: self.nulls)
- if newValue == true {
- BitUtility.setBit(index + self.offset, buffer: self.values)
- } else {
- BitUtility.clearBit(index + self.offset, buffer: self.values)
- }
-
- } else {
- self.nullCount += 1
- BitUtility.clearBit(index + self.offset, buffer: self.nulls)
- BitUtility.clearBit(index + self.offset, buffer: self.values)
- }
- }
-
- public func resize(_ length: UInt) {
- if (length / 8) > self.values.length {
- let resizeLength = resizeLength(self.values)
- var values = ArrowBuffer.createBuffer(resizeLength, size: UInt(MemoryLayout<UInt8>.size))
- var nulls = ArrowBuffer.createBuffer(resizeLength, size: UInt(MemoryLayout<UInt8>.size))
- ArrowBuffer.copyCurrent(self.values, to: &values, len: self.values.capacity)
- ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: self.nulls.capacity)
- self.values = values
- self.nulls = nulls
- }
- }
-
- public func finish() -> [ArrowBuffer] {
- let length = self.length
- var values = ArrowBuffer.createBuffer(length, size: UInt(MemoryLayout<UInt8>.size))
- var nulls = ArrowBuffer.createBuffer(length, size: UInt(MemoryLayout<UInt8>.size))
- ArrowBuffer.copyCurrent(self.values, to: &values, len: values.capacity)
- ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: nulls.capacity)
- return [nulls, values]
- }
-}
-
-public class VariableBufferBuilder<T>: ValuesBufferBuilder<T>, ArrowBufferBuilder {
- public typealias ItemType = T
- var offsets: ArrowBuffer
- let binaryStride = MemoryLayout<UInt8>.stride
- public required init() throws {
- let values = ArrowBuffer.createBuffer(0, size: UInt(binaryStride))
- let nulls = ArrowBuffer.createBuffer(0, size: UInt(binaryStride))
- self.offsets = ArrowBuffer.createBuffer(0, size: UInt(MemoryLayout<Int32>.stride))
- super.init(values: values, nulls: nulls, stride: binaryStride)
- }
-
- public func append(_ newValue: ItemType?) {
- let index = UInt(self.length)
- self.length += 1
- let offsetIndex = MemoryLayout<Int32>.stride * Int(index)
- if self.length >= self.offsets.length {
- self.resize(UInt(self.offsets.length + 1))
- }
- var binData: Data
- var isNull = false
- if let val = newValue {
- binData = getBytesFor(val)!
- } else {
- var nullVal = 0
- isNull = true
- binData = Data(bytes: &nullVal, count: MemoryLayout<UInt32>.size)
- }
-
- var currentIndex: Int32 = 0
- var currentOffset: Int32 = Int32(binData.count)
- if index > 0 {
- currentIndex = self.offsets.rawPointer.advanced(by: offsetIndex).load(as: Int32.self)
- currentOffset += currentIndex
- if currentOffset > self.values.length {
- self.value_resize(UInt(currentOffset))
- }
- }
-
- if isNull {
- self.nullCount += 1
- BitUtility.clearBit(index + self.offset, buffer: self.nulls)
- } else {
- BitUtility.setBit(index + self.offset, buffer: self.nulls)
- }
-
- binData.withUnsafeBytes { bufferPointer in
- let rawPointer = bufferPointer.baseAddress!
- self.values.rawPointer.advanced(by: Int(currentIndex))
- .copyMemory(from: rawPointer, byteCount: binData.count)
- }
-
- self.offsets.rawPointer.advanced(by: (offsetIndex + MemoryLayout<Int32>.stride))
- .storeBytes(of: currentOffset, as: Int32.self)
- }
-
- public func value_resize(_ length: UInt) {
- if length > self.values.length {
- let resizeLength = resizeLength(self.values, len: length)
- var values = ArrowBuffer.createBuffer(resizeLength, size: UInt(MemoryLayout<UInt8>.size))
- ArrowBuffer.copyCurrent(self.values, to: &values, len: self.values.capacity)
- self.values = values
- }
- }
-
- public func resize(_ length: UInt) {
- if length > self.offsets.length {
- let resizeLength = resizeLength(self.offsets, len: length)
- var nulls = ArrowBuffer.createBuffer(
- resizeLength / 8 + 1, size: UInt(MemoryLayout<UInt8>.size))
- var offsets = ArrowBuffer.createBuffer(resizeLength, size: UInt(MemoryLayout<Int32>.size))
- ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: self.nulls.capacity)
- ArrowBuffer.copyCurrent(self.offsets, to: &offsets, len: self.offsets.capacity)
- self.nulls = nulls
- self.offsets = offsets
- }
- }
-
- public func finish() -> [ArrowBuffer] {
- let length = self.length
- var values = ArrowBuffer.createBuffer(self.values.length, size: UInt(MemoryLayout<UInt8>.size))
- var nulls = ArrowBuffer.createBuffer(length / 8 + 1, size: UInt(MemoryLayout<UInt8>.size))
- var offsets = ArrowBuffer.createBuffer(length, size: UInt(MemoryLayout<Int32>.size))
- ArrowBuffer.copyCurrent(self.values, to: &values, len: values.capacity)
- ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: nulls.capacity)
- ArrowBuffer.copyCurrent(self.offsets, to: &offsets, len: offsets.capacity)
- return [nulls, offsets, values]
- }
-}
-
-public class AbstractWrapperBufferBuilder<T, U>: ArrowBufferBuilder {
- public typealias ItemType = T
- public var capacity: UInt { return self.bufferBuilder.capacity }
- public var length: UInt { return self.bufferBuilder.length }
- public var nullCount: UInt { return self.bufferBuilder.nullCount }
- public var offset: UInt { return self.bufferBuilder.offset }
- let bufferBuilder: FixedBufferBuilder<U>
- public required init() throws {
- self.bufferBuilder = try FixedBufferBuilder()
- }
-
- public func append(_ newValue: ItemType?) {
- fatalError("Method is not implemented")
- }
-
- public func isNull(_ index: UInt) -> Bool {
- return self.bufferBuilder.isNull(index)
- }
-
- public func resize(_ length: UInt) {
- self.bufferBuilder.resize(length)
- }
-
- public func finish() -> [ArrowBuffer] {
- return self.bufferBuilder.finish()
- }
-}
-
-public class Date32BufferBuilder: AbstractWrapperBufferBuilder<Date, Int32> {
- public override func append(_ newValue: ItemType?) {
- if let val = newValue {
- let daysSinceEpoch = Int32(val.timeIntervalSince1970 / 86400)
- self.bufferBuilder.append(daysSinceEpoch)
- } else {
- self.bufferBuilder.append(nil)
- }
- }
-}
-
-public class Date64BufferBuilder: AbstractWrapperBufferBuilder<Date, Int64> {
- public override func append(_ newValue: ItemType?) {
- if let val = newValue {
- let daysSinceEpoch = Int64(val.timeIntervalSince1970 * 1000)
- self.bufferBuilder.append(daysSinceEpoch)
- } else {
- self.bufferBuilder.append(nil)
- }
- }
-}
-
-public final class StructBufferBuilder: BaseBufferBuilder, ArrowBufferBuilder {
- public typealias ItemType = [Any?]
- var info: ArrowNestedType?
- public init() throws {
- let nulls = ArrowBuffer.createBuffer(0, size: UInt(MemoryLayout<UInt8>.stride))
- super.init(nulls)
- }
-
- public func initializeTypeInfo(_ fields: [ArrowField]) {
- info = ArrowNestedType(ArrowType.ArrowStruct, fields: fields)
- }
-
- public func append(_ newValue: [Any?]?) {
- let index = UInt(self.length)
- self.length += 1
- if length > self.nulls.length {
- self.resize(length)
- }
-
- if newValue != nil {
- BitUtility.setBit(index + self.offset, buffer: self.nulls)
- } else {
- self.nullCount += 1
- BitUtility.clearBit(index + self.offset, buffer: self.nulls)
- }
- }
-
- public func resize(_ length: UInt) {
- if length > self.nulls.length {
- let resizeLength = resizeLength(self.nulls)
- var nulls = ArrowBuffer.createBuffer(
- resizeLength / 8 + 1, size: UInt(MemoryLayout<UInt8>.size))
- ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: self.nulls.capacity)
- self.nulls = nulls
- }
- }
-
- public func finish() -> [ArrowBuffer] {
- let length = self.length
- var nulls = ArrowBuffer.createBuffer(length / 8 + 1, size: UInt(MemoryLayout<UInt8>.size))
- ArrowBuffer.copyCurrent(self.nulls, to: &nulls, len: nulls.capacity)
- return [nulls]
- }
-}
diff --git a/Sources/SparkConnect/ArrowData.swift b/Sources/SparkConnect/ArrowData.swift
deleted file mode 100644
index cb55875..0000000
--- a/Sources/SparkConnect/ArrowData.swift
+++ /dev/null
@@ -1,72 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import Foundation
-
-/// @nodoc
-public class ArrowData {
- public let type: ArrowType
- public let buffers: [ArrowBuffer]
- public let children: [ArrowData]
- public let nullCount: UInt
- public let length: UInt
- public let stride: Int
-
- convenience init(_ arrowType: ArrowType, buffers: [ArrowBuffer], nullCount: UInt) throws {
- try self.init(
- arrowType, buffers: buffers,
- children: [ArrowData](), nullCount: nullCount,
- length: buffers[1].length)
- }
-
- init(
- _ arrowType: ArrowType, buffers: [ArrowBuffer], children: [ArrowData], nullCount: UInt,
- length: UInt
- ) throws {
- let infoType = arrowType.info
- switch infoType {
- case let .primitiveInfo(typeId):
- if typeId == ArrowTypeId.unknown {
- throw ArrowError.unknownType("Unknown primitive type for data")
- }
- case let .variableInfo(typeId):
- if typeId == ArrowTypeId.unknown {
- throw ArrowError.unknownType("Unknown variable type for data")
- }
- case let .timeInfo(typeId):
- if typeId == ArrowTypeId.unknown {
- throw ArrowError.unknownType("Unknown time type for data")
- }
- case let .complexInfo(typeId):
- if typeId == ArrowTypeId.unknown {
- throw ArrowError.unknownType("Unknown complex type for data")
- }
- }
-
- self.type = arrowType
- self.buffers = buffers
- self.children = children
- self.nullCount = nullCount
- self.length = length
- self.stride = arrowType.getStride()
- }
-
- public func isNull(_ at: UInt) -> Bool {
- let nullBuffer = buffers[0]
- return nullBuffer.length > 0 && !BitUtility.isSet(at, buffer: nullBuffer)
- }
-}
diff --git a/Sources/SparkConnect/ArrowDecoder.swift b/Sources/SparkConnect/ArrowDecoder.swift
deleted file mode 100644
index 1f12b8b..0000000
--- a/Sources/SparkConnect/ArrowDecoder.swift
+++ /dev/null
@@ -1,373 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import Foundation
-
-/// @nodoc
-public class ArrowDecoder: Decoder {
- var rbIndex: UInt = 0
- var singleRBCol: Int = 0
- public var codingPath: [CodingKey] = []
- public var userInfo: [CodingUserInfoKey: Any] = [:]
- public let rb: RecordBatch
- public let nameToCol: [String: ArrowArrayHolder]
- public let columns: [ArrowArrayHolder]
- public init(_ decoder: ArrowDecoder) {
- self.userInfo = decoder.userInfo
- self.codingPath = decoder.codingPath
- self.rb = decoder.rb
- self.columns = decoder.columns
- self.nameToCol = decoder.nameToCol
- self.rbIndex = decoder.rbIndex
- }
-
- public init(_ rb: RecordBatch) {
- self.rb = rb
- var colMapping = [String: ArrowArrayHolder]()
- var columns = [ArrowArrayHolder]()
- for index in 0..<self.rb.schema.fields.count {
- let field = self.rb.schema.fields[index]
- columns.append(self.rb.column(index))
- colMapping[field.name] = self.rb.column(index)
- }
-
- self.columns = columns
- self.nameToCol = colMapping
- }
-
- public func decode<T: Decodable, U: Decodable>(_ type: [T: U].Type) throws -> [T: U] {
- var output = [T: U]()
- if rb.columnCount != 2 {
- throw ArrowError.invalid("RecordBatch column count of 2 is required to decode to map")
- }
-
- for index in 0..<rb.length {
- self.rbIndex = index
- self.singleRBCol = 0
- let key = try T.init(from: self)
- self.singleRBCol = 1
- let value = try U.init(from: self)
- output[key] = value
- }
-
- self.singleRBCol = 0
- return output
- }
-
- public func decode<T: Decodable>(_ type: T.Type) throws -> [T] {
- var output = [T]()
- for index in 0..<rb.length {
- self.rbIndex = index
- output.append(try type.init(from: self))
- }
-
- return output
- }
-
- public func container<Key>(
- keyedBy type: Key.Type
- ) -> KeyedDecodingContainer<Key> where Key: CodingKey {
- let container = ArrowKeyedDecoding<Key>(self, codingPath: codingPath)
- return KeyedDecodingContainer(container)
- }
-
- public func unkeyedContainer() -> UnkeyedDecodingContainer {
- return ArrowUnkeyedDecoding(self, codingPath: codingPath)
- }
-
- public func singleValueContainer() -> SingleValueDecodingContainer {
- return ArrowSingleValueDecoding(self, codingPath: codingPath)
- }
-
- func getCol(_ name: String) throws -> AnyArray {
- guard let col = self.nameToCol[name] else {
- throw ArrowError.invalid("Column for key \"\(name)\" not found")
- }
-
- return col.array
- }
-
- func getCol(_ index: Int) throws -> AnyArray {
- if index >= self.columns.count {
- throw ArrowError.outOfBounds(index: Int64(index))
- }
-
- return self.columns[index].array
- }
-
- func doDecode<T>(_ key: CodingKey) throws -> T? {
- let array: AnyArray = try self.getCol(key.stringValue)
- return array.asAny(self.rbIndex) as? T
- }
-
- func doDecode<T>(_ col: Int) throws -> T? {
- let array: AnyArray = try self.getCol(col)
- return array.asAny(self.rbIndex) as? T
- }
-
- func isNull(_ key: CodingKey) throws -> Bool {
- let array: AnyArray = try self.getCol(key.stringValue)
- return array.asAny(self.rbIndex) == nil
- }
-
- func isNull(_ col: Int) throws -> Bool {
- let array: AnyArray = try self.getCol(col)
- return array.asAny(self.rbIndex) == nil
- }
-}
-
-private struct ArrowUnkeyedDecoding: UnkeyedDecodingContainer {
- var codingPath: [CodingKey]
- var count: Int? = 0
- var isAtEnd: Bool = false
- var currentIndex: Int = 0
- let decoder: ArrowDecoder
-
- init(_ decoder: ArrowDecoder, codingPath: [CodingKey]) {
- self.decoder = decoder
- self.codingPath = codingPath
- self.count = self.decoder.columns.count
- }
-
- mutating func increment() {
- self.currentIndex += 1
- self.isAtEnd = self.currentIndex >= self.count!
- }
-
- mutating func decodeNil() throws -> Bool {
- defer { increment() }
- return try self.decoder.isNull(self.currentIndex)
- }
-
- mutating func decode<T>(_ type: T.Type) throws -> T where T: Decodable {
- if type == Int8?.self || type == Int16?.self || type == Int32?.self || type == Int64?.self
- || type == UInt8?.self || type == UInt16?.self || type == UInt32?.self || type == UInt64?.self
- || type == String?.self || type == Double?.self || type == Float?.self || type == Date?.self
- || type == Bool?.self || type == Bool.self || type == Int8.self || type == Int16.self
- || type == Int32.self || type == Int64.self || type == UInt8.self || type == UInt16.self
- || type == UInt32.self || type == UInt64.self || type == String.self || type == Double.self
- || type == Float.self || type == Date.self
- {
- defer { increment() }
- return try self.decoder.doDecode(self.currentIndex)!
- } else {
- throw ArrowError.invalid("Type \(type) is currently not supported")
- }
- }
-
- func nestedContainer<NestedKey>(
- keyedBy type: NestedKey.Type
- ) throws -> KeyedDecodingContainer<NestedKey> where NestedKey: CodingKey {
- throw ArrowError.invalid("Nested decoding is currently not supported.")
- }
-
- func nestedUnkeyedContainer() throws -> UnkeyedDecodingContainer {
- throw ArrowError.invalid("Nested decoding is currently not supported.")
- }
-
- func superDecoder() throws -> Decoder {
- throw ArrowError.invalid("super decoding is currently not supported.")
- }
-}
-
-private struct ArrowKeyedDecoding<Key: CodingKey>: KeyedDecodingContainerProtocol {
- var codingPath = [CodingKey]()
- var allKeys = [Key]()
- let decoder: ArrowDecoder
-
- init(_ decoder: ArrowDecoder, codingPath: [CodingKey]) {
- self.decoder = decoder
- self.codingPath = codingPath
- }
-
- func contains(_ key: Key) -> Bool {
- return self.decoder.nameToCol.keys.contains(key.stringValue)
- }
-
- func decodeNil(forKey key: Key) throws -> Bool {
- try self.decoder.isNull(key)
- }
-
- func decode(_ type: Bool.Type, forKey key: Key) throws -> Bool {
- return try self.decoder.doDecode(key)!
- }
-
- func decode(_ type: String.Type, forKey key: Key) throws -> String {
- return try self.decoder.doDecode(key)!
- }
-
- func decode(_ type: Double.Type, forKey key: Key) throws -> Double {
- return try self.decoder.doDecode(key)!
- }
-
- func decode(_ type: Float.Type, forKey key: Key) throws -> Float {
- return try self.decoder.doDecode(key)!
- }
-
- func decode(_ type: Int.Type, forKey key: Key) throws -> Int {
- throw ArrowError.invalid(
- "Int type is not supported (please use Int8, Int16, Int32 or Int64)")
- }
-
- func decode(_ type: Int8.Type, forKey key: Key) throws -> Int8 {
- return try self.decoder.doDecode(key)!
- }
-
- func decode(_ type: Int16.Type, forKey key: Key) throws -> Int16 {
- return try self.decoder.doDecode(key)!
- }
-
- func decode(_ type: Int32.Type, forKey key: Key) throws -> Int32 {
- return try self.decoder.doDecode(key)!
- }
-
- func decode(_ type: Int64.Type, forKey key: Key) throws -> Int64 {
- return try self.decoder.doDecode(key)!
- }
-
- func decode(_ type: UInt.Type, forKey key: Key) throws -> UInt {
- throw ArrowError.invalid(
- "UInt type is not supported (please use UInt8, UInt16, UInt32 or UInt64)")
- }
-
- func decode(_ type: UInt8.Type, forKey key: Key) throws -> UInt8 {
- return try self.decoder.doDecode(key)!
- }
-
- func decode(_ type: UInt16.Type, forKey key: Key) throws -> UInt16 {
- return try self.decoder.doDecode(key)!
- }
-
- func decode(_ type: UInt32.Type, forKey key: Key) throws -> UInt32 {
- return try self.decoder.doDecode(key)!
- }
-
- func decode(_ type: UInt64.Type, forKey key: Key) throws -> UInt64 {
- return try self.decoder.doDecode(key)!
- }
-
- func decode<T>(_ type: T.Type, forKey key: Key) throws -> T where T: Decodable {
- if ArrowArrayBuilders.isValidBuilderType(type) || type == Date.self {
- return try self.decoder.doDecode(key)!
- } else {
- throw ArrowError.invalid("Type \(type) is currently not supported")
- }
- }
-
- func nestedContainer<NestedKey>(
- keyedBy type: NestedKey.Type,
- forKey key: Key
- ) throws -> KeyedDecodingContainer<NestedKey> where NestedKey: CodingKey {
- throw ArrowError.invalid("Nested decoding is currently not supported.")
- }
-
- func nestedUnkeyedContainer(forKey key: Key) throws -> UnkeyedDecodingContainer {
- throw ArrowError.invalid("Nested decoding is currently not supported.")
- }
-
- func superDecoder() throws -> Decoder {
- throw ArrowError.invalid("super decoding is currently not supported.")
- }
-
- func superDecoder(forKey key: Key) throws -> Decoder {
- throw ArrowError.invalid("super decoding is currently not supported.")
- }
-}
-
-private struct ArrowSingleValueDecoding: SingleValueDecodingContainer {
- var codingPath = [CodingKey]()
- let decoder: ArrowDecoder
-
- init(_ decoder: ArrowDecoder, codingPath: [CodingKey]) {
- self.decoder = decoder
- self.codingPath = codingPath
- }
-
- func decodeNil() -> Bool {
- do {
- return try self.decoder.isNull(self.decoder.singleRBCol)
- } catch {
- return false
- }
- }
-
- func decode(_ type: Bool.Type) throws -> Bool {
- return try self.decoder.doDecode(self.decoder.singleRBCol)!
- }
-
- func decode(_ type: String.Type) throws -> String {
- return try self.decoder.doDecode(self.decoder.singleRBCol)!
- }
-
- func decode(_ type: Double.Type) throws -> Double {
- return try self.decoder.doDecode(self.decoder.singleRBCol)!
- }
-
- func decode(_ type: Float.Type) throws -> Float {
- return try self.decoder.doDecode(self.decoder.singleRBCol)!
- }
-
- func decode(_ type: Int.Type) throws -> Int {
- throw ArrowError.invalid(
- "Int type is not supported (please use Int8, Int16, Int32 or Int64)")
- }
-
- func decode(_ type: Int8.Type) throws -> Int8 {
- return try self.decoder.doDecode(self.decoder.singleRBCol)!
- }
-
- func decode(_ type: Int16.Type) throws -> Int16 {
- return try self.decoder.doDecode(self.decoder.singleRBCol)!
- }
-
- func decode(_ type: Int32.Type) throws -> Int32 {
- return try self.decoder.doDecode(self.decoder.singleRBCol)!
- }
-
- func decode(_ type: Int64.Type) throws -> Int64 {
- return try self.decoder.doDecode(self.decoder.singleRBCol)!
- }
-
- func decode(_ type: UInt.Type) throws -> UInt {
- throw ArrowError.invalid(
- "UInt type is not supported (please use UInt8, UInt16, UInt32 or UInt64)")
- }
-
- func decode(_ type: UInt8.Type) throws -> UInt8 {
- return try self.decoder.doDecode(self.decoder.singleRBCol)!
- }
-
- func decode(_ type: UInt16.Type) throws -> UInt16 {
- return try self.decoder.doDecode(self.decoder.singleRBCol)!
- }
-
- func decode(_ type: UInt32.Type) throws -> UInt32 {
- return try self.decoder.doDecode(self.decoder.singleRBCol)!
- }
-
- func decode(_ type: UInt64.Type) throws -> UInt64 {
- return try self.decoder.doDecode(self.decoder.singleRBCol)!
- }
-
- func decode<T>(_ type: T.Type) throws -> T where T: Decodable {
- if ArrowArrayBuilders.isValidBuilderType(type) || type == Date.self {
- return try self.decoder.doDecode(self.decoder.singleRBCol)!
- } else {
- throw ArrowError.invalid("Type \(type) is currently not supported")
- }
- }
-}
diff --git a/Sources/SparkConnect/ArrowEncoder.swift b/Sources/SparkConnect/ArrowEncoder.swift
deleted file mode 100644
index 6ad1b19..0000000
--- a/Sources/SparkConnect/ArrowEncoder.swift
+++ /dev/null
@@ -1,460 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import Foundation
-
-/// @nodoc
-public class ArrowEncoder: Encoder {
- public private(set) var builders = [String: ArrowArrayHolderBuilder]()
- private var byIndex = [String]()
- public var codingPath: [CodingKey] = []
- public var userInfo: [CodingUserInfoKey: Any] = [:]
- var errorMsg: String?
- // this is used for Dictionary types. A dictionary type
- // will give each key and value there own index so instead
- // of having a 2 column RecordBatch you would have
- // 2 * length(dictionary) column RecordBatch. Which would not
- // be the expected output.
- var modForIndex: Int?
-
- public init() {}
-
- public init(_ builders: [String: ArrowArrayHolderBuilder], byIndex: [String]) {
- self.builders = builders
- self.byIndex = byIndex
- }
-
- public static func encode<T: Encodable>(_ data: T) throws -> RecordBatch {
- let encoder = try loadEncoder(data)
- try data.encode(to: encoder)
- return try encoder.finish()
- }
-
- public static func encode<T: Encodable>(_ rows: [T]) throws -> RecordBatch? {
- if rows.isEmpty {
- return nil
- }
-
- let encoder = try loadEncoder(rows[0])
- for row in rows {
- try row.encode(to: encoder)
- }
-
- return try encoder.finish()
- }
-
- static func loadEncoder<T>(_ data: T) throws -> ArrowEncoder {
- // this will check if T is a simple built in type
- // (UInt, Int, Int8, String, Date, etc...).
- if ArrowArrayBuilders.isValidBuilderType(T.self) {
- let builders = ["col0": try ArrowArrayBuilders.loadBuilder(T.self)]
- return ArrowEncoder(builders, byIndex: ["col0"])
- } else {
- let encoder = ArrowEncoder()
- if data is [AnyHashable: Any] {
- encoder.modForIndex = 2
- }
-
- return encoder
- }
- }
-
- public func finish() throws -> RecordBatch {
- try throwIfInvalid()
- let batchBuilder = RecordBatch.Builder()
- for key in byIndex {
- batchBuilder.addColumn(key, arrowArray: try builders[key]!.toHolder())
- }
-
- switch batchBuilder.finish() {
- case .success(let rb):
- return rb
- case .failure(let error):
- throw error
- }
- }
-
- public func container<Key>(keyedBy type: Key.Type) -> KeyedEncodingContainer<Key>
- where Key: CodingKey {
- var container = ArrowKeyedEncoding<Key>(self)
- container.codingPath = codingPath
- return KeyedEncodingContainer(container)
- }
-
- public func unkeyedContainer() -> UnkeyedEncodingContainer {
- return ArrowUnkeyedEncoding(self, codingPath: self.codingPath)
- }
-
- public func singleValueContainer() -> SingleValueEncodingContainer {
- return ArrowSingleValueEncoding(self, codingPath: codingPath)
- }
-
- func doEncodeNil(key: CodingKey) throws {
- try throwIfInvalid()
- guard let builder = builders[key.stringValue] else {
- throw ArrowError.invalid("Column not found for key: \(key)")
- }
-
- builder.appendAny(nil)
- }
-
- // This is required by the keyed and unkeyed encoders as columns are
- // added when the first row of the data is encoded. This is done due
- // to limitations in the Swifts Mirror API (ex: it is unable to correctly
- // find the type for String? in [Int: String?])
- @discardableResult
- func ensureColumnExists<T>(_ value: T, key: String) throws -> ArrowArrayHolderBuilder {
- try throwIfInvalid()
- var builder = builders[key]
- if builder == nil {
- builder = try ArrowArrayBuilders.loadBuilder(T.self)
- builders[key] = builder
- byIndex.append(key)
- }
-
- return builder!
- }
-
- func getIndex(_ index: Int) -> Int {
- return self.modForIndex == nil ? index : index % self.modForIndex!
- }
-
- func doEncodeNil(_ keyIndex: Int) throws {
- try throwIfInvalid()
- let index = self.getIndex(keyIndex)
- if index >= builders.count {
- throw ArrowError.outOfBounds(index: Int64(index))
- }
-
- builders[byIndex[index]]!.appendAny(nil)
- }
-
- func doEncode<T>(_ value: T, key: CodingKey) throws {
- try throwIfInvalid()
- let builder = try ensureColumnExists(value, key: key.stringValue)
- builder.appendAny(value)
- }
-
- func doEncode<T>(_ value: T, keyIndex: Int) throws {
- try throwIfInvalid()
- let index = self.getIndex(keyIndex)
- if index >= builders.count {
- if index == builders.count {
- try ensureColumnExists(value, key: "col\(index)")
- } else {
- throw ArrowError.outOfBounds(index: Int64(index))
- }
- }
-
- builders[byIndex[index]]!.appendAny(value)
- }
-
- func throwIfInvalid() throws {
- if let errorMsg = self.errorMsg {
- throw ArrowError.invalid(errorMsg)
- }
- }
-}
-
-private struct ArrowKeyedEncoding<Key: CodingKey>: KeyedEncodingContainerProtocol {
- var codingPath: [CodingKey] = []
- let encoder: ArrowEncoder
- init(_ encoder: ArrowEncoder) {
- self.encoder = encoder
- }
-
- // If this method is called on row 0 and the encoder is
- // lazily bulding holders then this will produce an error
- // as this method does not know what the underlying type
- // is for the column. This method is not called for
- // nullable types (String?, Int32?, Date?) and the workaround
- // for this issue would be to predefine the builders for the
- // encoder. (I have only encoutered this issue when allowing
- // nullable types at the encode func level which is currently
- // not allowed)
- mutating func encodeNil(forKey key: Key) throws {
- try encoder.doEncodeNil(key: key)
- }
-
- mutating func doEncodeIf<T>(_ value: T?, forKey key: Key) throws {
- if value == nil {
- try encoder.ensureColumnExists(value, key: key.stringValue)
- try encoder.doEncodeNil(key: key)
- } else {
- try encoder.doEncode(value, key: key)
- }
- }
-
- mutating func encode(_ value: Bool, forKey key: Key) throws {
- try encoder.doEncode(value, key: key)
- }
-
- mutating func encodeIfPresent(_ value: Bool?, forKey key: Key) throws {
- try doEncodeIf(value, forKey: key)
- }
-
- mutating func encode(_ value: String, forKey key: Key) throws {
- try encoder.doEncode(value, key: key)
- }
-
- mutating func encodeIfPresent(_ value: String?, forKey key: Key) throws {
- try doEncodeIf(value, forKey: key)
- }
-
- mutating func encode(_ value: Double, forKey key: Key) throws {
- try encoder.doEncode(value, key: key)
- }
-
- mutating func encodeIfPresent(_ value: Double?, forKey key: Key) throws {
- try doEncodeIf(value, forKey: key)
- }
-
- mutating func encode(_ value: Float, forKey key: Key) throws {
- try encoder.doEncode(value, key: key)
- }
-
- mutating func encodeIfPresent(_ value: Float?, forKey key: Key) throws {
- try doEncodeIf(value, forKey: key)
- }
-
- mutating func encode(_ value: Int, forKey key: Key) throws {
- throw ArrowError.invalid(
- "Int type is not supported (please use Int8, Int16, Int32 or Int64)")
- }
-
- mutating func encodeIfPresent(_ value: Int?, forKey key: Key) throws {
- throw ArrowError.invalid(
- "Int type is not supported (please use Int8, Int16, Int32 or Int64)")
- }
-
- mutating func encode(_ value: Int8, forKey key: Key) throws {
- try encoder.doEncode(value, key: key)
- }
-
- mutating func encodeIfPresent(_ value: Int8?, forKey key: Key) throws {
- try doEncodeIf(value, forKey: key)
- }
-
- mutating func encode(_ value: Int16, forKey key: Key) throws {
- try encoder.doEncode(value, key: key)
- }
-
- mutating func encodeIfPresent(_ value: Int16?, forKey key: Key) throws {
- try doEncodeIf(value, forKey: key)
- }
-
- mutating func encode(_ value: Int32, forKey key: Key) throws {
- try encoder.doEncode(value, key: key)
- }
-
- mutating func encodeIfPresent(_ value: Int32?, forKey key: Key) throws {
- try doEncodeIf(value, forKey: key)
- }
-
- mutating func encode(_ value: Int64, forKey key: Key) throws {
- try encoder.doEncode(value, key: key)
- }
-
- mutating func encodeIfPresent(_ value: Int64?, forKey key: Key) throws {
- try doEncodeIf(value, forKey: key)
- }
-
- mutating func encode(_ value: UInt, forKey key: Key) throws {
- throw ArrowError.invalid(
- "UInt type is not supported (please use UInt8, UInt16, UInt32 or UInt64)")
- }
-
- mutating func encodeIfPresent(_ value: UInt?, forKey key: Key) throws {
- throw ArrowError.invalid(
- "UInt type is not supported (please use UInt8, UInt16, UInt32 or UInt64)")
- }
-
- mutating func encode(_ value: UInt8, forKey key: Key) throws {
- try encoder.doEncode(value, key: key)
- }
-
- mutating func encodeIfPresent(_ value: UInt8?, forKey key: Key) throws {
- try doEncodeIf(value, forKey: key)
- }
-
- mutating func encode(_ value: UInt16, forKey key: Key) throws {
- try encoder.doEncode(value, key: key)
- }
-
- mutating func encodeIfPresent(_ value: UInt16?, forKey key: Key) throws {
- try doEncodeIf(value, forKey: key)
- }
-
- mutating func encode(_ value: UInt32, forKey key: Key) throws {
- try encoder.doEncode(value, key: key)
- }
-
- mutating func encodeIfPresent(_ value: UInt32?, forKey key: Key) throws {
- try doEncodeIf(value, forKey: key)
- }
-
- mutating func encode(_ value: UInt64, forKey key: Key) throws {
- try encoder.doEncode(value, key: key)
- }
-
- mutating func encodeIfPresent(_ value: UInt64?, forKey key: Key) throws {
- try doEncodeIf(value, forKey: key)
- }
-
- mutating func encode<T: Encodable>(_ value: T, forKey key: Key) throws {
- if ArrowArrayBuilders.isValidBuilderType(T.self) {
- try encoder.doEncode(value, key: key)
- } else {
- throw ArrowError.invalid("Type \(T.self) is currently not supported")
- }
- }
-
- mutating func encodeIfPresent<T>(_ value: T?, forKey key: Self.Key) throws where T: Encodable {
- if ArrowArrayBuilders.isValidBuilderType(T?.self) {
- try doEncodeIf(value, forKey: key)
- } else {
- throw ArrowError.invalid("Type \(T.self) is currently not supported")
- }
- }
-
- // nested container is currently not allowed. This method doesn't throw
- // so setting an error mesg that will be throw by the encoder at the next
- // method call that throws
- mutating func nestedContainer<NestedKey: CodingKey>(
- keyedBy keyType: NestedKey.Type,
- forKey key: Key
- ) -> KeyedEncodingContainer<NestedKey> {
- self.encoder.errorMsg = "Nested decoding is currently not supported."
- var container = ArrowKeyedEncoding<NestedKey>(self.encoder)
- container.codingPath = codingPath
- return KeyedEncodingContainer(container)
- }
-
- // nested container is currently not allowed. This method doesn't throw
- // so setting an error mesg that will be throw by the encoder at the next
- // method call that throws
- mutating func nestedUnkeyedContainer(forKey key: Key) -> UnkeyedEncodingContainer {
- self.encoder.errorMsg = "Nested decoding is currently not supported."
- return ArrowUnkeyedEncoding(self.encoder, codingPath: self.codingPath)
- }
-
- // super encoding is currently not allowed. This method doesn't throw
- // so setting an error mesg that will be throw by the encoder at the next
- // method call that throws
- mutating func superEncoder() -> Encoder {
- self.encoder.errorMsg = "super encoding is currently not supported."
- return self.encoder
- }
-
- // super encoding is currently not allowed. This method doesn't throw
- // so setting an error mesg that will be throw by the encoder at the next
- // method call that throws
- mutating func superEncoder(forKey key: Key) -> Encoder {
- self.encoder.errorMsg = "super encoding is currently not supported."
- return self.encoder
- }
-}
-
-private struct ArrowUnkeyedEncoding: UnkeyedEncodingContainer {
- public private(set) var encoder: ArrowEncoder
- var codingPath: [CodingKey] = []
- var currentIndex: Int
- var count: Int = 0
-
- init(_ encoder: ArrowEncoder, codingPath: [CodingKey], currentIndex: Int = 0) {
- self.encoder = encoder
- self.currentIndex = currentIndex
- }
-
- mutating func increment() {
- self.currentIndex += 1
- }
-
- // If this method is called on row 0 and the encoder is
- // lazily bulding holders then this will produce an error
- // as this method does not know what the underlying type
- // is for the column. This method is not called for
- // nullable types (String?, Int32?, Date?) and the workaround
- // for this issue would be to predefine the builders for the
- // encoder. (I have only encoutered this issue when allowing
- // nullable types at the encode func level which is currently
- // not allowed)
- mutating func encodeNil() throws {
- try encoder.doEncodeNil(self.currentIndex)
- }
-
- mutating func encode<T>(_ value: T) throws where T: Encodable {
- let type = T.self
- if ArrowArrayBuilders.isValidBuilderType(type) {
- defer { increment() }
- return try self.encoder.doEncode(value, keyIndex: self.currentIndex)
- } else {
- throw ArrowError.invalid("Type \(type) is currently not supported")
- }
- }
-
- // nested container is currently not allowed. This method doesn't throw
- // so setting an error mesg that will be throw by the encoder at the next
- // method call that throws
- mutating func nestedContainer<NestedKey>(
- keyedBy keyType: NestedKey.Type
- ) -> KeyedEncodingContainer<NestedKey> where NestedKey: CodingKey {
- self.encoder.errorMsg = "Nested decoding is currently not supported."
- var container = ArrowKeyedEncoding<NestedKey>(self.encoder)
- container.codingPath = codingPath
- return KeyedEncodingContainer(container)
- }
-
- // nested container is currently not allowed. This method doesn't throw
- // so setting an error mesg that will be throw by the encoder at the next
- // method call that throws
- mutating func nestedUnkeyedContainer() -> UnkeyedEncodingContainer {
- self.encoder.errorMsg = "Nested decoding is currently not supported."
- return ArrowUnkeyedEncoding(self.encoder, codingPath: self.codingPath)
- }
-
- // super encoding is currently not allowed. This method doesn't throw
- // so setting an error mesg that will be throw by the encoder at the next
- // method call that throws
- mutating func superEncoder() -> Encoder {
- self.encoder.errorMsg = "super encoding is currently not supported."
- return self.encoder
- }
-}
-
-private struct ArrowSingleValueEncoding: SingleValueEncodingContainer {
- public private(set) var encoder: ArrowEncoder
- var codingPath: [CodingKey] = []
-
- public init(_ encoder: ArrowEncoder, codingPath: [CodingKey]) {
- self.encoder = encoder
- self.codingPath = codingPath
- }
-
- mutating func encodeNil() throws {
- return try self.encoder.doEncodeNil(0)
- }
-
- mutating func encode<T: Encodable>(_ value: T) throws {
- if ArrowArrayBuilders.isValidBuilderType(T.self) {
- return try self.encoder.doEncode(value, keyIndex: 0)
- } else {
- throw ArrowError.invalid("Type \(T.self) is currently not supported")
- }
- }
-}
-// swiftlint:disable:this file_length
diff --git a/Sources/SparkConnect/ArrowReader.swift b/Sources/SparkConnect/ArrowReader.swift
deleted file mode 100644
index de7af4f..0000000
--- a/Sources/SparkConnect/ArrowReader.swift
+++ /dev/null
@@ -1,372 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import FlatBuffers
-import Foundation
-
-let FILEMARKER = "ARROW1"
-let CONTINUATIONMARKER = -1
-
-/// @nodoc
-public class ArrowReader { // swiftlint:disable:this type_body_length
- private class RecordBatchData {
- let schema: org_apache_arrow_flatbuf_Schema
- let recordBatch: org_apache_arrow_flatbuf_RecordBatch
- private var fieldIndex: Int32 = 0
- private var nodeIndex: Int32 = 0
- private var bufferIndex: Int32 = 0
- init(
- _ recordBatch: org_apache_arrow_flatbuf_RecordBatch,
- schema: org_apache_arrow_flatbuf_Schema
- ) {
- self.recordBatch = recordBatch
- self.schema = schema
- }
-
- func nextNode() -> org_apache_arrow_flatbuf_FieldNode? {
- if nodeIndex >= self.recordBatch.nodesCount { return nil }
- defer { nodeIndex += 1 }
- return self.recordBatch.nodes(at: nodeIndex)
- }
-
- func nextBuffer() -> org_apache_arrow_flatbuf_Buffer? {
- if bufferIndex >= self.recordBatch.buffersCount { return nil }
- defer { bufferIndex += 1 }
- return self.recordBatch.buffers(at: bufferIndex)
- }
-
- func nextField() -> org_apache_arrow_flatbuf_Field? {
- if fieldIndex >= self.schema.fieldsCount { return nil }
- defer { fieldIndex += 1 }
- return self.schema.fields(at: fieldIndex)
- }
-
- func isDone() -> Bool {
- return nodeIndex >= self.recordBatch.nodesCount
- }
- }
-
- private struct DataLoadInfo {
- let fileData: Data
- let messageOffset: Int64
- var batchData: RecordBatchData
- }
-
- public class ArrowReaderResult {
- fileprivate var messageSchema: org_apache_arrow_flatbuf_Schema?
- public var schema: ArrowSchema?
- public var batches = [RecordBatch]()
- }
-
- public init() {}
-
- private func loadSchema(_ schema: org_apache_arrow_flatbuf_Schema) -> Result<
- ArrowSchema, ArrowError
- > {
- let builder = ArrowSchema.Builder()
- for index in 0..<schema.fieldsCount {
- let field = schema.fields(at: index)!
- let fieldType = findArrowType(field)
- if fieldType.info == ArrowType.ArrowUnknown {
- return .failure(.unknownType("Unsupported field type found: \(field.typeType)"))
- }
- let arrowField = ArrowField(field.name!, type: fieldType, isNullable: field.nullable)
- builder.addField(arrowField)
- }
-
- return .success(builder.finish())
- }
-
- private func loadStructData(
- _ loadInfo: DataLoadInfo,
- field: org_apache_arrow_flatbuf_Field
- )
- -> Result<ArrowArrayHolder, ArrowError>
- {
- guard let node = loadInfo.batchData.nextNode() else {
- return .failure(.invalid("Node not found"))
- }
-
- guard let nullBuffer = loadInfo.batchData.nextBuffer() else {
- return .failure(.invalid("Null buffer not found"))
- }
-
- let nullLength = UInt(ceil(Double(node.length) / 8))
- let arrowNullBuffer = makeBuffer(
- nullBuffer, fileData: loadInfo.fileData,
- length: nullLength, messageOffset: loadInfo.messageOffset)
- var children = [ArrowData]()
- for index in 0..<field.childrenCount {
- let childField = field.children(at: index)!
- switch loadField(loadInfo, field: childField) {
- case .success(let holder):
- children.append(holder.array.arrowData)
- case .failure(let error):
- return .failure(error)
- }
- }
-
- return makeArrayHolder(
- field, buffers: [arrowNullBuffer],
- nullCount: UInt(node.nullCount), children: children,
- rbLength: UInt(loadInfo.batchData.recordBatch.length))
- }
-
- private func loadPrimitiveData(
- _ loadInfo: DataLoadInfo,
- field: org_apache_arrow_flatbuf_Field
- )
- -> Result<ArrowArrayHolder, ArrowError>
- {
- guard let node = loadInfo.batchData.nextNode() else {
- return .failure(.invalid("Node not found"))
- }
-
- guard let nullBuffer = loadInfo.batchData.nextBuffer() else {
- return .failure(.invalid("Null buffer not found"))
- }
-
- guard let valueBuffer = loadInfo.batchData.nextBuffer() else {
- return .failure(.invalid("Value buffer not found"))
- }
-
- let nullLength = UInt(ceil(Double(node.length) / 8))
- let arrowNullBuffer = makeBuffer(
- nullBuffer, fileData: loadInfo.fileData,
- length: nullLength, messageOffset: loadInfo.messageOffset)
- let arrowValueBuffer = makeBuffer(
- valueBuffer, fileData: loadInfo.fileData,
- length: UInt(node.length), messageOffset: loadInfo.messageOffset)
- return makeArrayHolder(
- field, buffers: [arrowNullBuffer, arrowValueBuffer],
- nullCount: UInt(node.nullCount), children: nil,
- rbLength: UInt(loadInfo.batchData.recordBatch.length))
- }
-
- private func loadVariableData(
- _ loadInfo: DataLoadInfo,
- field: org_apache_arrow_flatbuf_Field
- )
- -> Result<ArrowArrayHolder, ArrowError>
- {
- guard let node = loadInfo.batchData.nextNode() else {
- return .failure(.invalid("Node not found"))
- }
-
- guard let nullBuffer = loadInfo.batchData.nextBuffer() else {
- return .failure(.invalid("Null buffer not found"))
- }
-
- guard let offsetBuffer = loadInfo.batchData.nextBuffer() else {
- return .failure(.invalid("Offset buffer not found"))
- }
-
- guard let valueBuffer = loadInfo.batchData.nextBuffer() else {
- return .failure(.invalid("Value buffer not found"))
- }
-
- let nullLength = UInt(ceil(Double(node.length) / 8))
- let arrowNullBuffer = makeBuffer(
- nullBuffer, fileData: loadInfo.fileData,
- length: nullLength, messageOffset: loadInfo.messageOffset)
- let arrowOffsetBuffer = makeBuffer(
- offsetBuffer, fileData: loadInfo.fileData,
- length: UInt(node.length), messageOffset: loadInfo.messageOffset)
- let arrowValueBuffer = makeBuffer(
- valueBuffer, fileData: loadInfo.fileData,
- length: UInt(node.length), messageOffset: loadInfo.messageOffset)
- return makeArrayHolder(
- field, buffers: [arrowNullBuffer, arrowOffsetBuffer, arrowValueBuffer],
- nullCount: UInt(node.nullCount), children: nil,
- rbLength: UInt(loadInfo.batchData.recordBatch.length))
- }
-
- private func loadField(
- _ loadInfo: DataLoadInfo,
- field: org_apache_arrow_flatbuf_Field
- )
- -> Result<ArrowArrayHolder, ArrowError>
- {
- if isNestedType(field.typeType) {
- return loadStructData(loadInfo, field: field)
- } else if isFixedPrimitive(field.typeType) {
- return loadPrimitiveData(loadInfo, field: field)
- } else {
- return loadVariableData(loadInfo, field: field)
- }
- }
-
- private func loadRecordBatch(
- _ recordBatch: org_apache_arrow_flatbuf_RecordBatch,
- schema: org_apache_arrow_flatbuf_Schema,
- arrowSchema: ArrowSchema,
- data: Data,
- messageEndOffset: Int64
- ) -> Result<RecordBatch, ArrowError> {
- var columns: [ArrowArrayHolder] = []
- let batchData = RecordBatchData(recordBatch, schema: schema)
- let loadInfo = DataLoadInfo(
- fileData: data,
- messageOffset: messageEndOffset,
- batchData: batchData)
- while !batchData.isDone() {
- guard let field = batchData.nextField() else {
- return .failure(.invalid("Field not found"))
- }
-
- let result = loadField(loadInfo, field: field)
- switch result {
- case .success(let holder):
- columns.append(holder)
- case .failure(let error):
- return .failure(error)
- }
- }
-
- return .success(RecordBatch(arrowSchema, columns: columns))
- }
-
- public func fromStream( // swiftlint:disable:this function_body_length
- _ fileData: Data,
- useUnalignedBuffers: Bool = false
- ) -> Result<ArrowReaderResult, ArrowError> {
- let footerLength = fileData.withUnsafeBytes { rawBuffer in
- rawBuffer.loadUnaligned(fromByteOffset: fileData.count - 4, as: Int32.self)
- }
-
- let result = ArrowReaderResult()
- let footerStartOffset = fileData.count - Int(footerLength + 4)
- let footerData = fileData[footerStartOffset...]
- let footerBuffer = ByteBuffer(
- data: footerData,
- allowReadingUnalignedBuffers: useUnalignedBuffers)
- let footer = org_apache_arrow_flatbuf_Footer.getRootAsFooter(bb: footerBuffer)
- let schemaResult = loadSchema(footer.schema!)
- switch schemaResult {
- case .success(let schema):
- result.schema = schema
- case .failure(let error):
- return .failure(error)
- }
-
- for index in 0..<footer.recordBatchesCount {
- let recordBatch = footer.recordBatches(at: index)!
- var messageLength = fileData.withUnsafeBytes { rawBuffer in
- rawBuffer.loadUnaligned(fromByteOffset: Int(recordBatch.offset), as: Int32.self)
- }
-
- var messageOffset: Int64 = 1
- if messageLength == CONTINUATIONMARKER {
- messageOffset += 1
- messageLength = fileData.withUnsafeBytes { rawBuffer in
- rawBuffer.loadUnaligned(
- fromByteOffset: Int(recordBatch.offset + Int64(MemoryLayout<Int32>.size)),
- as: Int32.self)
- }
- }
-
- let messageStartOffset =
- recordBatch.offset + (Int64(MemoryLayout<Int32>.size) * messageOffset)
- let messageEndOffset = messageStartOffset + Int64(messageLength)
- let recordBatchData = fileData[messageStartOffset..<messageEndOffset]
- let mbb = ByteBuffer(
- data: recordBatchData,
- allowReadingUnalignedBuffers: useUnalignedBuffers)
- let message = org_apache_arrow_flatbuf_Message.getRootAsMessage(bb: mbb)
- switch message.headerType {
- case .recordbatch:
- do {
- let rbMessage = message.header(type: org_apache_arrow_flatbuf_RecordBatch.self)!
- let recordBatch = try loadRecordBatch(
- rbMessage,
- schema: footer.schema!,
- arrowSchema: result.schema!,
- data: fileData,
- messageEndOffset: messageEndOffset
- ).get()
- result.batches.append(recordBatch)
- } catch let error {
- return .failure(error)
- }
- default:
- return .failure(.unknownError("Unhandled header type: \(message.headerType)"))
- }
- }
-
- return .success(result)
- }
-
- public func fromFile(_ fileURL: URL) -> Result<ArrowReaderResult, ArrowError> {
- do {
- let fileData = try Data(contentsOf: fileURL)
- if !validateFileData(fileData) {
- return .failure(.ioError("Not a valid arrow file."))
- }
-
- let markerLength = FILEMARKER.utf8.count
- let footerLengthEnd = Int(fileData.count - markerLength)
- let data = fileData[..<(footerLengthEnd)]
- return fromStream(data)
- } catch {
- return .failure(.unknownError("Error loading file: \(error)"))
- }
- }
-
- static public func makeArrowReaderResult() -> ArrowReaderResult {
- return ArrowReaderResult()
- }
-
- public func fromMessage(
- _ dataHeader: Data,
- dataBody: Data,
- result: ArrowReaderResult,
- useUnalignedBuffers: Bool = false
- ) -> Result<Void, ArrowError> {
- let mbb = ByteBuffer(
- data: dataHeader,
- allowReadingUnalignedBuffers: useUnalignedBuffers)
- let message = org_apache_arrow_flatbuf_Message.getRootAsMessage(bb: mbb)
- switch message.headerType {
- case .schema:
- let sMessage = message.header(type: org_apache_arrow_flatbuf_Schema.self)!
- switch loadSchema(sMessage) {
- case .success(let schema):
- result.schema = schema
- result.messageSchema = sMessage
- return .success(())
- case .failure(let error):
- return .failure(error)
- }
- case .recordbatch:
- let rbMessage = message.header(type: org_apache_arrow_flatbuf_RecordBatch.self)!
- do {
- let recordBatch = try loadRecordBatch(
- rbMessage, schema: result.messageSchema!, arrowSchema: result.schema!,
- data: dataBody, messageEndOffset: 0
- ).get()
- result.batches.append(recordBatch)
- return .success(())
- } catch let error {
- return .failure(error)
- }
-
- default:
- return .failure(.unknownError("Unhandled header type: \(message.headerType)"))
- }
- }
-
-}
diff --git a/Sources/SparkConnect/ArrowReaderHelper.swift b/Sources/SparkConnect/ArrowReaderHelper.swift
deleted file mode 100644
index c0bd55b..0000000
--- a/Sources/SparkConnect/ArrowReaderHelper.swift
+++ /dev/null
@@ -1,314 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import FlatBuffers
-import Foundation
-
-/// @nodoc
-private func makeBinaryHolder(
- _ buffers: [ArrowBuffer],
- nullCount: UInt
-) -> Result<ArrowArrayHolder, ArrowError> {
- do {
- let arrowType = ArrowType(ArrowType.ArrowBinary)
- let arrowData = try ArrowData(arrowType, buffers: buffers, nullCount: nullCount)
- return .success(ArrowArrayHolderImpl(try BinaryArray(arrowData)))
- } catch let error as ArrowError {
- return .failure(error)
- } catch {
- return .failure(.unknownError("\(error)"))
- }
-}
-
-private func makeStringHolder(
- _ buffers: [ArrowBuffer],
- nullCount: UInt
-) -> Result<ArrowArrayHolder, ArrowError> {
- do {
- let arrowType = ArrowType(ArrowType.ArrowString)
- let arrowData = try ArrowData(arrowType, buffers: buffers, nullCount: nullCount)
- return .success(ArrowArrayHolderImpl(try StringArray(arrowData)))
- } catch let error as ArrowError {
- return .failure(error)
- } catch {
- return .failure(.unknownError("\(error)"))
- }
-}
-
-private func makeDateHolder(
- _ field: ArrowField,
- buffers: [ArrowBuffer],
- nullCount: UInt
-) -> Result<ArrowArrayHolder, ArrowError> {
- do {
- if field.type.id == .date32 {
- let arrowData = try ArrowData(field.type, buffers: buffers, nullCount: nullCount)
- return .success(ArrowArrayHolderImpl(try Date32Array(arrowData)))
- }
-
- let arrowData = try ArrowData(field.type, buffers: buffers, nullCount: nullCount)
- return .success(ArrowArrayHolderImpl(try Date64Array(arrowData)))
- } catch let error as ArrowError {
- return .failure(error)
- } catch {
- return .failure(.unknownError("\(error)"))
- }
-}
-
-private func makeTimeHolder(
- _ field: ArrowField,
- buffers: [ArrowBuffer],
- nullCount: UInt
-) -> Result<ArrowArrayHolder, ArrowError> {
- do {
- if field.type.id == .time32 {
- if let arrowType = field.type as? ArrowTypeTime32 {
- let arrowData = try ArrowData(arrowType, buffers: buffers, nullCount: nullCount)
- return .success(ArrowArrayHolderImpl(try FixedArray<Time32>(arrowData)))
- } else {
- return .failure(.invalid("Incorrect field type for time: \(field.type)"))
- }
- }
-
- if let arrowType = field.type as? ArrowTypeTime64 {
- let arrowData = try ArrowData(arrowType, buffers: buffers, nullCount: nullCount)
- return .success(ArrowArrayHolderImpl(try FixedArray<Time64>(arrowData)))
- } else {
- return .failure(.invalid("Incorrect field type for time: \(field.type)"))
- }
- } catch let error as ArrowError {
- return .failure(error)
- } catch {
- return .failure(.unknownError("\(error)"))
- }
-}
-
-private func makeBoolHolder(
- _ buffers: [ArrowBuffer],
- nullCount: UInt
-) -> Result<ArrowArrayHolder, ArrowError> {
- do {
- let arrowType = ArrowType(ArrowType.ArrowBool)
- let arrowData = try ArrowData(arrowType, buffers: buffers, nullCount: nullCount)
- return .success(ArrowArrayHolderImpl(try BoolArray(arrowData)))
- } catch let error as ArrowError {
- return .failure(error)
- } catch {
- return .failure(.unknownError("\(error)"))
- }
-}
-
-private func makeFixedHolder<T>(
- _: T.Type, field: ArrowField, buffers: [ArrowBuffer],
- nullCount: UInt
-) -> Result<ArrowArrayHolder, ArrowError> {
- do {
- let arrowData = try ArrowData(field.type, buffers: buffers, nullCount: nullCount)
- return .success(ArrowArrayHolderImpl(try FixedArray<T>(arrowData)))
- } catch let error as ArrowError {
- return .failure(error)
- } catch {
- return .failure(.unknownError("\(error)"))
- }
-}
-
-func makeStructHolder(
- _ field: ArrowField,
- buffers: [ArrowBuffer],
- nullCount: UInt,
- children: [ArrowData],
- rbLength: UInt
-) -> Result<ArrowArrayHolder, ArrowError> {
- do {
- let arrowData = try ArrowData(
- field.type,
- buffers: buffers, children: children,
- nullCount: nullCount, length: rbLength)
- return .success(ArrowArrayHolderImpl(try StructArray(arrowData)))
- } catch let error as ArrowError {
- return .failure(error)
- } catch {
- return .failure(.unknownError("\(error)"))
- }
-}
-
-func makeArrayHolder(
- _ field: org_apache_arrow_flatbuf_Field,
- buffers: [ArrowBuffer],
- nullCount: UInt,
- children: [ArrowData]?,
- rbLength: UInt
-) -> Result<ArrowArrayHolder, ArrowError> {
- let arrowField = fromProto(field: field)
- return makeArrayHolder(
- arrowField, buffers: buffers, nullCount: nullCount, children: children, rbLength: rbLength)
-}
-
-func makeArrayHolder( // swiftlint:disable:this cyclomatic_complexity
- _ field: ArrowField,
- buffers: [ArrowBuffer],
- nullCount: UInt,
- children: [ArrowData]?,
- rbLength: UInt
-) -> Result<ArrowArrayHolder, ArrowError> {
- let typeId = field.type.id
- switch typeId {
- case .int8:
- return makeFixedHolder(Int8.self, field: field, buffers: buffers, nullCount: nullCount)
- case .uint8:
- return makeFixedHolder(UInt8.self, field: field, buffers: buffers, nullCount: nullCount)
- case .int16:
- return makeFixedHolder(Int16.self, field: field, buffers: buffers, nullCount: nullCount)
- case .uint16:
- return makeFixedHolder(UInt16.self, field: field, buffers: buffers, nullCount: nullCount)
- case .int32:
- return makeFixedHolder(Int32.self, field: field, buffers: buffers, nullCount: nullCount)
- case .uint32:
- return makeFixedHolder(UInt32.self, field: field, buffers: buffers, nullCount: nullCount)
- case .int64:
- return makeFixedHolder(Int64.self, field: field, buffers: buffers, nullCount: nullCount)
- case .uint64:
- return makeFixedHolder(UInt64.self, field: field, buffers: buffers, nullCount: nullCount)
- case .boolean:
- return makeBoolHolder(buffers, nullCount: nullCount)
- case .float:
- return makeFixedHolder(Float.self, field: field, buffers: buffers, nullCount: nullCount)
- case .double:
- return makeFixedHolder(Double.self, field: field, buffers: buffers, nullCount: nullCount)
- case .string:
- return makeStringHolder(buffers, nullCount: nullCount)
- case .binary:
- return makeBinaryHolder(buffers, nullCount: nullCount)
- case .date32, .date64:
- return makeDateHolder(field, buffers: buffers, nullCount: nullCount)
- case .time32, .time64:
- return makeTimeHolder(field, buffers: buffers, nullCount: nullCount)
- case .strct:
- return makeStructHolder(
- field, buffers: buffers, nullCount: nullCount, children: children!, rbLength: rbLength)
- default:
- return .failure(.unknownType("Type \(typeId) currently not supported"))
- }
-}
-
-func makeBuffer(
- _ buffer: org_apache_arrow_flatbuf_Buffer, fileData: Data,
- length: UInt, messageOffset: Int64
-) -> ArrowBuffer {
- let startOffset = messageOffset + buffer.offset
- let endOffset = startOffset + buffer.length
- let bufferData = [UInt8](fileData[startOffset..<endOffset])
- return ArrowBuffer.createBuffer(bufferData, length: length)
-}
-
-func isFixedPrimitive(_ type: org_apache_arrow_flatbuf_Type_) -> Bool {
- switch type {
- case .int, .bool, .floatingpoint, .date, .time:
- return true
- default:
- return false
- }
-}
-
-func isNestedType(_ type: org_apache_arrow_flatbuf_Type_) -> Bool {
- switch type {
- case .struct_:
- return true
- default:
- return false
- }
-}
-
-func findArrowType( // swiftlint:disable:this cyclomatic_complexity function_body_length
- _ field: org_apache_arrow_flatbuf_Field
-) -> ArrowType {
- let type = field.typeType
- switch type {
- case .int:
- let intType = field.type(type: org_apache_arrow_flatbuf_Int.self)!
- let bitWidth = intType.bitWidth
- if bitWidth == 8 {
- return ArrowType(intType.isSigned ? ArrowType.ArrowInt8 : ArrowType.ArrowUInt8)
- }
- if bitWidth == 16 {
- return ArrowType(intType.isSigned ? ArrowType.ArrowInt16 : ArrowType.ArrowUInt16)
- }
- if bitWidth == 32 {
- return ArrowType(intType.isSigned ? ArrowType.ArrowInt32 : ArrowType.ArrowUInt32)
- }
- if bitWidth == 64 {
- return ArrowType(intType.isSigned ? ArrowType.ArrowInt64 : ArrowType.ArrowUInt64)
- }
- return ArrowType(ArrowType.ArrowUnknown)
- case .bool:
- return ArrowType(ArrowType.ArrowBool)
- case .floatingpoint:
- let floatType = field.type(type: org_apache_arrow_flatbuf_FloatingPoint.self)!
- switch floatType.precision {
- case .single:
- return ArrowType(ArrowType.ArrowFloat)
- case .double:
- return ArrowType(ArrowType.ArrowDouble)
- default:
- return ArrowType(ArrowType.ArrowUnknown)
- }
- case .utf8:
- return ArrowType(ArrowType.ArrowString)
- case .binary:
- return ArrowType(ArrowType.ArrowBinary)
- case .date:
- let dateType = field.type(type: org_apache_arrow_flatbuf_Date.self)!
- if dateType.unit == .day {
- return ArrowType(ArrowType.ArrowDate32)
- }
-
- return ArrowType(ArrowType.ArrowDate64)
- case .time:
- let timeType = field.type(type: org_apache_arrow_flatbuf_Time.self)!
- if timeType.unit == .second || timeType.unit == .millisecond {
- return ArrowTypeTime32(timeType.unit == .second ? .seconds : .milliseconds)
- }
-
- return ArrowTypeTime64(timeType.unit == .microsecond ? .microseconds : .nanoseconds)
- case .struct_:
- _ = field.type(type: org_apache_arrow_flatbuf_Struct_.self)!
- var fields = [ArrowField]()
- for index in 0..<field.childrenCount {
- let childField = field.children(at: index)!
- let childType = findArrowType(childField)
- fields.append(
- ArrowField(childField.name ?? "", type: childType, isNullable: childField.nullable))
- }
-
- return ArrowNestedType(ArrowType.ArrowStruct, fields: fields)
- default:
- return ArrowType(ArrowType.ArrowUnknown)
- }
-}
-
-func validateBufferIndex(_ recordBatch: org_apache_arrow_flatbuf_RecordBatch, index: Int32) throws {
- if index >= recordBatch.buffersCount {
- throw ArrowError.outOfBounds(index: Int64(index))
- }
-}
-
-func validateFileData(_ data: Data) -> Bool {
- let markerLength = FILEMARKER.utf8.count
- let startString = String(decoding: data[..<markerLength], as: UTF8.self)
- let endString = String(decoding: data[(data.count - markerLength)...], as: UTF8.self)
- return startString == FILEMARKER && endString == FILEMARKER
-}
diff --git a/Sources/SparkConnect/ArrowSchema.swift b/Sources/SparkConnect/ArrowSchema.swift
deleted file mode 100644
index f5c2d1a..0000000
--- a/Sources/SparkConnect/ArrowSchema.swift
+++ /dev/null
@@ -1,76 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import Foundation
-
-/// @nodoc
-public class ArrowField {
- public let type: ArrowType
- public let name: String
- public let isNullable: Bool
-
- init(_ name: String, type: ArrowType, isNullable: Bool) {
- self.name = name
- self.type = type
- self.isNullable = isNullable
- }
-}
-
-/// @nodoc
-public class ArrowSchema {
- public let fields: [ArrowField]
- public let fieldLookup: [String: Int]
- init(_ fields: [ArrowField]) {
- var fieldLookup = [String: Int]()
- for (index, field) in fields.enumerated() {
- fieldLookup[field.name] = index
- }
-
- self.fields = fields
- self.fieldLookup = fieldLookup
- }
-
- public func field(_ index: Int) -> ArrowField {
- return self.fields[index]
- }
-
- public func fieldIndex(_ name: String) -> Int? {
- return self.fieldLookup[name]
- }
-
- public class Builder {
- private var fields: [ArrowField] = []
-
- public init() {}
-
- @discardableResult
- public func addField(_ field: ArrowField) -> Builder {
- fields.append(field)
- return self
- }
-
- @discardableResult
- public func addField(_ name: String, type: ArrowType, isNullable: Bool) -> Builder {
- fields.append(ArrowField(name, type: type, isNullable: isNullable))
- return self
- }
-
- public func finish() -> ArrowSchema {
- return ArrowSchema(fields)
- }
- }
-}
diff --git a/Sources/SparkConnect/ArrowTable.swift b/Sources/SparkConnect/ArrowTable.swift
deleted file mode 100644
index 716ff26..0000000
--- a/Sources/SparkConnect/ArrowTable.swift
+++ /dev/null
@@ -1,205 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import Foundation
-
-/// @nodoc
-public class ArrowColumn {
- public let field: ArrowField
- fileprivate let dataHolder: ChunkedArrayHolder
- public var type: ArrowType { return self.dataHolder.type }
- public var length: UInt { return self.dataHolder.length }
- public var nullCount: UInt { return self.dataHolder.nullCount }
-
- public func data<T>() -> ChunkedArray<T> {
- return (self.dataHolder.holder as! ChunkedArray<T>) // swiftlint:disable:this force_cast
- }
-
- public var name: String { return field.name }
- public init(_ field: ArrowField, chunked: ChunkedArrayHolder) {
- self.field = field
- self.dataHolder = chunked
- }
-}
-
-/// @nodoc
-public class ArrowTable {
- public let schema: ArrowSchema
- public var columnCount: UInt { return UInt(self.columns.count) }
- public let rowCount: UInt
- public let columns: [ArrowColumn]
- init(_ schema: ArrowSchema, columns: [ArrowColumn]) {
- self.schema = schema
- self.columns = columns
- self.rowCount = columns[0].length
- }
-
- public static func from(recordBatches: [RecordBatch]) -> Result<ArrowTable, ArrowError> {
- if recordBatches.isEmpty {
- return .failure(.arrayHasNoElements)
- }
-
- var holders = [[ArrowArrayHolder]]()
- let schema = recordBatches[0].schema
- for recordBatch in recordBatches {
- for index in 0..<schema.fields.count {
- if holders.count <= index {
- holders.append([ArrowArrayHolder]())
- }
- holders[index].append(recordBatch.columns[index])
- }
- }
-
- let builder = ArrowTable.Builder()
- for index in 0..<schema.fields.count {
- switch makeArrowColumn(schema.fields[index], holders: holders[index]) {
- case .success(let column):
- builder.addColumn(column)
- case .failure(let error):
- return .failure(error)
- }
- }
-
- return .success(builder.finish())
- }
-
- private static func makeArrowColumn(
- _ field: ArrowField,
- holders: [ArrowArrayHolder]
- ) -> Result<ArrowColumn, ArrowError> {
- do {
- return .success(try holders[0].getArrowColumn(field, holders))
- } catch {
- return .failure(.runtimeError("\(error)"))
- }
- }
-
- public class Builder {
- let schemaBuilder = ArrowSchema.Builder()
- var columns = [ArrowColumn]()
-
- public init() {}
-
- @discardableResult
- public func addColumn<T>(_ fieldName: String, arrowArray: ArrowArray<T>) throws -> Builder {
- return self.addColumn(fieldName, chunked: try ChunkedArray([arrowArray]))
- }
-
- @discardableResult
- public func addColumn<T>(_ fieldName: String, chunked: ChunkedArray<T>) -> Builder {
- let field = ArrowField(fieldName, type: chunked.type, isNullable: chunked.nullCount != 0)
- self.schemaBuilder.addField(field)
- self.columns.append(ArrowColumn(field, chunked: ChunkedArrayHolder(chunked)))
- return self
- }
-
- @discardableResult
- public func addColumn<T>(_ field: ArrowField, arrowArray: ArrowArray<T>) throws -> Builder {
- self.schemaBuilder.addField(field)
- let holder = ChunkedArrayHolder(try ChunkedArray([arrowArray]))
- self.columns.append(ArrowColumn(field, chunked: holder))
- return self
- }
-
- @discardableResult
- public func addColumn<T>(_ field: ArrowField, chunked: ChunkedArray<T>) -> Builder {
- self.schemaBuilder.addField(field)
- self.columns.append(ArrowColumn(field, chunked: ChunkedArrayHolder(chunked)))
- return self
- }
-
- @discardableResult
- public func addColumn(_ column: ArrowColumn) -> Builder {
- self.schemaBuilder.addField(column.field)
- self.columns.append(column)
- return self
- }
-
- public func finish() -> ArrowTable {
- return ArrowTable(self.schemaBuilder.finish(), columns: self.columns)
- }
- }
-}
-
-public class RecordBatch {
- public let schema: ArrowSchema
- public var columnCount: UInt { return UInt(self.columns.count) }
- public let columns: [ArrowArrayHolder]
- public let length: UInt
- public init(_ schema: ArrowSchema, columns: [ArrowArrayHolder]) {
- self.schema = schema
- self.columns = columns
- self.length = columns[0].length
- }
-
- public class Builder {
- let schemaBuilder = ArrowSchema.Builder()
- var columns = [ArrowArrayHolder]()
-
- public init() {}
-
- @discardableResult
- public func addColumn(_ fieldName: String, arrowArray: ArrowArrayHolder) -> Builder {
- let field = ArrowField(
- fieldName, type: arrowArray.type, isNullable: arrowArray.nullCount != 0)
- self.schemaBuilder.addField(field)
- self.columns.append(arrowArray)
- return self
- }
-
- @discardableResult
- public func addColumn(_ field: ArrowField, arrowArray: ArrowArrayHolder) -> Builder {
- self.schemaBuilder.addField(field)
- self.columns.append(arrowArray)
- return self
- }
-
- public func finish() -> Result<RecordBatch, ArrowError> {
- if columns.count > 0 {
- let columnLength = columns[0].length
- for column in columns {
- if column.length != columnLength { // swiftlint:disable:this for_where
- return .failure(.runtimeError("Columns have different sizes"))
- }
- }
- }
- return .success(RecordBatch(self.schemaBuilder.finish(), columns: self.columns))
- }
- }
-
- public func data<T>(for columnIndex: Int) -> ArrowArray<T> {
- let arrayHolder = column(columnIndex)
- return (arrayHolder.array as! ArrowArray<T>) // swiftlint:disable:this force_cast
- }
-
- public func anyData(for columnIndex: Int) -> AnyArray {
- let arrayHolder = column(columnIndex)
- return arrayHolder.array
- }
-
- public func column(_ index: Int) -> ArrowArrayHolder {
- return self.columns[index]
- }
-
- public func column(_ name: String) -> ArrowArrayHolder? {
- if let index = self.schema.fieldIndex(name) {
- return self.columns[index]
- }
-
- return nil
- }
-}
diff --git a/Sources/SparkConnect/ArrowType.swift b/Sources/SparkConnect/ArrowType.swift
deleted file mode 100644
index cdf9772..0000000
--- a/Sources/SparkConnect/ArrowType.swift
+++ /dev/null
@@ -1,417 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import Foundation
-
-/// @nodoc
-public typealias Time32 = Int32
-/// @nodoc
-public typealias Time64 = Int64
-/// @nodoc
-public typealias Date32 = Int32
-/// @nodoc
-public typealias Date64 = Int64
-
-func FlatBuffersVersion_23_1_4() { // swiftlint:disable:this identifier_name
-}
-
-/// @nodoc
-public enum ArrowError: Error {
- case none
- case unknownType(String)
- case runtimeError(String)
- case outOfBounds(index: Int64)
- case arrayHasNoElements
- case unknownError(String)
- case notImplemented
- case ioError(String)
- case invalid(String)
-}
-
-public enum ArrowTypeId: Sendable {
- case binary
- case boolean
- case date32
- case date64
- case dateType
- case decimal128
- case decimal256
- case dictionary
- case double
- case fixedSizeBinary
- case fixedWidthType
- case float
- // case HalfFloatType
- case int16
- case int32
- case int64
- case int8
- case integer
- case intervalUnit
- case list
- case nested
- case null
- case number
- case string
- case strct
- case time32
- case time64
- case time
- case uint16
- case uint32
- case uint64
- case uint8
- case union
- case unknown
-}
-
-public enum ArrowTime32Unit {
- case seconds
- case milliseconds
-}
-
-public enum ArrowTime64Unit {
- case microseconds
- case nanoseconds
-}
-
-/// @nodoc
-public class ArrowTypeTime32: ArrowType {
- let unit: ArrowTime32Unit
- public init(_ unit: ArrowTime32Unit) {
- self.unit = unit
- super.init(ArrowType.ArrowTime32)
- }
-
- public override var cDataFormatId: String {
- get throws {
- switch self.unit {
- case .milliseconds:
- return "ttm"
- case .seconds:
- return "tts"
- }
- }
- }
-}
-
-/// @nodoc
-public class ArrowTypeTime64: ArrowType {
- let unit: ArrowTime64Unit
- public init(_ unit: ArrowTime64Unit) {
- self.unit = unit
- super.init(ArrowType.ArrowTime64)
- }
-
- public override var cDataFormatId: String {
- get throws {
- switch self.unit {
- case .microseconds:
- return "ttu"
- case .nanoseconds:
- return "ttn"
- }
- }
- }
-}
-
-/// @nodoc
-public class ArrowNestedType: ArrowType {
- let fields: [ArrowField]
- public init(_ info: ArrowType.Info, fields: [ArrowField]) {
- self.fields = fields
- super.init(info)
- }
-}
-
-/// @nodoc
-public class ArrowType {
- public private(set) var info: ArrowType.Info
- public static let ArrowInt8 = Info.primitiveInfo(ArrowTypeId.int8)
- public static let ArrowInt16 = Info.primitiveInfo(ArrowTypeId.int16)
- public static let ArrowInt32 = Info.primitiveInfo(ArrowTypeId.int32)
- public static let ArrowInt64 = Info.primitiveInfo(ArrowTypeId.int64)
- public static let ArrowUInt8 = Info.primitiveInfo(ArrowTypeId.uint8)
- public static let ArrowUInt16 = Info.primitiveInfo(ArrowTypeId.uint16)
- public static let ArrowUInt32 = Info.primitiveInfo(ArrowTypeId.uint32)
- public static let ArrowUInt64 = Info.primitiveInfo(ArrowTypeId.uint64)
- public static let ArrowFloat = Info.primitiveInfo(ArrowTypeId.float)
- public static let ArrowDouble = Info.primitiveInfo(ArrowTypeId.double)
- public static let ArrowUnknown = Info.primitiveInfo(ArrowTypeId.unknown)
- public static let ArrowString = Info.variableInfo(ArrowTypeId.string)
- public static let ArrowBool = Info.primitiveInfo(ArrowTypeId.boolean)
- public static let ArrowDate32 = Info.primitiveInfo(ArrowTypeId.date32)
- public static let ArrowDate64 = Info.primitiveInfo(ArrowTypeId.date64)
- public static let ArrowBinary = Info.variableInfo(ArrowTypeId.binary)
- public static let ArrowTime32 = Info.timeInfo(ArrowTypeId.time32)
- public static let ArrowTime64 = Info.timeInfo(ArrowTypeId.time64)
- public static let ArrowStruct = Info.complexInfo(ArrowTypeId.strct)
-
- public init(_ info: ArrowType.Info) {
- self.info = info
- }
-
- public var id: ArrowTypeId {
- switch self.info {
- case .primitiveInfo(let id):
- return id
- case .timeInfo(let id):
- return id
- case .variableInfo(let id):
- return id
- case .complexInfo(let id):
- return id
- }
- }
-
- public enum Info: Sendable {
- case primitiveInfo(ArrowTypeId)
- case variableInfo(ArrowTypeId)
- case timeInfo(ArrowTypeId)
- case complexInfo(ArrowTypeId)
- }
-
- public static func infoForType( // swiftlint:disable:this cyclomatic_complexity
- _ type: Any.Type
- ) -> ArrowType.Info {
- if type == String.self {
- return ArrowType.ArrowString
- } else if type == Date.self {
- return ArrowType.ArrowDate64
- } else if type == Bool.self {
- return ArrowType.ArrowBool
- } else if type == Data.self {
- return ArrowType.ArrowBinary
- } else if type == Int8.self {
- return ArrowType.ArrowInt8
- } else if type == Int16.self {
- return ArrowType.ArrowInt16
- } else if type == Int32.self {
- return ArrowType.ArrowInt32
- } else if type == Int64.self {
- return ArrowType.ArrowInt64
- } else if type == UInt8.self {
- return ArrowType.ArrowUInt8
- } else if type == UInt16.self {
- return ArrowType.ArrowUInt16
- } else if type == UInt32.self {
- return ArrowType.ArrowUInt32
- } else if type == UInt64.self {
- return ArrowType.ArrowUInt64
- } else if type == Float.self {
- return ArrowType.ArrowFloat
- } else if type == Double.self {
- return ArrowType.ArrowDouble
- } else {
- return ArrowType.ArrowUnknown
- }
- }
-
- public static func infoForNumericType<T>(_ type: T.Type) -> ArrowType.Info {
- if type == Int8.self {
- return ArrowType.ArrowInt8
- } else if type == Int16.self {
- return ArrowType.ArrowInt16
- } else if type == Int32.self {
- return ArrowType.ArrowInt32
- } else if type == Int64.self {
- return ArrowType.ArrowInt64
- } else if type == UInt8.self {
- return ArrowType.ArrowUInt8
- } else if type == UInt16.self {
- return ArrowType.ArrowUInt16
- } else if type == UInt32.self {
- return ArrowType.ArrowUInt32
- } else if type == UInt64.self {
- return ArrowType.ArrowUInt64
- } else if type == Float.self {
- return ArrowType.ArrowFloat
- } else if type == Double.self {
- return ArrowType.ArrowDouble
- } else {
- return ArrowType.ArrowUnknown
- }
- }
-
- public func getStride( // swiftlint:disable:this cyclomatic_complexity
- ) -> Int
- {
- switch self.id {
- case .int8:
- return MemoryLayout<Int8>.stride
- case .int16:
- return MemoryLayout<Int16>.stride
- case .int32:
- return MemoryLayout<Int32>.stride
- case .int64:
- return MemoryLayout<Int64>.stride
- case .uint8:
- return MemoryLayout<UInt8>.stride
- case .uint16:
- return MemoryLayout<UInt16>.stride
- case .uint32:
- return MemoryLayout<UInt32>.stride
- case .uint64:
- return MemoryLayout<UInt64>.stride
- case .float:
- return MemoryLayout<Float>.stride
- case .double:
- return MemoryLayout<Double>.stride
- case .boolean:
- return MemoryLayout<Bool>.stride
- case .date32:
- return MemoryLayout<Date32>.stride
- case .date64:
- return MemoryLayout<Date64>.stride
- case .time32:
- return MemoryLayout<Time32>.stride
- case .time64:
- return MemoryLayout<Time64>.stride
- case .binary:
- return MemoryLayout<Int8>.stride
- case .string:
- return MemoryLayout<Int8>.stride
- case .strct:
- return 0
- default:
- fatalError("Stride requested for unknown type: \(self)")
- }
- }
-
- public var cDataFormatId: String {
- get throws {
- switch self.id {
- case ArrowTypeId.int8:
- return "c"
- case ArrowTypeId.int16:
- return "s"
- case ArrowTypeId.int32:
- return "i"
- case ArrowTypeId.int64:
- return "l"
- case ArrowTypeId.uint8:
- return "C"
- case ArrowTypeId.uint16:
- return "S"
- case ArrowTypeId.uint32:
- return "I"
- case ArrowTypeId.uint64:
- return "L"
- case ArrowTypeId.float:
- return "f"
- case ArrowTypeId.double:
- return "g"
- case ArrowTypeId.boolean:
- return "b"
- case ArrowTypeId.date32:
- return "tdD"
- case ArrowTypeId.date64:
- return "tdm"
- case ArrowTypeId.time32:
- if let time32 = self as? ArrowTypeTime32 {
- return try time32.cDataFormatId
- }
- return "tts"
- case ArrowTypeId.time64:
- if let time64 = self as? ArrowTypeTime64 {
- return try time64.cDataFormatId
- }
- return "ttu"
- case ArrowTypeId.binary:
- return "z"
- case ArrowTypeId.string:
- return "u"
- default:
- throw ArrowError.notImplemented
- }
- }
- }
-
- public static func fromCDataFormatId( // swiftlint:disable:this cyclomatic_complexity
- _ from: String
- ) throws -> ArrowType {
- if from == "c" {
- return ArrowType(ArrowType.ArrowInt8)
- } else if from == "s" {
- return ArrowType(ArrowType.ArrowInt16)
- } else if from == "i" {
- return ArrowType(ArrowType.ArrowInt32)
- } else if from == "l" {
- return ArrowType(ArrowType.ArrowInt64)
- } else if from == "C" {
- return ArrowType(ArrowType.ArrowUInt8)
- } else if from == "S" {
- return ArrowType(ArrowType.ArrowUInt16)
- } else if from == "I" {
- return ArrowType(ArrowType.ArrowUInt32)
- } else if from == "L" {
- return ArrowType(ArrowType.ArrowUInt64)
- } else if from == "f" {
- return ArrowType(ArrowType.ArrowFloat)
- } else if from == "g" {
- return ArrowType(ArrowType.ArrowDouble)
- } else if from == "b" {
- return ArrowType(ArrowType.ArrowBool)
- } else if from == "tdD" {
- return ArrowType(ArrowType.ArrowDate32)
- } else if from == "tdm" {
- return ArrowType(ArrowType.ArrowDate64)
- } else if from == "tts" {
- return ArrowTypeTime32(.seconds)
- } else if from == "ttm" {
- return ArrowTypeTime32(.milliseconds)
- } else if from == "ttu" {
- return ArrowTypeTime64(.microseconds)
- } else if from == "ttn" {
- return ArrowTypeTime64(.nanoseconds)
- } else if from == "z" {
- return ArrowType(ArrowType.ArrowBinary)
- } else if from == "u" {
- return ArrowType(ArrowType.ArrowString)
- }
-
- throw ArrowError.notImplemented
- }
-}
-
-extension ArrowType.Info: Equatable {
- public static func == (lhs: ArrowType.Info, rhs: ArrowType.Info) -> Bool {
- switch (lhs, rhs) {
- case (.primitiveInfo(let lhsId), .primitiveInfo(let rhsId)):
- return lhsId == rhsId
- case (.variableInfo(let lhsId), .variableInfo(let rhsId)):
- return lhsId == rhsId
- case (.timeInfo(let lhsId), .timeInfo(let rhsId)):
- return lhsId == rhsId
- case (.complexInfo(let lhsId), .complexInfo(let rhsId)):
- return lhsId == rhsId
- default:
- return false
- }
- }
-}
-
-func getBytesFor<T>(_ data: T) -> Data? {
- if let temp = data as? String {
- return temp.data(using: .utf8)
- } else if T.self == Data.self {
- return data as? Data
- } else {
- return nil
- }
-}
-// swiftlint:disable:this file_length
diff --git a/Sources/SparkConnect/ArrowWriter.swift b/Sources/SparkConnect/ArrowWriter.swift
deleted file mode 100644
index 4b644cf..0000000
--- a/Sources/SparkConnect/ArrowWriter.swift
+++ /dev/null
@@ -1,366 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import FlatBuffers
-import Foundation
-
-/// @nodoc
-public protocol DataWriter {
- var count: Int { get }
- func append(_ data: Data)
-}
-
-/// @nodoc
-public class ArrowWriter { // swiftlint:disable:this type_body_length
- public class InMemDataWriter: DataWriter {
- public private(set) var data: Data
- public var count: Int { return data.count }
- public init(_ data: Data) {
- self.data = data
- }
- convenience init() {
- self.init(Data())
- }
-
- public func append(_ data: Data) {
- self.data.append(data)
- }
- }
-
- public class FileDataWriter: DataWriter {
- private var handle: FileHandle
- private var currentSize: Int = 0
- public var count: Int { return currentSize }
- public init(_ handle: FileHandle) {
- self.handle = handle
- }
-
- public func append(_ data: Data) {
- self.handle.write(data)
- self.currentSize += data.count
- }
- }
-
- public class Info {
- public let type: org_apache_arrow_flatbuf_MessageHeader
- public let schema: ArrowSchema
- public let batches: [RecordBatch]
- public init(
- _ type: org_apache_arrow_flatbuf_MessageHeader, schema: ArrowSchema, batches: [RecordBatch]
- ) {
- self.type = type
- self.schema = schema
- self.batches = batches
- }
-
- public convenience init(_ type: org_apache_arrow_flatbuf_MessageHeader, schema: ArrowSchema) {
- self.init(type, schema: schema, batches: [RecordBatch]())
- }
- }
-
- public init() {}
-
- private func writeField(_ fbb: inout FlatBufferBuilder, field: ArrowField) -> Result<
- Offset, ArrowError
- > {
- let nameOffset = fbb.create(string: field.name)
- let fieldTypeOffsetResult = toFBType(&fbb, arrowType: field.type)
- let startOffset = org_apache_arrow_flatbuf_Field.startField(&fbb)
- org_apache_arrow_flatbuf_Field.add(name: nameOffset, &fbb)
- org_apache_arrow_flatbuf_Field.add(nullable: field.isNullable, &fbb)
- switch toFBTypeEnum(field.type) {
- case .success(let type):
- org_apache_arrow_flatbuf_Field.add(typeType: type, &fbb)
- case .failure(let error):
- return .failure(error)
- }
-
- switch fieldTypeOffsetResult {
- case .success(let offset):
- org_apache_arrow_flatbuf_Field.add(type: offset, &fbb)
- return .success(org_apache_arrow_flatbuf_Field.endField(&fbb, start: startOffset))
- case .failure(let error):
- return .failure(error)
- }
- }
-
- private func writeSchema(_ fbb: inout FlatBufferBuilder, schema: ArrowSchema) -> Result<
- Offset, ArrowError
- > {
- var fieldOffsets = [Offset]()
- for field in schema.fields {
- switch writeField(&fbb, field: field) {
- case .success(let offset):
- fieldOffsets.append(offset)
- case .failure(let error):
- return .failure(error)
- }
-
- }
-
- let fieldsOffset: Offset = fbb.createVector(ofOffsets: fieldOffsets)
- let schemaOffset =
- org_apache_arrow_flatbuf_Schema.createSchema(
- &fbb,
- endianness: .little,
- fieldsVectorOffset: fieldsOffset)
- return .success(schemaOffset)
-
- }
-
- private func writeRecordBatches(
- _ writer: inout DataWriter,
- batches: [RecordBatch]
- ) -> Result<[org_apache_arrow_flatbuf_Block], ArrowError> {
- var rbBlocks = [org_apache_arrow_flatbuf_Block]()
-
- for batch in batches {
- let startIndex = writer.count
- switch writeRecordBatch(batch: batch) {
- case .success(let rbResult):
- withUnsafeBytes(of: rbResult.1.o.littleEndian) { writer.append(Data($0)) }
- writer.append(rbResult.0)
- switch writeRecordBatchData(&writer, batch: batch) {
- case .success:
- rbBlocks.append(
- org_apache_arrow_flatbuf_Block(
- offset: Int64(startIndex),
- metaDataLength: Int32(0),
- bodyLength: Int64(rbResult.1.o)))
- case .failure(let error):
- return .failure(error)
- }
- case .failure(let error):
- return .failure(error)
- }
- }
-
- return .success(rbBlocks)
- }
-
- private func writeRecordBatch(batch: RecordBatch) -> Result<(Data, Offset), ArrowError> {
- let schema = batch.schema
- var fbb = FlatBufferBuilder()
-
- // write out field nodes
- var fieldNodeOffsets = [Offset]()
- fbb.startVector(
- schema.fields.count, elementSize: MemoryLayout<org_apache_arrow_flatbuf_FieldNode>.size)
- for index in (0..<schema.fields.count).reversed() {
- let column = batch.column(index)
- let fieldNode =
- org_apache_arrow_flatbuf_FieldNode(
- length: Int64(column.length),
- nullCount: Int64(column.nullCount))
- fieldNodeOffsets.append(fbb.create(struct: fieldNode))
- }
-
- let nodeOffset = fbb.endVector(len: schema.fields.count)
-
- // write out buffers
- var buffers = [org_apache_arrow_flatbuf_Buffer]()
- var bufferOffset = Int(0)
- for index in 0..<batch.schema.fields.count {
- let column = batch.column(index)
- let colBufferDataSizes = column.getBufferDataSizes()
- for var bufferDataSize in colBufferDataSizes {
- bufferDataSize = getPadForAlignment(bufferDataSize)
- let buffer = org_apache_arrow_flatbuf_Buffer(
- offset: Int64(bufferOffset), length: Int64(bufferDataSize))
- buffers.append(buffer)
- bufferOffset += bufferDataSize
- }
- }
-
- org_apache_arrow_flatbuf_RecordBatch.startVectorOfBuffers(batch.schema.fields.count, in: &fbb)
- for buffer in buffers.reversed() {
- fbb.create(struct: buffer)
- }
-
- let batchBuffersOffset = fbb.endVector(len: buffers.count)
- let startRb = org_apache_arrow_flatbuf_RecordBatch.startRecordBatch(&fbb)
- org_apache_arrow_flatbuf_RecordBatch.addVectorOf(nodes: nodeOffset, &fbb)
- org_apache_arrow_flatbuf_RecordBatch.addVectorOf(buffers: batchBuffersOffset, &fbb)
- org_apache_arrow_flatbuf_RecordBatch.add(length: Int64(batch.length), &fbb)
- let recordBatchOffset = org_apache_arrow_flatbuf_RecordBatch.endRecordBatch(
- &fbb, start: startRb)
- let bodySize = Int64(bufferOffset)
- let startMessage = org_apache_arrow_flatbuf_Message.startMessage(&fbb)
- org_apache_arrow_flatbuf_Message.add(version: .max, &fbb)
- org_apache_arrow_flatbuf_Message.add(bodyLength: Int64(bodySize), &fbb)
- org_apache_arrow_flatbuf_Message.add(headerType: .recordbatch, &fbb)
- org_apache_arrow_flatbuf_Message.add(header: recordBatchOffset, &fbb)
- let messageOffset = org_apache_arrow_flatbuf_Message.endMessage(&fbb, start: startMessage)
- fbb.finish(offset: messageOffset)
- return .success((fbb.data, Offset(offset: UInt32(fbb.data.count))))
- }
-
- private func writeRecordBatchData(_ writer: inout DataWriter, batch: RecordBatch) -> Result<
- Bool, ArrowError
- > {
- for index in 0..<batch.schema.fields.count {
- let column = batch.column(index)
- let colBufferData = column.getBufferData()
- for var bufferData in colBufferData {
- addPadForAlignment(&bufferData)
- writer.append(bufferData)
- }
- }
-
- return .success(true)
- }
-
- private func writeFooter(
- schema: ArrowSchema,
- rbBlocks: [org_apache_arrow_flatbuf_Block]
- ) -> Result<Data, ArrowError> {
- var fbb: FlatBufferBuilder = FlatBufferBuilder()
- switch writeSchema(&fbb, schema: schema) {
- case .success(let schemaOffset):
- fbb.startVector(
- rbBlocks.count, elementSize: MemoryLayout<org_apache_arrow_flatbuf_Block>.size)
- for blkInfo in rbBlocks.reversed() {
- fbb.create(struct: blkInfo)
- }
-
- let rbBlkEnd = fbb.endVector(len: rbBlocks.count)
- let footerStartOffset = org_apache_arrow_flatbuf_Footer.startFooter(&fbb)
- org_apache_arrow_flatbuf_Footer.add(schema: schemaOffset, &fbb)
- org_apache_arrow_flatbuf_Footer.addVectorOf(recordBatches: rbBlkEnd, &fbb)
- let footerOffset = org_apache_arrow_flatbuf_Footer.endFooter(&fbb, start: footerStartOffset)
- fbb.finish(offset: footerOffset)
- case .failure(let error):
- return .failure(error)
- }
-
- return .success(fbb.data)
- }
-
- private func writeStream(_ writer: inout DataWriter, info: ArrowWriter.Info) -> Result<
- Bool, ArrowError
- > {
- var fbb: FlatBufferBuilder = FlatBufferBuilder()
- switch writeSchema(&fbb, schema: info.schema) {
- case .success(let schemaOffset):
- fbb.finish(offset: schemaOffset)
- writer.append(fbb.data)
- case .failure(let error):
- return .failure(error)
- }
-
- switch writeRecordBatches(&writer, batches: info.batches) {
- case .success(let rbBlocks):
- switch writeFooter(schema: info.schema, rbBlocks: rbBlocks) {
- case .success(let footerData):
- fbb.finish(offset: Offset(offset: fbb.buffer.size))
- let footerOffset = writer.count
- writer.append(footerData)
- addPadForAlignment(&writer)
-
- withUnsafeBytes(of: Int32(0).littleEndian) { writer.append(Data($0)) }
- let footerDiff = (UInt32(writer.count) - UInt32(footerOffset))
- withUnsafeBytes(of: footerDiff.littleEndian) { writer.append(Data($0)) }
- case .failure(let error):
- return .failure(error)
- }
- case .failure(let error):
- return .failure(error)
- }
-
- return .success(true)
- }
-
- public func toStream(_ info: ArrowWriter.Info) -> Result<Data, ArrowError> {
- var writer: any DataWriter = InMemDataWriter()
- switch writeStream(&writer, info: info) {
- case .success:
- if let memWriter = writer as? InMemDataWriter {
- return .success(memWriter.data)
- } else {
- return .failure(.invalid("Unable to cast writer"))
- }
- case .failure(let error):
- return .failure(error)
- }
- }
-
- public func toFile(_ fileName: URL, info: ArrowWriter.Info) -> Result<Bool, ArrowError> {
- do {
- try Data().write(to: fileName)
- } catch {
- return .failure(.ioError("\(error)"))
- }
-
- let fileHandle = FileHandle(forUpdatingAtPath: fileName.path)!
- defer { fileHandle.closeFile() }
-
- var markerData = FILEMARKER.data(using: .utf8)!
- addPadForAlignment(&markerData)
-
- var writer: any DataWriter = FileDataWriter(fileHandle)
- writer.append(FILEMARKER.data(using: .utf8)!)
- switch writeStream(&writer, info: info) {
- case .success:
- writer.append(FILEMARKER.data(using: .utf8)!)
- case .failure(let error):
- return .failure(error)
- }
-
- return .success(true)
- }
-
- public func toMessage(_ batch: RecordBatch) -> Result<[Data], ArrowError> {
- var writer: any DataWriter = InMemDataWriter()
- switch writeRecordBatch(batch: batch) {
- case .success(let message):
- writer.append(message.0)
- addPadForAlignment(&writer)
- var dataWriter: any DataWriter = InMemDataWriter()
- switch writeRecordBatchData(&dataWriter, batch: batch) {
- case .success:
- return .success([
- (writer as! InMemDataWriter).data, // swiftlint:disable:this force_cast
- (dataWriter as! InMemDataWriter).data, // swiftlint:disable:this force_cast
- ])
- case .failure(let error):
- return .failure(error)
- }
- case .failure(let error):
- return .failure(error)
- }
- }
-
- public func toMessage(_ schema: ArrowSchema) -> Result<Data, ArrowError> {
- var schemaSize: Int32 = 0
- var fbb = FlatBufferBuilder()
- switch writeSchema(&fbb, schema: schema) {
- case .success(let schemaOffset):
- schemaSize = Int32(schemaOffset.o)
- case .failure(let error):
- return .failure(error)
- }
-
- let startMessage = org_apache_arrow_flatbuf_Message.startMessage(&fbb)
- org_apache_arrow_flatbuf_Message.add(bodyLength: Int64(0), &fbb)
- org_apache_arrow_flatbuf_Message.add(headerType: .schema, &fbb)
- org_apache_arrow_flatbuf_Message.add(header: Offset(offset: UOffset(schemaSize)), &fbb)
- org_apache_arrow_flatbuf_Message.add(version: .max, &fbb)
- let messageOffset = org_apache_arrow_flatbuf_Message.endMessage(&fbb, start: startMessage)
- fbb.finish(offset: messageOffset)
- return .success(fbb.data)
- }
-}
diff --git a/Sources/SparkConnect/ArrowWriterHelper.swift b/Sources/SparkConnect/ArrowWriterHelper.swift
deleted file mode 100644
index 7702e48..0000000
--- a/Sources/SparkConnect/ArrowWriterHelper.swift
+++ /dev/null
@@ -1,142 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import FlatBuffers
-import Foundation
-
-extension Data {
- func hexEncodedString() -> String {
- return map { String(format: "%02hhx", $0) }.joined()
- }
-}
-
-func toFBTypeEnum(_ arrowType: ArrowType) -> Result<org_apache_arrow_flatbuf_Type_, ArrowError> {
- let infoType = arrowType.info
- if infoType == ArrowType.ArrowInt8 || infoType == ArrowType.ArrowInt16
- || infoType == ArrowType.ArrowInt64 || infoType == ArrowType.ArrowUInt8
- || infoType == ArrowType.ArrowUInt16 || infoType == ArrowType.ArrowUInt32
- || infoType == ArrowType.ArrowUInt64 || infoType == ArrowType.ArrowInt32
- {
- return .success(org_apache_arrow_flatbuf_Type_.int)
- } else if infoType == ArrowType.ArrowFloat || infoType == ArrowType.ArrowDouble {
- return .success(org_apache_arrow_flatbuf_Type_.floatingpoint)
- } else if infoType == ArrowType.ArrowString {
- return .success(org_apache_arrow_flatbuf_Type_.utf8)
- } else if infoType == ArrowType.ArrowBinary {
- return .success(org_apache_arrow_flatbuf_Type_.binary)
- } else if infoType == ArrowType.ArrowBool {
- return .success(org_apache_arrow_flatbuf_Type_.bool)
- } else if infoType == ArrowType.ArrowDate32 || infoType == ArrowType.ArrowDate64 {
- return .success(org_apache_arrow_flatbuf_Type_.date)
- } else if infoType == ArrowType.ArrowTime32 || infoType == ArrowType.ArrowTime64 {
- return .success(org_apache_arrow_flatbuf_Type_.time)
- }
- return .failure(.unknownType("Unable to find flatbuf type for Arrow type: \(infoType)"))
-}
-
-func toFBType( // swiftlint:disable:this cyclomatic_complexity
- _ fbb: inout FlatBufferBuilder,
- arrowType: ArrowType
-) -> Result<Offset, ArrowError> {
- let infoType = arrowType.info
- if infoType == ArrowType.ArrowInt8 || infoType == ArrowType.ArrowUInt8 {
- return .success(
- org_apache_arrow_flatbuf_Int.createInt(
- &fbb, bitWidth: 8, isSigned: infoType == ArrowType.ArrowInt8))
- } else if infoType == ArrowType.ArrowInt16 || infoType == ArrowType.ArrowUInt16 {
- return .success(
- org_apache_arrow_flatbuf_Int.createInt(
- &fbb, bitWidth: 16, isSigned: infoType == ArrowType.ArrowInt16))
- } else if infoType == ArrowType.ArrowInt32 || infoType == ArrowType.ArrowUInt32 {
- return .success(
- org_apache_arrow_flatbuf_Int.createInt(
- &fbb, bitWidth: 32, isSigned: infoType == ArrowType.ArrowInt32))
- } else if infoType == ArrowType.ArrowInt64 || infoType == ArrowType.ArrowUInt64 {
- return .success(
- org_apache_arrow_flatbuf_Int.createInt(
- &fbb, bitWidth: 64, isSigned: infoType == ArrowType.ArrowInt64))
- } else if infoType == ArrowType.ArrowFloat {
- return .success(
- org_apache_arrow_flatbuf_FloatingPoint.createFloatingPoint(&fbb, precision: .single))
- } else if infoType == ArrowType.ArrowDouble {
- return .success(
- org_apache_arrow_flatbuf_FloatingPoint.createFloatingPoint(&fbb, precision: .double))
- } else if infoType == ArrowType.ArrowString {
- return .success(
- org_apache_arrow_flatbuf_Utf8.endUtf8(
- &fbb, start: org_apache_arrow_flatbuf_Utf8.startUtf8(&fbb)))
- } else if infoType == ArrowType.ArrowBinary {
- return .success(
- org_apache_arrow_flatbuf_Binary.endBinary(
- &fbb, start: org_apache_arrow_flatbuf_Binary.startBinary(&fbb)))
- } else if infoType == ArrowType.ArrowBool {
- return .success(
- org_apache_arrow_flatbuf_Bool.endBool(
- &fbb, start: org_apache_arrow_flatbuf_Bool.startBool(&fbb)))
- } else if infoType == ArrowType.ArrowDate32 {
- let startOffset = org_apache_arrow_flatbuf_Date.startDate(&fbb)
- org_apache_arrow_flatbuf_Date.add(unit: .day, &fbb)
- return .success(org_apache_arrow_flatbuf_Date.endDate(&fbb, start: startOffset))
- } else if infoType == ArrowType.ArrowDate64 {
- let startOffset = org_apache_arrow_flatbuf_Date.startDate(&fbb)
- org_apache_arrow_flatbuf_Date.add(unit: .millisecond, &fbb)
- return .success(org_apache_arrow_flatbuf_Date.endDate(&fbb, start: startOffset))
- } else if infoType == ArrowType.ArrowTime32 {
- let startOffset = org_apache_arrow_flatbuf_Time.startTime(&fbb)
- if let timeType = arrowType as? ArrowTypeTime32 {
- org_apache_arrow_flatbuf_Time.add(
- unit: timeType.unit == .seconds ? .second : .millisecond, &fbb)
- return .success(org_apache_arrow_flatbuf_Time.endTime(&fbb, start: startOffset))
- }
-
- return .failure(.invalid("Unable to case to Time32"))
- } else if infoType == ArrowType.ArrowTime64 {
- let startOffset = org_apache_arrow_flatbuf_Time.startTime(&fbb)
- if let timeType = arrowType as? ArrowTypeTime64 {
- org_apache_arrow_flatbuf_Time.add(
- unit: timeType.unit == .microseconds ? .microsecond : .nanosecond, &fbb)
- return .success(org_apache_arrow_flatbuf_Time.endTime(&fbb, start: startOffset))
- }
-
- return .failure(.invalid("Unable to case to Time64"))
- }
-
- return .failure(.unknownType("Unable to add flatbuf type for Arrow type: \(infoType)"))
-}
-
-func addPadForAlignment(_ data: inout Data, alignment: Int = 8) {
- let padding = data.count % Int(alignment)
- if padding > 0 {
- data.append(Data([UInt8](repeating: 0, count: alignment - padding)))
- }
-}
-
-func addPadForAlignment(_ writer: inout DataWriter, alignment: Int = 8) {
- let padding = writer.count % Int(alignment)
- if padding > 0 {
- writer.append(Data([UInt8](repeating: 0, count: alignment - padding)))
- }
-}
-
-func getPadForAlignment(_ count: Int, alignment: Int = 8) -> Int {
- let padding = count % Int(alignment)
- if padding > 0 {
- return count + (alignment - padding)
- }
-
- return count
-}
diff --git a/Sources/SparkConnect/BearerTokenInterceptor.swift b/Sources/SparkConnect/BearerTokenInterceptor.swift
deleted file mode 100644
index 07040f5..0000000
--- a/Sources/SparkConnect/BearerTokenInterceptor.swift
+++ /dev/null
@@ -1,44 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-
-import Foundation
-import GRPCCore
-
-struct BearerTokenInterceptor: ClientInterceptor {
- let token: String
-
- init(token: String) {
- self.token = token
- }
-
- func intercept<Input: Sendable, Output: Sendable>(
- request: StreamingClientRequest<Input>,
- context: ClientContext,
- next: (
- _ request: StreamingClientRequest<Input>,
- _ context: ClientContext
- ) async throws -> StreamingClientResponse<Output>
- ) async throws -> StreamingClientResponse<Output> {
- var request = request
- request.metadata.addString("Bearer \(self.token)", forKey: "Authorization")
-
- // Forward the request to the next interceptor.
- return try await next(request, context)
- }
-}
diff --git a/Sources/SparkConnect/BitUtility.swift b/Sources/SparkConnect/BitUtility.swift
deleted file mode 100644
index f461d65..0000000
--- a/Sources/SparkConnect/BitUtility.swift
+++ /dev/null
@@ -1,40 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import Foundation
-
-class BitUtility {
- static func isSet(_ bit: UInt, buffer: ArrowBuffer) -> Bool {
- let byteIndex = UInt(bit / 8)
- let theByte = buffer.rawPointer.load(fromByteOffset: Int(byteIndex), as: UInt8.self)
- return theByte & UInt8(1 << (bit % 8)) > 0
- }
-
- static func setBit(_ bit: UInt, buffer: ArrowBuffer) {
- let byteIndex = UInt(bit / 8)
- var theByte = buffer.rawPointer.load(fromByteOffset: Int(byteIndex), as: UInt8.self)
- theByte |= UInt8(1 << (bit % 8))
- buffer.rawPointer.storeBytes(of: theByte, toByteOffset: Int(byteIndex), as: UInt8.self)
- }
-
- static func clearBit(_ bit: UInt, buffer: ArrowBuffer) {
- let byteIndex = UInt(bit / 8)
- var theByte = buffer.rawPointer.load(fromByteOffset: Int(byteIndex), as: UInt8.self)
- theByte &= ~(UInt8(1 << (bit % 8)))
- buffer.rawPointer.storeBytes(of: theByte, toByteOffset: Int(byteIndex), as: UInt8.self)
- }
-}
diff --git a/Sources/SparkConnect/CaseInsensitiveDictionary.swift b/Sources/SparkConnect/CaseInsensitiveDictionary.swift
deleted file mode 100644
index ae6508a..0000000
--- a/Sources/SparkConnect/CaseInsensitiveDictionary.swift
+++ /dev/null
@@ -1,63 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-import Foundation
-
-/// A dictionary in which keys are case insensitive. The input dictionary can be
-/// accessed for cases where case-sensitive information is required.
-public struct CaseInsensitiveDictionary: Sendable {
- public var originalDictionary: [String: Sendable]
- private var keyLowerCasedDictionary: [String: Sendable] = [:]
-
- init(_ originalDictionary: [String: Sendable] = [:]) {
- self.originalDictionary = originalDictionary
- for (key, value) in originalDictionary {
- keyLowerCasedDictionary[key.lowercased()] = value
- }
- }
-
- subscript(key: String) -> Sendable? {
- get {
- return keyLowerCasedDictionary[key.lowercased()]
- }
- set {
- let lowerKey = key.lowercased()
- if let newValue = newValue {
- keyLowerCasedDictionary[lowerKey] = newValue
- } else {
- keyLowerCasedDictionary.removeValue(forKey: lowerKey)
- }
- originalDictionary = originalDictionary.filter { $0.key.lowercased() != lowerKey }
- if let newValue = newValue {
- originalDictionary[key] = newValue
- }
- }
- }
-
- public func toDictionary() -> [String: Sendable] {
- return originalDictionary
- }
-
- public func toStringDictionary() -> [String: String] {
- return originalDictionary.mapValues { String(describing: $0) }
- }
-
- public var count: Int {
- return keyLowerCasedDictionary.count
- }
-}
diff --git a/Sources/SparkConnect/Catalog.swift b/Sources/SparkConnect/Catalog.swift
deleted file mode 100644
index 6c86f61..0000000
--- a/Sources/SparkConnect/Catalog.swift
+++ /dev/null
@@ -1,444 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-import Foundation
-
-public struct CatalogMetadata: Sendable, Equatable {
- public var name: String
- public var description: String? = nil
-}
-
-public struct Database: Sendable, Equatable {
- public var name: String
- public var catalog: String? = nil
- public var description: String? = nil
- public var locationUri: String
-}
-
-// TODO: Rename `SparkTable` to `Table` after removing Arrow and Flatbuffer
-// from `SparkConnect` module. Currently, `SparkTable` is used to avoid the name conflict.
-public struct SparkTable: Sendable, Equatable {
- public var name: String
- public var catalog: String?
- public var namespace: [String]?
- public var description: String?
- public var tableType: String
- public var isTemporary: Bool
- public var database: String? {
- get {
- guard let namespace else {
- return nil
- }
- if namespace.count == 1 {
- return namespace[0]
- } else {
- return nil
- }
- }
- }
-}
-
-public struct Column: Sendable, Equatable {
- public var name: String
- public var description: String?
- public var dataType: String
- public var nullable: Bool
- public var isPartition: Bool
- public var isBucket: Bool
- public var isCluster: Bool
-}
-
-public struct Function: Sendable, Equatable {
- public var name: String
- public var catalog: String?
- public var namespace: [String]?
- public var description: String?
- public var className: String
- public var isTemporary: Bool
-}
-
-/// Interface through which the user may create, drop, alter or query underlying databases, tables, functions etc.
-/// To access this, use ``SparkSession.catalog``.
-public actor Catalog: Sendable {
- var spark: SparkSession
-
- init(spark: SparkSession) {
- self.spark = spark
- }
-
- /// A helper method to create a `Spark_Connect_Catalog`-based plan.
- /// - Parameter f: A lambda function to create `Spark_Connect_Catalog`.
- /// - Returns: A ``DataFrame`` contains the result of the given catalog operation.
- private func getDataFrame(_ f: () -> Spark_Connect_Catalog) -> DataFrame {
- var relation = Relation()
- relation.catalog = f()
- var plan = Plan()
- plan.opType = .root(relation)
- return DataFrame(spark: spark, plan: plan)
- }
-
- /// Returns the current default catalog in this session.
- /// - Returns: A catalog name.
- public func currentCatalog() async throws -> String {
- let df = getDataFrame({
- var catalog = Spark_Connect_Catalog()
- catalog.catType = .currentCatalog(Spark_Connect_CurrentCatalog())
- return catalog
- })
- return try await df.collect()[0][0] as! String
- }
-
- /// Sets the current default catalog in this session.
- /// - Parameter catalogName: name of the catalog to set
- public func setCurrentCatalog(_ catalogName: String) async throws {
- let df = getDataFrame({
- var setCurrentCatalog = Spark_Connect_SetCurrentCatalog()
- setCurrentCatalog.catalogName = catalogName
-
- var catalog = Spark_Connect_Catalog()
- catalog.catType = .setCurrentCatalog(setCurrentCatalog)
- return catalog
- })
- try await df.count()
- }
-
- /// Returns a list of catalogs in this session.
- /// - Returns: A list of ``CatalogMetadata``.
- public func listCatalogs(pattern: String? = nil) async throws -> [CatalogMetadata] {
- let df = getDataFrame({
- var listCatalogs = Spark_Connect_ListCatalogs()
- if let pattern {
- listCatalogs.pattern = pattern
- }
- var catalog = Spark_Connect_Catalog()
- catalog.catType = .listCatalogs(listCatalogs)
- return catalog
- })
- return try await df.collect().map {
- try CatalogMetadata(name: $0[0] as! String, description: $0[1] as? String)
- }
- }
-
- /// Returns the current default database in this session.
- /// - Returns: The current default database name.
- public func currentDatabase() async throws -> String {
- let df = getDataFrame({
- var catalog = Spark_Connect_Catalog()
- catalog.catType = .currentDatabase(Spark_Connect_CurrentDatabase())
- return catalog
- })
- return try await df.collect()[0][0] as! String
- }
-
- /// Sets the current default database in this session.
- /// - Parameter dbName: name of the catalog to set
- public func setCurrentDatabase(_ dbName: String) async throws {
- let df = getDataFrame({
- var setCurrentDatabase = Spark_Connect_SetCurrentDatabase()
- setCurrentDatabase.dbName = dbName
-
- var catalog = Spark_Connect_Catalog()
- catalog.catType = .setCurrentDatabase(setCurrentDatabase)
- return catalog
- })
- try await df.count()
- }
-
- /// Returns a list of databases available across all sessions.
- /// - Parameter pattern: The pattern that the database name needs to match.
- /// - Returns: A list of ``Database``.
- public func listDatabases(pattern: String? = nil) async throws -> [Database] {
- let df = getDataFrame({
- var listDatabases = Spark_Connect_ListDatabases()
- if let pattern {
- listDatabases.pattern = pattern
- }
- var catalog = Spark_Connect_Catalog()
- catalog.catType = .listDatabases(listDatabases)
- return catalog
- })
- return try await df.collect().map {
- try Database(name: $0[0] as! String, catalog: $0[1] as? String, description: $0[2] as? String, locationUri: $0[3] as! String)
- }
- }
-
- /// Get the database with the specified name.
- /// - Parameter dbName: name of the database to get.
- /// - Returns: The database found by the name.
- public func getDatabase(_ dbName: String) async throws -> Database {
- let df = getDataFrame({
- var db = Spark_Connect_GetDatabase()
- db.dbName = dbName
- var catalog = Spark_Connect_Catalog()
- catalog.catType = .getDatabase(db)
- return catalog
- })
- return try await df.collect().map {
- try Database(name: $0[0] as! String, catalog: $0[1] as? String, description: $0[2] as? String, locationUri: $0[3] as! String)
- }.first!
- }
-
- /// Check if the database with the specified name exists.
- /// - Parameter dbName: name of the database to check existence
- /// - Returns: Indicating whether the database exists.
- public func databaseExists(_ dbName: String) async throws -> Bool {
- return try await self.listDatabases(pattern: dbName).count > 0
- }
-
- /// Creates a table from the given path and returns the corresponding ``DataFrame``.
- /// - Parameters:
- /// - tableName: A qualified or unqualified name that designates a table. If no database
- /// identifier is provided, it refers to a table in the current database.
- /// - path: A path to load a table.
- /// - source: A data source.
- /// - description: A table description.
- /// - options: A dictionary for table options
- /// - Returns: A ``DataFrame``.
- public func createTable(
- _ tableName: String,
- _ path: String? = nil,
- source: String? = nil,
- description: String? = nil,
- options: [String: String]? = nil
- ) -> DataFrame {
- let df = getDataFrame({
- var createTable = Spark_Connect_CreateTable()
- createTable.tableName = tableName
- if let source {
- createTable.source = source
- }
- createTable.description_p = description ?? ""
- if let options {
- for (k, v) in options {
- createTable.options[k] = v
- }
- }
- if let path {
- createTable.options["path"] = path
- }
- var catalog = Spark_Connect_Catalog()
- catalog.createTable = createTable
- return catalog
- })
- return df
- }
-
- /// Check if the table or view with the specified name exists. This can either be a temporary
- /// view or a table/view.
- /// - Parameter tableName: a qualified or unqualified name that designates a table/view. It follows the same
- /// resolution rule with SQL: search for temp views first then table/views in the current
- /// database (namespace).
- /// - Returns: Return true if it exists.
- public func tableExists(_ tableName: String) async throws -> Bool {
- let df = getDataFrame({
- var tableExists = Spark_Connect_TableExists()
- tableExists.tableName = tableName
- var catalog = Spark_Connect_Catalog()
- catalog.tableExists = tableExists
- return catalog
- })
- return try await df.collect()[0].getAsBool(0)
- }
-
- /// Check if the table or view with the specified name exists. This can either be a temporary
- /// view or a table/view.
- /// - Parameters:
- /// - dbName: an unqualified name that designates a database.
- /// - tableName: an unqualified name that designates a table.
- /// - Returns: Return true if it exists.
- public func tableExists(_ dbName: String, _ tableName: String) async throws -> Bool {
- let df = getDataFrame({
- var tableExists = Spark_Connect_TableExists()
- tableExists.tableName = tableName
- tableExists.dbName = dbName
- var catalog = Spark_Connect_Catalog()
- catalog.tableExists = tableExists
- return catalog
- })
- return try await df.collect()[0].getAsBool(0)
- }
-
- /// Returns a list of columns for the given table/view or temporary view.
- /// - Parameter tableName: a qualified or unqualified name that designates a table/view. It follows the same
- /// resolution rule with SQL: search for temp views first then table/views in the current
- /// database (namespace).
- /// - Returns: A ``DataFrame`` of ``Column``.
- public func listColumns(_ tableName: String) async throws -> DataFrame {
- let df = getDataFrame({
- var listColumns = Spark_Connect_ListColumns()
- listColumns.tableName = tableName
- var catalog = Spark_Connect_Catalog()
- catalog.listColumns = listColumns
- return catalog
- })
- return df
- }
-
- /// Check if the function with the specified name exists. This can either be a temporary function
- /// or a function.
- /// - Parameter functionName: a qualified or unqualified name that designates a function. It follows the same
- /// resolution rule with SQL: search for built-in/temp functions first then functions in the
- /// current database (namespace).
- /// - Returns: Return true if it exists.
- public func functionExists(_ functionName: String) async throws -> Bool {
- let df = getDataFrame({
- var functionExists = Spark_Connect_FunctionExists()
- functionExists.functionName = functionName
- var catalog = Spark_Connect_Catalog()
- catalog.functionExists = functionExists
- return catalog
- })
- return try await df.collect()[0].getAsBool(0)
- }
-
- /// Check if the function with the specified name exists in the specified database under the Hive
- /// Metastore.
- /// - Parameters:
- /// - dbName: an unqualified name that designates a database.
- /// - functionName: an unqualified name that designates a function.
- /// - Returns: Return true if it exists.
- public func functionExists(_ dbName: String, _ functionName: String) async throws -> Bool {
- let df = getDataFrame({
- var functionExists = Spark_Connect_FunctionExists()
- functionExists.functionName = functionName
- functionExists.dbName = dbName
- var catalog = Spark_Connect_Catalog()
- catalog.functionExists = functionExists
- return catalog
- })
- return try await df.collect()[0].getAsBool(0)
- }
-
- /// Caches the specified table in-memory.
- /// - Parameters:
- /// - tableName: A qualified or unqualified name that designates a table/view.
- /// If no database identifier is provided, it refers to a temporary view or a table/view in the current database.
- /// - storageLevel: storage level to cache table.
- public func cacheTable(_ tableName: String, _ storageLevel: StorageLevel? = nil) async throws {
- let df = getDataFrame({
- var cacheTable = Spark_Connect_CacheTable()
- cacheTable.tableName = tableName
- if let storageLevel {
- cacheTable.storageLevel = storageLevel.toSparkConnectStorageLevel
- }
- var catalog = Spark_Connect_Catalog()
- catalog.cacheTable = cacheTable
- return catalog
- })
- try await df.count()
- }
-
- /// Returns true if the table is currently cached in-memory.
- /// - Parameter tableName: A qualified or unqualified name that designates a table/view.
- /// If no database identifier is provided, it refers to a temporary view or a table/view in the current database.
- public func isCached(_ tableName: String) async throws -> Bool {
- let df = getDataFrame({
- var isCached = Spark_Connect_IsCached()
- isCached.tableName = tableName
- var catalog = Spark_Connect_Catalog()
- catalog.isCached = isCached
- return catalog
- })
- return try await df.collect()[0].getAsBool(0)
- }
-
- /// Invalidates and refreshes all the cached data and metadata of the given table.
- /// - Parameter tableName: A qualified or unqualified name that designates a table/view.
- /// If no database identifier is provided, it refers to a temporary view or a table/view in the current database.
- public func refreshTable(_ tableName: String) async throws {
- let df = getDataFrame({
- var refreshTable = Spark_Connect_RefreshTable()
- refreshTable.tableName = tableName
- var catalog = Spark_Connect_Catalog()
- catalog.refreshTable = refreshTable
- return catalog
- })
- try await df.count()
- }
-
- /// Invalidates and refreshes all the cached data (and the associated metadata) for any ``DataFrame``
- /// that contains the given data source path. Path matching is by checking for sub-directories,
- /// i.e. "/" would invalidate everything that is cached and "/test/parent" would invalidate
- /// everything that is a subdirectory of "/test/parent".
- public func refreshByPath(_ path: String) async throws {
- let df = getDataFrame({
- var refreshByPath = Spark_Connect_RefreshByPath()
- refreshByPath.path = path
- var catalog = Spark_Connect_Catalog()
- catalog.refreshByPath = refreshByPath
- return catalog
- })
- try await df.count()
- }
-
- /// Removes the specified table from the in-memory cache.
- /// - Parameter tableName: A qualified or unqualified name that designates a table/view.
- /// If no database identifier is provided, it refers to a temporary view or a table/view in the current database.
- public func uncacheTable(_ tableName: String) async throws {
- let df = getDataFrame({
- var uncacheTable = Spark_Connect_UncacheTable()
- uncacheTable.tableName = tableName
- var catalog = Spark_Connect_Catalog()
- catalog.uncacheTable = uncacheTable
- return catalog
- })
- try await df.count()
- }
-
- /// Removes all cached tables from the in-memory cache.
- public func clearCache() async throws {
- let df = getDataFrame({
- var catalog = Spark_Connect_Catalog()
- catalog.clearCache_p = Spark_Connect_ClearCache()
- return catalog
- })
- try await df.count()
- }
-
- /// Drops the local temporary view with the given view name in the catalog. If the view has been
- /// cached before, then it will also be uncached.
- /// - Parameter viewName: The name of the temporary view to be dropped.
- /// - Returns: true if the view is dropped successfully, false otherwise.
- @discardableResult
- public func dropTempView(_ viewName: String) async throws -> Bool {
- let df = getDataFrame({
- var dropTempView = Spark_Connect_DropTempView()
- dropTempView.viewName = viewName
- var catalog = Spark_Connect_Catalog()
- catalog.dropTempView = dropTempView
- return catalog
- })
- return try await df.collect().first!.getAsBool(0)
- }
-
- /// Drops the global temporary view with the given view name in the catalog. If the view has been
- /// cached before, then it will also be uncached.
- /// - Parameter viewName: The unqualified name of the temporary view to be dropped.
- /// - Returns: true if the view is dropped successfully, false otherwise.
- @discardableResult
- public func dropGlobalTempView(_ viewName: String) async throws -> Bool {
- let df = getDataFrame({
- var dropGlobalTempView = Spark_Connect_DropGlobalTempView()
- dropGlobalTempView.viewName = viewName
- var catalog = Spark_Connect_Catalog()
- catalog.dropGlobalTempView = dropGlobalTempView
- return catalog
- })
- return try await df.collect()[0].getAsBool(0)
- }
-}
diff --git a/Sources/SparkConnect/ChunkedArray.swift b/Sources/SparkConnect/ChunkedArray.swift
deleted file mode 100644
index a6b4e03..0000000
--- a/Sources/SparkConnect/ChunkedArray.swift
+++ /dev/null
@@ -1,153 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-import Foundation
-
-/// @nodoc
-public protocol AnyArray {
- var arrowData: ArrowData { get }
- func asAny(_ index: UInt) -> Any?
- var length: UInt { get }
-}
-
-/// @nodoc
-public protocol AsString {
- func asString(_ index: UInt) -> String
-}
-
-/// @nodoc
-public class ChunkedArrayHolder {
- public let type: ArrowType
- public let length: UInt
- public let nullCount: UInt
- public let holder: Any
-
- public let getBufferData: () -> Result<[Data], ArrowError>
- public let getBufferDataSizes: () -> Result<[Int], ArrowError>
- public init<T>(_ chunked: ChunkedArray<T>) { // swiftlint:disable:this cyclomatic_complexity
- self.holder = chunked
- self.length = chunked.length
- self.type = chunked.type
- self.nullCount = chunked.nullCount
- self.getBufferData = { () -> Result<[Data], ArrowError> in
- var bufferData = [Data]()
- var numBuffers = 2
- switch toFBTypeEnum(chunked.type) {
- case .success(let fbType):
- if !isFixedPrimitive(fbType) {
- numBuffers = 3
- }
- case .failure(let error):
- return .failure(error)
- }
-
- for _ in 0..<numBuffers {
- bufferData.append(Data())
- }
-
- for arrowData in chunked.arrays {
- for index in 0..<numBuffers {
- arrowData.arrowData.buffers[index].append(to: &bufferData[index])
- }
- }
-
- return .success(bufferData)
- }
-
- self.getBufferDataSizes = { () -> Result<[Int], ArrowError> in
- var bufferDataSizes = [Int]()
- var numBuffers = 2
-
- switch toFBTypeEnum(chunked.type) {
- case .success(let fbType):
- if !isFixedPrimitive(fbType) {
- numBuffers = 3
- }
- case .failure(let error):
- return .failure(error)
- }
-
- for _ in 0..<numBuffers {
- bufferDataSizes.append(Int(0))
- }
-
- for arrowData in chunked.arrays {
- for index in 0..<numBuffers {
- bufferDataSizes[index] += Int(arrowData.arrowData.buffers[index].capacity)
- }
- }
-
- return .success(bufferDataSizes)
- }
- }
-}
-
-/// @nodoc
-public class ChunkedArray<T>: AsString {
- public let arrays: [ArrowArray<T>]
- public let type: ArrowType
- public let nullCount: UInt
- public let length: UInt
- public var arrayCount: UInt { return UInt(self.arrays.count) }
-
- public init(_ arrays: [ArrowArray<T>]) throws {
- if arrays.count == 0 {
- throw ArrowError.arrayHasNoElements
- }
-
- self.type = arrays[0].arrowData.type
- var len: UInt = 0
- var nullCount: UInt = 0
- for array in arrays {
- len += array.length
- nullCount += array.nullCount
- }
-
- self.arrays = arrays
- self.length = len
- self.nullCount = nullCount
- }
-
- public subscript(_ index: UInt) -> T? {
- if arrays.count == 0 {
- return nil
- }
-
- var localIndex = index
- var arrayIndex = 0
- var len: UInt = arrays[arrayIndex].length
- while localIndex > (len - 1) {
- arrayIndex += 1
- if arrayIndex > arrays.count {
- return nil
- }
-
- localIndex -= len
- len = arrays[arrayIndex].length
- }
-
- return arrays[arrayIndex][localIndex]
- }
-
- public func asString(_ index: UInt) -> String {
- if self[index] == nil {
- return ""
- }
-
- return "\(self[index]!)"
- }
-}
diff --git a/Sources/SparkConnect/DataFrame.swift b/Sources/SparkConnect/DataFrame.swift
deleted file mode 100644
index 1c7b08d..0000000
--- a/Sources/SparkConnect/DataFrame.swift
+++ /dev/null
@@ -1,1404 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-import Atomics
-import Foundation
-import GRPCCore
-import GRPCNIOTransportHTTP2
-import GRPCProtobuf
-import Synchronization
-
-/// A distributed collection of data organized into named columns.
-///
-/// A DataFrame is equivalent to a relational table in Spark SQL, and can be created using various
-/// functions in ``SparkSession``. Once created, it can be manipulated using the various domain-specific
-/// language (DSL) functions defined in: ``DataFrame``, ``Column``, and functions.
-///
-/// ## Creating DataFrames
-///
-/// DataFrames can be created from various sources:
-///
-/// ```swift
-/// // From a range
-/// let df1 = try await spark.range(1, 100)
-///
-/// // From a SQL query
-/// let df2 = try await spark.sql("SELECT * FROM users")
-///
-/// // From files
-/// let df3 = try await spark.read.csv("data.csv")
-/// ```
-///
-/// ## Common Operations
-///
-/// ### Transformations
-///
-/// ```swift
-/// // Select specific columns
-/// let names = try await df.select("name", "age")
-///
-/// // Filter rows
-/// let adults = try await df.filter("age >= 18")
-///
-/// // Group and aggregate
-/// let stats = try await df.groupBy("department").agg("avg(salary)", "count(*)")
-/// ```
-///
-/// ### Actions
-///
-/// ```swift
-/// // Show the first 20 rows
-/// try await df.show()
-///
-/// // Collect all data to the driver
-/// let rows = try await df.collect()
-///
-/// // Count rows
-/// let count = try await df.count()
-/// ```
-///
-/// ## Topics
-///
-/// ### Basic Information
-/// - ``columns``
-/// - ``schema``
-/// - ``dtypes``
-/// - ``sparkSession``
-///
-/// ### Data Collection
-/// - ``count()``
-/// - ``collect()``
-/// - ``first()``
-/// - ``head()``
-/// - ``head(_:)``
-/// - ``take(_:)``
-/// - ``tail(_:)``
-/// - ``show()``
-/// - ``show(_:)``
-/// - ``show(_:_:)``
-/// - ``show(_:_:_:)``
-///
-/// ### Transformation Operations
-/// - ``toDF(_:)``
-/// - ``toJSON()``
-/// - ``select(_:)``
-/// - ``selectExpr(_:)``
-/// - ``filter(_:)``
-/// - ``where(_:)``
-/// - ``sort(_:)``
-/// - ``orderBy(_:)``
-/// - ``limit(_:)``
-/// - ``offset(_:)``
-/// - ``drop(_:)``
-/// - ``dropDuplicates(_:)``
-/// - ``dropDuplicatesWithinWatermark(_:)``
-/// - ``distinct()``
-/// - ``withColumnRenamed(_:_:)``
-/// - ``unpivot(_:_:_:)``
-/// - ``unpivot(_:_:_:_:)``
-/// - ``melt(_:_:_:)``
-/// - ``melt(_:_:_:_:)``
-/// - ``transpose()``
-/// - ``transpose(_:)``
-///
-/// ### Join Operations
-/// - ``join(_:)``
-/// - ``join(_:_:_:)``
-/// - ``join(_:joinExprs:)``
-/// - ``join(_:joinExprs:joinType:)``
-/// - ``crossJoin(_:)``
-/// - ``lateralJoin(_:)``
-/// - ``lateralJoin(_:joinType:)``
-/// - ``lateralJoin(_:joinExprs:)``
-/// - ``lateralJoin(_:joinExprs:joinType:)``
-///
-/// ### Set Operations
-/// - ``union(_:)``
-/// - ``unionAll(_:)``
-/// - ``unionByName(_:_:)``
-/// - ``intersect(_:)``
-/// - ``intersectAll(_:)``
-/// - ``except(_:)``
-/// - ``exceptAll(_:)``
-///
-/// ### Partitioning
-/// - ``repartition(_:)``
-/// - ``repartition(_:_:)``
-/// - ``repartitionByExpression(_:_:)``
-/// - ``coalesce(_:)``
-///
-/// ### Grouping Operations
-/// - ``groupBy(_:)``
-/// - ``rollup(_:)``
-/// - ``cube(_:)``
-///
-/// ### Persistence
-/// - ``cache()``
-/// - ``persist(storageLevel:)``
-/// - ``unpersist(blocking:)``
-/// - ``storageLevel``
-///
-/// ### Schema Information
-/// - ``printSchema()``
-/// - ``printSchema(_:)``
-/// - ``explain()``
-/// - ``explain(_:)``
-///
-/// ### View Creation
-/// - ``createTempView(_:)``
-/// - ``createOrReplaceTempView(_:)``
-/// - ``createGlobalTempView(_:)``
-/// - ``createOrReplaceGlobalTempView(_:)``
-///
-/// ### Write Operations
-/// - ``write``
-/// - ``writeTo(_:)``
-/// - ``writeStream``
-///
-/// ### Sampling
-/// - ``sample(_:_:_:)``
-/// - ``sample(_:_:)``
-/// - ``sample(_:)``
-///
-/// ### Statistics
-/// - ``describe(_:)``
-/// - ``summary(_:)``
-///
-/// ### Utility Methods
-/// - ``isEmpty()``
-/// - ``isLocal()``
-/// - ``isStreaming()``
-/// - ``inputFiles()``
-/// - ``semanticHash()``
-/// - ``sameSemantics(other:)``
-///
-/// ### Internal Methods
-/// - ``rdd()``
-/// - ``getPlan()``
-public actor DataFrame: Sendable {
- var spark: SparkSession
- var plan: Plan
- private var _schema: DataType? = nil
- private var batches: [RecordBatch] = [RecordBatch]()
-
- /// Create a new `DataFrame`instance with the given Spark session and plan.
- /// - Parameters:
- /// - spark: A ``SparkSession`` instance to use.
- /// - plan: A plan to execute.
- init(spark: SparkSession, plan: Plan) {
- self.spark = spark
- self.plan = plan
- }
-
- /// Create a new `DataFrame` instance with the given SparkSession and a SQL statement.
- /// - Parameters:
- /// - spark: A `SparkSession` instance to use.
- /// - sqlText: A SQL statement.
- /// - posArgs: An array of strings.
- init(spark: SparkSession, sqlText: String, _ posArgs: [Sendable]? = nil) async throws {
- self.spark = spark
- if let posArgs {
- self.plan = sqlText.toSparkConnectPlan(posArgs)
- } else {
- self.plan = sqlText.toSparkConnectPlan
- }
- }
-
- init(spark: SparkSession, sqlText: String, _ args: [String: Sendable]) async throws {
- self.spark = spark
- self.plan = sqlText.toSparkConnectPlan(args)
- }
-
- public func getPlan() -> Sendable {
- return self.plan
- }
-
- /// Set the schema. This is used to store the analized schema response from `Spark Connect` server.
- /// - Parameter schema: <#schema description#>
- private func setSchema(_ schema: DataType) {
- self._schema = schema
- }
-
- /// Add `Apache Arrow`'s `RecordBatch`s to the internal array.
- /// - Parameter batches: An array of ``RecordBatch``.
- private func addBatches(_ batches: [RecordBatch]) {
- self.batches.append(contentsOf: batches)
- }
-
- /// Return the `SparkSession` of this `DataFrame`.
- public var sparkSession: SparkSession {
- get async throws {
- return self.spark
- }
- }
-
- /// A method to access the underlying Spark's `RDD`.
- /// In `Spark Connect`, this feature is not allowed by design.
- public func rdd() throws {
- // SQLSTATE: 0A000
- // [UNSUPPORTED_CONNECT_FEATURE.RDD]
- // Feature is not supported in Spark Connect: Resilient Distributed Datasets (RDDs).
- throw SparkConnectError.UnsupportedOperationException
- }
-
- /// Return an array of column name strings
- public var columns: [String] {
- get async throws {
- var columns: [String] = []
- try await analyzePlanIfNeeded()
- for field in self._schema!.struct.fields {
- columns.append(field.name)
- }
- return columns
- }
- }
-
- /// Return a `JSON` string of data type because we cannot expose the internal type ``DataType``.
- public var schema: String {
- get async throws {
- try await analyzePlanIfNeeded()
- return try self._schema!.jsonString()
- }
- }
-
- /// Returns all column names and their data types as an array.
- public var dtypes: [(String, String)] {
- get async throws {
- try await analyzePlanIfNeeded()
- return try self._schema!.struct.fields.map { ($0.name, try $0.dataType.simpleString) }
- }
- }
-
- private func withGPRC<Result: Sendable>(
- _ f: (GRPCClient<GRPCNIOTransportHTTP2.HTTP2ClientTransport.Posix>) async throws -> Result
- ) async throws -> Result {
- try await withGRPCClient(
- transport: .http2NIOPosix(
- target: .dns(host: spark.client.host, port: spark.client.port),
- transportSecurity: spark.client.transportSecurity
- ),
- interceptors: spark.client.getIntercepters()
- ) { client in
- return try await f(client)
- }
- }
-
- private func analyzePlanIfNeeded() async throws {
- if self._schema != nil {
- return
- }
- try await withGPRC { client in
- let service = Spark_Connect_SparkConnectService.Client(wrapping: client)
- let response = try await service.analyzePlan(
- spark.client.getAnalyzePlanRequest(spark.sessionID, plan))
- self.setSchema(response.schema.schema)
- }
- }
-
- /// Return the total number of rows.
- /// - Returns: a `Int64` value.
- @discardableResult
- public func count() async throws -> Int64 {
- let counter = Atomic(Int64(0))
-
- try await withGPRC { client in
- let service = Spark_Connect_SparkConnectService.Client(wrapping: client)
- try await service.executePlan(spark.client.getExecutePlanRequest(plan)) {
- response in
- for try await m in response.messages {
- counter.add(m.arrowBatch.rowCount, ordering: .relaxed)
- }
- }
- }
- return counter.load(ordering: .relaxed)
- }
-
- /// Execute the plan and try to fill `schema` and `batches`.
- private func execute() async throws {
- // Clear all existing batches.
- self.batches.removeAll()
-
- try await withGPRC { client in
- let service = Spark_Connect_SparkConnectService.Client(wrapping: client)
- try await service.executePlan(spark.client.getExecutePlanRequest(plan)) {
- response in
- for try await m in response.messages {
- if m.hasSchema {
- // The original schema should arrive before ArrowBatches
- await self.setSchema(m.schema)
- }
- let ipcStreamBytes = m.arrowBatch.data
- if !ipcStreamBytes.isEmpty && m.arrowBatch.rowCount > 0 {
- let IPC_CONTINUATION_TOKEN = Int32(-1)
- // Schema
- assert(ipcStreamBytes[0..<4].int32 == IPC_CONTINUATION_TOKEN)
- let schemaSize = Int64(ipcStreamBytes[4..<8].int32)
- let schema = Data(ipcStreamBytes[8..<(8 + schemaSize)])
-
- // Arrow IPC Data
- assert(
- ipcStreamBytes[(8 + schemaSize)..<(8 + schemaSize + 4)].int32
- == IPC_CONTINUATION_TOKEN)
- var pos: Int64 = 8 + schemaSize + 4
- let dataHeaderSize = Int64(ipcStreamBytes[pos..<(pos + 4)].int32)
- pos += 4
- let dataHeader = Data(ipcStreamBytes[pos..<(pos + dataHeaderSize)])
- pos += dataHeaderSize
- let dataBodySize = Int64(ipcStreamBytes.count) - pos - 8
- let dataBody = Data(ipcStreamBytes[pos..<(pos + dataBodySize)])
-
- // Read ArrowBatches
- let reader = ArrowReader()
- let arrowResult = ArrowReader.makeArrowReaderResult()
- _ = reader.fromMessage(schema, dataBody: Data(), result: arrowResult)
- _ = reader.fromMessage(dataHeader, dataBody: dataBody, result: arrowResult)
- await self.addBatches(arrowResult.batches)
- }
- }
- }
- }
- }
-
- /// Execute the plan and return the result as ``[Row]``.
- /// - Returns: ``[Row]``
- public func collect() async throws -> [Row] {
- try await execute()
-
- var result: [Row] = []
- for batch in self.batches {
- for i in 0..<batch.length {
- var values: [Sendable?] = []
- for column in batch.columns {
- if column.data.isNull(i) {
- values.append(nil)
- } else {
- let array = column.array
- switch column.data.type.info {
- case .primitiveInfo(.boolean):
- values.append(array.asAny(i) as? Bool)
- case .primitiveInfo(.int8):
- values.append(array.asAny(i) as? Int8)
- case .primitiveInfo(.int16):
- values.append(array.asAny(i) as? Int16)
- case .primitiveInfo(.int32):
- values.append(array.asAny(i) as? Int32)
- case .primitiveInfo(.int64):
- values.append(array.asAny(i) as! Int64)
- case .primitiveInfo(.float):
- values.append(array.asAny(i) as? Float)
- case .primitiveInfo(.double):
- values.append(array.asAny(i) as? Double)
- case .primitiveInfo(.date32):
- values.append(array.asAny(i) as! Date)
- case ArrowType.ArrowBinary:
- values.append((array as! AsString).asString(i).utf8)
- case .complexInfo(.strct):
- values.append((array as! AsString).asString(i))
- default:
- values.append(array.asAny(i) as? String)
- }
- }
- }
- result.append(Row(valueArray: values))
- }
- }
-
- return result
- }
-
- /// Displays the top 20 rows of ``DataFrame`` in a tabular form.
- public func show() async throws {
- try await show(20)
- }
-
- /// Displays the top 20 rows of ``DataFrame`` in a tabular form.
- /// - Parameter truncate: Whether truncate long strings. If true, strings more than 20 characters will be truncated
- /// and all cells will be aligned right
- public func show(_ truncate: Bool) async throws {
- try await show(20, truncate)
- }
-
- /// Displays the ``DataFrame`` in a tabular form.
- /// - Parameters:
- /// - numRows: Number of rows to show
- /// - truncate: Whether truncate long strings. If true, strings more than 20 characters will be truncated
- /// and all cells will be aligned right
- public func show(_ numRows: Int32 = 20, _ truncate: Bool = true) async throws {
- try await show(numRows, truncate ? 20 : 0)
- }
-
- /// Displays the ``DataFrame`` in a tabular form.
- /// - Parameters:
- /// - numRows: Number of rows to show
- /// - truncate: If set to more than 0, truncates strings to `truncate` characters and all cells will be aligned right.
- /// - vertical: If set to true, prints output rows vertically (one line per column value).
- public func show(_ numRows: Int32, _ truncate: Int32, _ vertical: Bool = false) async throws {
- let rows = try await showString(numRows, truncate, vertical).collect()
- assert(rows.count == 1)
- assert(rows[0].length == 1)
- print(try rows[0].get(0) as! String)
- }
-
- func showString(_ numRows: Int32, _ truncate: Int32, _ vertical: Bool) -> DataFrame {
- let plan = SparkConnectClient.getShowString(self.plan.root, numRows, truncate, vertical)
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Selects a subset of existing columns using column names.
- /// - Parameter cols: Column names
- /// - Returns: A ``DataFrame`` with subset of columns.
- public func select(_ cols: String...) -> DataFrame {
- return DataFrame(spark: self.spark, plan: SparkConnectClient.getProject(self.plan.root, cols))
- }
-
- /// Selects a subset of existing columns using column names.
- /// - Parameter cols: Column names
- /// - Returns: A ``DataFrame`` with subset of columns.
- public func toDF(_ cols: String...) -> DataFrame {
- let df = if cols.isEmpty {
- DataFrame(spark: self.spark, plan: self.plan)
- } else {
- DataFrame(spark: self.spark, plan: SparkConnectClient.getProject(self.plan.root, cols))
- }
- return df
- }
-
- /// Returns the content of the Dataset as a Dataset of JSON strings.
- /// - Returns: A ``DataFrame`` with a single string column whose content is JSON.
- public func toJSON() -> DataFrame {
- return selectExpr("to_json(struct(*))")
- }
-
- /// Projects a set of expressions and returns a new ``DataFrame``.
- /// - Parameter exprs: Expression strings
- /// - Returns: A ``DataFrame`` with subset of columns.
- public func selectExpr(_ exprs: String...) -> DataFrame {
- return DataFrame(spark: self.spark, plan: SparkConnectClient.getProjectExprs(self.plan.root, exprs))
- }
-
- /// Returns a new Dataset with a column dropped. This is a no-op if schema doesn't contain column name.
- /// - Parameter cols: Column names
- /// - Returns: A ``DataFrame`` with subset of columns.
- public func drop(_ cols: String...) -> DataFrame {
- return DataFrame(spark: self.spark, plan: SparkConnectClient.getDrop(self.plan.root, cols))
- }
-
- /// Returns a new ``DataFrame`` that contains only the unique rows from this ``DataFrame``.
- /// This is an alias for `distinct`. If column names are given, Spark considers only those columns.
- /// - Parameter cols: Column names
- /// - Returns: A ``DataFrame``.
- public func dropDuplicates(_ cols: String...) -> DataFrame {
- let plan = SparkConnectClient.getDropDuplicates(self.plan.root, cols, withinWatermark: false)
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Returns a new Dataset with duplicates rows removed, within watermark.
- /// If column names are given, Spark considers only those columns.
- /// - Parameter cols: Column names
- /// - Returns: A ``DataFrame``.
- public func dropDuplicatesWithinWatermark(_ cols: String...) -> DataFrame {
- let plan = SparkConnectClient.getDropDuplicates(self.plan.root, cols, withinWatermark: true)
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Computes basic statistics for numeric and string columns, including count, mean, stddev, min,
- /// and max. If no columns are given, this function computes statistics for all numerical or
- /// string columns.
- /// - Parameter cols: Column names.
- /// - Returns: A ``DataFrame`` containing basic statistics.
- public func describe(_ cols: String...) -> DataFrame {
- return DataFrame(spark: self.spark, plan: SparkConnectClient.getDescribe(self.plan.root, cols))
- }
-
- /// Computes specified statistics for numeric and string columns. Available statistics are:
- /// count, mean, stddev, min, max, arbitrary approximate percentiles specified as a percentage (e.g. 75%)
- /// count_distinct, approx_count_distinct . If no statistics are given, this function computes count, mean,
- /// stddev, min, approximate quartiles (percentiles at 25%, 50%, and 75%), and max.
- /// - Parameter statistics: Statistics names.
- /// - Returns: A ``DataFrame`` containing specified statistics.
- public func summary(_ statistics: String...) -> DataFrame {
- return DataFrame(spark: self.spark, plan: SparkConnectClient.getSummary(self.plan.root, statistics))
- }
-
- /// Returns a new Dataset with a column renamed. This is a no-op if schema doesn't contain existingName.
- /// - Parameters:
- /// - existingName: A existing column name to be renamed.
- /// - newName: A new column name.
- /// - Returns: A ``DataFrame`` with the renamed column.
- public func withColumnRenamed(_ existingName: String, _ newName: String) -> DataFrame {
- return withColumnRenamed([existingName: newName])
- }
-
- /// Returns a new Dataset with columns renamed. This is a no-op if schema doesn't contain existingName.
- /// - Parameters:
- /// - colNames: A list of existing colum names to be renamed.
- /// - newColNames: A list of new column names.
- /// - Returns: A ``DataFrame`` with the renamed columns.
- public func withColumnRenamed(_ colNames: [String], _ newColNames: [String]) -> DataFrame {
- let dic = Dictionary(uniqueKeysWithValues: zip(colNames, newColNames))
- return DataFrame(spark: self.spark, plan: SparkConnectClient.getWithColumnRenamed(self.plan.root, dic))
- }
-
- /// Returns a new Dataset with columns renamed. This is a no-op if schema doesn't contain existingName.
- /// - Parameter colsMap: A dictionary of existing column name and new column name.
- /// - Returns: A ``DataFrame`` with the renamed columns.
- public func withColumnRenamed(_ colsMap: [String: String]) -> DataFrame {
- return DataFrame(spark: self.spark, plan: SparkConnectClient.getWithColumnRenamed(self.plan.root, colsMap))
- }
-
- /// Filters rows using the given condition.
- ///
- /// The condition should be a SQL expression that evaluates to a boolean value.
- ///
- /// ```swift
- /// // Filter with simple condition
- /// let adults = df.filter("age >= 18")
- ///
- /// // Filter with complex condition
- /// let qualifiedUsers = df.filter("age >= 21 AND department = 'Engineering'")
- ///
- /// // Filter with SQL functions
- /// let recent = df.filter("date_diff(current_date(), join_date) < 30")
- /// ```
- ///
- /// - Parameter conditionExpr: A SQL expression string for filtering
- /// - Returns: A new DataFrame containing only rows that match the condition
- public func filter(_ conditionExpr: String) -> DataFrame {
- return DataFrame(spark: self.spark, plan: SparkConnectClient.getFilter(self.plan.root, conditionExpr))
- }
-
- /// Filters rows using the given condition (alias for filter).
- ///
- /// This method is an alias for ``filter(_:)`` and behaves identically.
- ///
- /// ```swift
- /// let highSalary = df.where("salary > 100000")
- /// ```
- ///
- /// - Parameter conditionExpr: A SQL expression string for filtering
- /// - Returns: A new DataFrame containing only rows that match the condition
- public func `where`(_ conditionExpr: String) -> DataFrame {
- return filter(conditionExpr)
- }
-
- /// Returns a new DataFrame sorted by the specified columns.
- ///
- /// By default, sorts in ascending order. Use `desc("column")` for descending order.
- ///
- /// ```swift
- /// // Sort by single column (ascending)
- /// let sorted = df.sort("age")
- ///
- /// // Sort by multiple columns
- /// let multiSort = df.sort("department", "salary")
- ///
- /// // Sort with mixed order
- /// let mixedSort = df.sort("department", "desc(salary)")
- /// ```
- ///
- /// - Parameter cols: Column names or expressions to sort by
- /// - Returns: A new DataFrame sorted by the specified columns
- public func sort(_ cols: String...) -> DataFrame {
- return DataFrame(spark: self.spark, plan: SparkConnectClient.getSort(self.plan.root, cols))
- }
-
- /// Return a new ``DataFrame`` sorted by the specified column(s).
- /// - Parameter cols: Column names.
- /// - Returns: A sorted ``DataFrame``
- public func orderBy(_ cols: String...) -> DataFrame {
- return DataFrame(spark: self.spark, plan: SparkConnectClient.getSort(self.plan.root, cols))
- }
-
- /// Limits the result count to the number specified.
- ///
- /// This transformation is often used for:
- /// - Previewing data
- /// - Reducing data size for testing
- /// - Implementing pagination
- ///
- /// ```swift
- /// // Get top 10 records
- /// let top10 = df.limit(10)
- ///
- /// // Preview data
- /// let preview = df.filter("status = 'active'").limit(100)
- /// ```
- ///
- /// - Parameter n: Maximum number of rows to return
- /// - Returns: A new DataFrame with at most n rows
- public func limit(_ n: Int32) -> DataFrame {
- return DataFrame(spark: self.spark, plan: SparkConnectClient.getLimit(self.plan.root, n))
- }
-
- /// Returns a new Dataset by skipping the first `n` rows.
- /// - Parameter n: Number of rows to skip.
- /// - Returns: A subset of the rows
- public func offset(_ n: Int32) -> DataFrame {
- return DataFrame(spark: self.spark, plan: SparkConnectClient.getOffset(self.plan.root, n))
- }
-
- /// Returns a new ``Dataset`` by sampling a fraction of rows, using a user-supplied seed.
- /// - Parameters:
- /// - withReplacement: Sample with replacement or not.
- /// - fraction: Fraction of rows to generate, range [0.0, 1.0].
- /// - seed: Seed for sampling.
- /// - Returns: A subset of the records.
- public func sample(_ withReplacement: Bool, _ fraction: Double, _ seed: Int64) -> DataFrame {
- return DataFrame(spark: self.spark, plan: SparkConnectClient.getSample(self.plan.root, withReplacement, fraction, seed))
- }
-
- /// Returns a new ``Dataset`` by sampling a fraction of rows, using a random seed.
- /// - Parameters:
- /// - withReplacement: Sample with replacement or not.
- /// - fraction: Fraction of rows to generate, range [0.0, 1.0].
- /// - Returns: A subset of the records.
- public func sample(_ withReplacement: Bool, _ fraction: Double) -> DataFrame {
- return sample(withReplacement, fraction, Int64.random(in: Int64.min...Int64.max))
- }
-
- /// Returns a new ``Dataset`` by sampling a fraction of rows (without replacement), using a
- /// user-supplied seed.
- /// - Parameters:
- /// - fraction: Fraction of rows to generate, range [0.0, 1.0].
- /// - seed: Seed for sampling.
- /// - Returns: A subset of the records.
- public func sample(_ fraction: Double, _ seed: Int64) -> DataFrame {
- return sample(false, fraction, seed)
- }
-
- /// Returns a new ``Dataset`` by sampling a fraction of rows (without replacement), using a
- /// random seed.
- /// - Parameters:
- /// - fraction: Fraction of rows to generate, range [0.0, 1.0].
- /// - Returns: A subset of the records.
- public func sample(_ fraction: Double) -> DataFrame {
- return sample(false, fraction)
- }
-
- /// Returns the first n rows.
- ///
- /// This method is useful for quickly examining the contents of a DataFrame.
- ///
- /// ```swift
- /// // Get the first row
- /// let firstRow = try await df.head()
- ///
- /// // Get the first 5 rows
- /// let firstFive = try await df.head(5)
- /// ```
- ///
- /// - Parameter n: Number of rows to return.
- /// - Returns: An array of ``Row`` objects
- /// - Throws: `SparkConnectError` if the operation fails
- public func head(_ n: Int32) async throws -> [Row] {
- return try await limit(n).collect()
- }
-
- /// Returns the first row.
- /// - Returns: A ``Row``.
- public func head() async throws -> Row {
- return try await head(1)[0]
- }
-
- /// Returns the first row. Alias for head().
- /// - Returns: A ``Row``.
- public func first() async throws -> Row {
- return try await head()
- }
-
- /// Returns the first n rows.
- /// - Parameter n: Number of rows to return.
- /// - Returns: An array of ``Row`` objects
- /// - Throws: `SparkConnectError` if the operation fails
- public func take(_ n: Int32) async throws -> [Row] {
- return try await head(n)
- }
-
- /// Returns the last `n` rows.
- /// - Parameter n: The number of rows.
- /// - Returns: ``[Row]``
- public func tail(_ n: Int32) async throws -> [Row] {
- let lastN = DataFrame(spark:spark, plan: SparkConnectClient.getTail(self.plan.root, n))
- return try await lastN.collect()
- }
-
- /// Returns true if the `collect` and `take` methods can be run locally
- /// (without any Spark executors).
- /// - Returns: True if the plan is local.
- public func isLocal() async throws -> Bool {
- try await withGPRC { client in
- let service = Spark_Connect_SparkConnectService.Client(wrapping: client)
- let response = try await service.analyzePlan(spark.client.getIsLocal(spark.sessionID, plan))
- return response.isLocal.isLocal
- }
- }
-
- /// Returns true if this `DataFrame` contains one or more sources that continuously return data as it
- /// arrives.
- /// - Returns: True if a plan is streaming.
- public func isStreaming() async throws -> Bool {
- try await withGPRC { client in
- let service = Spark_Connect_SparkConnectService.Client(wrapping: client)
- let response = try await service.analyzePlan(spark.client.getIsStreaming(spark.sessionID, plan))
- return response.isStreaming.isStreaming
- }
- }
-
- /// Checks if the ``DataFrame`` is empty and returns a boolean value.
- /// - Returns: `true` if the ``DataFrame`` is empty, `false` otherwise.
- public func isEmpty() async throws -> Bool {
- return try await select().limit(1).count() == 0
- }
-
- /// Persists this DataFrame with the default storage level (MEMORY_AND_DISK).
- ///
- /// Caching can significantly improve performance when a DataFrame is accessed multiple times.
- /// The cached data is stored in memory and/or disk depending on the storage level.
- ///
- /// ```swift
- /// // Cache a frequently used DataFrame
- /// let cachedDf = try await df.cache()
- ///
- /// // Use the cached DataFrame multiple times
- /// let count1 = try await cachedDf.count()
- /// let count2 = try await cachedDf.filter("age > 30").count()
- /// ```
- ///
- /// - Returns: The cached DataFrame
- /// - Throws: `SparkConnectError` if the operation fails
- public func cache() async throws -> DataFrame {
- return try await persist()
- }
-
- /// Persist this `DataFrame` with the given storage level.
- /// - Parameter storageLevel: A storage level to apply.
- @discardableResult
- public func persist(storageLevel: StorageLevel = StorageLevel.MEMORY_AND_DISK) async throws
- -> DataFrame
- {
- try await withGPRC { client in
- let service = Spark_Connect_SparkConnectService.Client(wrapping: client)
- _ = try await service.analyzePlan(
- spark.client.getPersist(spark.sessionID, plan, storageLevel))
- }
-
- return self
- }
-
- /// Mark the `DataFrame` as non-persistent, and remove all blocks for it from memory and disk.
- /// This will not un-persist any cached data that is built upon this `DataFrame`.
- /// - Parameter blocking: Whether to block until all blocks are deleted.
- /// - Returns: A `DataFrame`
- @discardableResult
- public func unpersist(blocking: Bool = false) async throws -> DataFrame {
- try await withGPRC { client in
- let service = Spark_Connect_SparkConnectService.Client(wrapping: client)
- _ = try await service.analyzePlan(spark.client.getUnpersist(spark.sessionID, plan, blocking))
- }
-
- return self
- }
-
- public var storageLevel: StorageLevel {
- get async throws {
- try await withGPRC { client in
- let service = Spark_Connect_SparkConnectService.Client(wrapping: client)
- return try await service
- .analyzePlan(spark.client.getStorageLevel(spark.sessionID, plan)).getStorageLevel.storageLevel.toStorageLevel
- }
- }
- }
-
- /// Returns a `hashCode` of the logical query plan against this ``DataFrame``.
- /// - Returns: A hashcode value.
- public func semanticHash() async throws -> Int32 {
- return try await self.spark.semanticHash(self.plan)
- }
-
- /// Returns `true` when the logical query plans inside both ``Dataset``s are equal and therefore
- /// return same results.
- /// - Parameter other: A ``DataFrame`` to compare.
- /// - Returns: Whether the both logical plans are equal.
- public func sameSemantics(other: DataFrame) async throws -> Bool {
- return try await self.spark.sameSemantics(self.plan, other.getPlan() as! Plan)
- }
-
- /// Prints the physical plan to the console for debugging purposes.
- public func explain() async throws {
- try await explain("simple")
- }
-
- /// Prints the plans (logical and physical) to the console for debugging purposes.
- /// - Parameter extended: If `false`, prints only the physical plan.
- public func explain(_ extended: Bool) async throws {
- if (extended) {
- try await explain("extended")
- } else {
- try await explain("simple")
- }
- }
-
- /// Prints the plans (logical and physical) with a format specified by a given explain mode.
- /// - Parameter mode: the expected output format of plans;
- /// `simple`, `extended`, `codegen`, `cost`, `formatted`.
- public func explain(_ mode: String) async throws {
- try await withGPRC { client in
- let service = Spark_Connect_SparkConnectService.Client(wrapping: client)
- let response = try await service.analyzePlan(spark.client.getExplain(spark.sessionID, plan, mode))
- print(response.explain.explainString)
- }
- }
-
- /// Returns a best-effort snapshot of the files that compose this Dataset. This method simply
- /// asks each constituent BaseRelation for its respective files and takes the union of all
- /// results. Depending on the source relations, this may not find all input files. Duplicates are removed.
- /// - Returns: An array of file path strings.
- public func inputFiles() async throws -> [String] {
- try await withGPRC { client in
- let service = Spark_Connect_SparkConnectService.Client(wrapping: client)
- let response = try await service.analyzePlan(spark.client.getInputFiles(spark.sessionID, plan))
- return response.inputFiles.files
- }
- }
-
- /// Prints the schema to the console in a nice tree format.
- public func printSchema() async throws {
- try await printSchema(Int32.max)
- }
-
- /// Prints the schema up to the given level to the console in a nice tree format.
- /// - Parameter level: A level to be printed.
- public func printSchema(_ level: Int32) async throws {
- try await withGPRC { client in
- let service = Spark_Connect_SparkConnectService.Client(wrapping: client)
- let response = try await service.analyzePlan(spark.client.getTreeString(spark.sessionID, plan, level))
- print(response.treeString.treeString)
- }
- }
-
- /// Join with another DataFrame.
- ///
- /// This performs an inner join and requires a subsequent join predicate.
- /// For other join types, use the overloaded methods with join type parameter.
- ///
- /// ```swift
- /// // Basic join (requires join condition)
- /// let joined = df1.join(df2)
- /// .where("df1.id = df2.user_id")
- ///
- /// // Join with condition
- /// let result = users.join(orders, "id")
- /// ```
- ///
- /// - Parameter right: The DataFrame to join with
- /// - Returns: A new DataFrame representing the join result
- public func join(_ right: DataFrame) async -> DataFrame {
- let right = await (right.getPlan() as! Plan).root
- let plan = SparkConnectClient.getJoin(self.plan.root, right, JoinType.inner)
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Equi-join with another DataFrame using the given column.
- ///
- /// This method performs an equi-join on a single column that exists in both DataFrames.
- ///
- /// ```swift
- /// // Inner join on a single column
- /// let joined = users.join(orders, "user_id")
- ///
- /// // Left outer join
- /// let leftJoined = users.join(orders, "user_id", "left")
- ///
- /// // Other join types: "inner", "outer", "left", "right", "semi", "anti"
- /// ```
- ///
- /// - Parameters:
- /// - right: The DataFrame to join with
- /// - usingColumn: Column name that exists in both DataFrames
- /// - joinType: Type of join (default: "inner")
- /// - Returns: A new DataFrame with the join result
- public func join(_ right: DataFrame, _ usingColumn: String, _ joinType: String = "inner") async -> DataFrame {
- await join(right, [usingColumn], joinType)
- }
-
- /// Inner equi-join with another `DataFrame` using the given columns.
- /// - Parameters:
- /// - right: Right side of the join operation.
- /// - usingColumn: Names of the columns to join on. These columns must exist on both sides.
- /// - joinType: A join type name.
- /// - Returns: A `DataFrame`.
- public func join(_ other: DataFrame, _ usingColumns: [String], _ joinType: String = "inner") async -> DataFrame {
- let right = await (other.getPlan() as! Plan).root
- let plan = SparkConnectClient.getJoin(
- self.plan.root,
- right,
- joinType.toJoinType,
- usingColumns: usingColumns
- )
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Inner equi-join with another `DataFrame` using the given columns.
- /// - Parameters:
- /// - right: Right side of the join operation.
- /// - joinExprs:A join expression string.
- /// - Returns: A `DataFrame`.
- public func join(_ right: DataFrame, joinExprs: String) async -> DataFrame {
- return await join(right, joinExprs: joinExprs, joinType: "inner")
- }
-
- /// Inner equi-join with another `DataFrame` using the given columns.
- /// - Parameters:
- /// - right: Right side of the join operation.
- /// - joinExprs:A join expression string.
- /// - joinType: A join type name.
- /// - Returns: A `DataFrame`.
- public func join(_ right: DataFrame, joinExprs: String, joinType: String) async -> DataFrame {
- let rightPlan = await (right.getPlan() as! Plan).root
- let plan = SparkConnectClient.getJoin(
- self.plan.root,
- rightPlan,
- joinType.toJoinType,
- joinCondition: joinExprs
- )
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Explicit cartesian join with another `DataFrame`.
- /// - Parameter right: Right side of the join operation.
- /// - Returns: A `DataFrame`.
- public func crossJoin(_ right: DataFrame) async -> DataFrame {
- let rightPlan = await (right.getPlan() as! Plan).root
- let plan = SparkConnectClient.getJoin(self.plan.root, rightPlan, JoinType.cross)
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Lateral join with another ``DataFrame``.
- ///
- /// Behaves as an JOIN LATERAL.
- ///
- /// - Parameters:
- /// - right: Right side of the join operation.
- /// - Returns: A ``DataFrame``.
- public func lateralJoin(_ right: DataFrame) async -> DataFrame {
- let rightPlan = await (right.getPlan() as! Plan).root
- let plan = SparkConnectClient.getLateralJoin(
- self.plan.root,
- rightPlan,
- JoinType.inner
- )
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Lateral join with another ``DataFrame``.
- ///
- /// Behaves as an JOIN LATERAL.
- ///
- /// - Parameters:
- /// - right: Right side of the join operation.
- /// - joinType: One of `inner` (default), `cross`, `left`, `leftouter`, `left_outer`.
- /// - Returns: A ``DataFrame``.
- public func lateralJoin(_ right: DataFrame, joinType: String) async -> DataFrame {
- let rightPlan = await (right.getPlan() as! Plan).root
- let plan = SparkConnectClient.getLateralJoin(
- self.plan.root,
- rightPlan,
- joinType.toJoinType
- )
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Lateral join with another ``DataFrame``.
- ///
- /// Behaves as an JOIN LATERAL.
- ///
- /// - Parameters:
- /// - right: Right side of the join operation.
- /// - joinExprs: A join expression string.
- /// - Returns: A ``DataFrame``.
- public func lateralJoin(_ right: DataFrame, joinExprs: String) async -> DataFrame {
- let rightPlan = await (right.getPlan() as! Plan).root
- let plan = SparkConnectClient.getLateralJoin(
- self.plan.root,
- rightPlan,
- JoinType.inner,
- joinCondition: joinExprs
- )
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Lateral join with another ``DataFrame``.
- ///
- /// Behaves as an JOIN LATERAL.
- ///
- /// - Parameters:
- /// - right: Right side of the join operation.
- /// - joinType: One of `inner` (default), `cross`, `left`, `leftouter`, `left_outer`.
- /// - joinExprs: A join expression string.
- /// - Returns: A ``DataFrame``.
- public func lateralJoin(
- _ right: DataFrame, joinExprs: String, joinType: String = "inner"
- ) async -> DataFrame {
- let rightPlan = await (right.getPlan() as! Plan).root
- let plan = SparkConnectClient.getLateralJoin(
- self.plan.root,
- rightPlan,
- joinType.toJoinType,
- joinCondition: joinExprs
- )
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Returns a new `DataFrame` containing rows in this `DataFrame` but not in another `DataFrame`.
- /// This is equivalent to `EXCEPT DISTINCT` in SQL.
- /// - Parameter other: A `DataFrame` to exclude.
- /// - Returns: A `DataFrame`.
- public func except(_ other: DataFrame) async -> DataFrame {
- let right = await (other.getPlan() as! Plan).root
- let plan = SparkConnectClient.getSetOperation(self.plan.root, right, SetOpType.except)
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Returns a new `DataFrame` containing rows in this `DataFrame` but not in another `DataFrame` while
- /// preserving the duplicates. This is equivalent to `EXCEPT ALL` in SQL.
- /// - Parameter other: A `DataFrame` to exclude.
- /// - Returns: A `DataFrame`.
- public func exceptAll(_ other: DataFrame) async -> DataFrame {
- let right = await (other.getPlan() as! Plan).root
- let plan = SparkConnectClient.getSetOperation(self.plan.root, right, SetOpType.except, isAll: true)
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Returns a new `DataFrame` containing rows only in both this `DataFrame` and another `DataFrame`.
- /// This is equivalent to `INTERSECT` in SQL.
- /// - Parameter other: A `DataFrame` to intersect with.
- /// - Returns: A `DataFrame`.
- public func intersect(_ other: DataFrame) async -> DataFrame {
- let right = await (other.getPlan() as! Plan).root
- let plan = SparkConnectClient.getSetOperation(self.plan.root, right, SetOpType.intersect)
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Returns a new `DataFrame` containing rows only in both this `DataFrame` and another `DataFrame` while
- /// preserving the duplicates. This is equivalent to `INTERSECT ALL` in SQL.
- /// - Parameter other: A `DataFrame` to intersect with.
- /// - Returns: A `DataFrame`.
- public func intersectAll(_ other: DataFrame) async -> DataFrame {
- let right = await (other.getPlan() as! Plan).root
- let plan = SparkConnectClient.getSetOperation(self.plan.root, right, SetOpType.intersect, isAll: true)
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Returns a new `DataFrame` containing union of rows in this `DataFrame` and another `DataFrame`.
- /// This is equivalent to `UNION ALL` in SQL. To do a SQL-style set union (that does
- /// deduplication of elements), use this function followed by a [[distinct]].
- /// Also as standard in SQL, this function resolves columns by position (not by name)
- /// - Parameter other: A `DataFrame` to union with.
- /// - Returns: A `DataFrame`.
- public func union(_ other: DataFrame) async -> DataFrame {
- let right = await (other.getPlan() as! Plan).root
- let plan = SparkConnectClient.getSetOperation(self.plan.root, right, SetOpType.union, isAll: true)
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Returns a new `DataFrame` containing union of rows in this `DataFrame` and another `DataFrame`.
- /// This is an alias of `union`.
- /// - Parameter other: A `DataFrame` to union with.
- /// - Returns: A `DataFrame`.
- public func unionAll(_ other: DataFrame) async -> DataFrame {
- return await union(other)
- }
-
- /// Returns a new `DataFrame` containing union of rows in this `DataFrame` and another `DataFrame`.
- /// The difference between this function and [[union]] is that this function resolves columns by
- /// name (not by position).
- /// When the parameter `allowMissingColumns` is `true`, the set of column names in this and other
- /// `DataFrame` can differ; missing columns will be filled with null. Further, the missing columns
- /// of this `DataFrame` will be added at the end in the schema of the union result
- /// - Parameter other: A `DataFrame` to union with.
- /// - Returns: A `DataFrame`.
- public func unionByName(_ other: DataFrame, _ allowMissingColumns: Bool = false) async -> DataFrame {
- let right = await (other.getPlan() as! Plan).root
- let plan = SparkConnectClient.getSetOperation(
- self.plan.root,
- right,
- SetOpType.union,
- isAll: true,
- byName: true,
- allowMissingColumns: allowMissingColumns
- )
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- private func buildRepartition(numPartitions: Int32, shuffle: Bool) -> DataFrame {
- let plan = SparkConnectClient.getRepartition(self.plan.root, numPartitions, shuffle)
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- private func buildRepartitionByExpression(numPartitions: Int32?, partitionExprs: [String]) -> DataFrame {
- let plan = SparkConnectClient.getRepartitionByExpression(self.plan.root, partitionExprs, numPartitions)
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Returns a new ``DataFrame`` that has exactly `numPartitions` partitions.
- /// - Parameter numPartitions: The number of partitions.
- /// - Returns: A `DataFrame`.
- public func repartition(_ numPartitions: Int32) -> DataFrame {
- return buildRepartition(numPartitions: numPartitions, shuffle: true)
- }
-
- /// Returns a new ``DataFrame`` partitioned by the given partitioning expressions, using
- /// `spark.sql.shuffle.partitions` as number of partitions. The resulting Dataset is hash
- /// partitioned.
- /// - Parameter partitionExprs: The partition expression strings.
- /// - Returns: A `DataFrame`.
- public func repartition(_ partitionExprs: String...) -> DataFrame {
- return buildRepartitionByExpression(numPartitions: nil, partitionExprs: partitionExprs)
- }
-
- /// Returns a new ``DataFrame`` partitioned by the given partitioning expressions, using
- /// `spark.sql.shuffle.partitions` as number of partitions. The resulting Dataset is hash
- /// partitioned.
- /// - Parameters:
- /// - numPartitions: The number of partitions.
- /// - partitionExprs: The partition expression strings.
- /// - Returns: A `DataFrame`.
- public func repartition(_ numPartitions: Int32, _ partitionExprs: String...) -> DataFrame {
- return buildRepartitionByExpression(numPartitions: numPartitions, partitionExprs: partitionExprs)
- }
-
- /// Returns a new ``DataFrame`` partitioned by the given partitioning expressions, using
- /// `spark.sql.shuffle.partitions` as number of partitions. The resulting Dataset is hash
- /// partitioned.
- /// - Parameter partitionExprs: The partition expression strings.
- /// - Returns: A `DataFrame`.
- public func repartitionByExpression(_ numPartitions: Int32?, _ partitionExprs: String...) -> DataFrame {
- return buildRepartitionByExpression(numPartitions: numPartitions, partitionExprs: partitionExprs)
- }
-
- /// Returns a new ``DataFrame`` that has exactly `numPartitions` partitions, when the fewer partitions
- /// are requested. If a larger number of partitions is requested, it will stay at the current
- /// number of partitions. Similar to coalesce defined on an `RDD`, this operation results in a
- /// narrow dependency, e.g. if you go from 1000 partitions to 100 partitions, there will not be a
- /// shuffle, instead each of the 100 new partitions will claim 10 of the current partitions.
- /// - Parameter numPartitions: The number of partitions.
- /// - Returns: A `DataFrame`.
- public func coalesce(_ numPartitions: Int32) -> DataFrame {
- return buildRepartition(numPartitions: numPartitions, shuffle: false)
- }
-
- /// Returns a new ``Dataset`` that contains only the unique rows from this ``Dataset``.
- /// This is an alias for `dropDuplicates`.
- /// - Returns: A `DataFrame`.
- public func distinct() -> DataFrame {
- return dropDuplicates()
- }
-
- /// Transposes a DataFrame, switching rows to columns. This function transforms the DataFrame
- /// such that the values in the first column become the new columns of the DataFrame.
- /// - Returns: A transposed ``DataFrame``.
- public func transpose() -> DataFrame {
- return buildTranspose([])
- }
-
- /// Unpivot a DataFrame from wide format to long format, optionally leaving identifier columns
- /// set. This is the reverse to `groupBy(...).pivot(...).agg(...)`, except for the aggregation,
- /// which cannot be reversed. This is an alias for `unpivot`.
- /// - Parameters:
- /// - ids: ID column names
- /// - values: Value column names to unpivot
- /// - variableColumnName: Name of the variable column
- /// - valueColumnName: Name of the value column
- /// - Returns: A ``DataFrame``.
- public func melt(
- _ ids: [String],
- _ values: [String],
- _ variableColumnName: String,
- _ valueColumnName: String
- ) -> DataFrame {
- return unpivot(ids, values, variableColumnName, valueColumnName)
- }
-
- /// Unpivot a DataFrame from wide format to long format, optionally leaving identifier columns
- /// set. This is the reverse to `groupBy(...).pivot(...).agg(...)`, except for the aggregation,
- /// which cannot be reversed. This is an alias for `unpivot`.
- /// - Parameters:
- /// - ids: ID column names
- /// - variableColumnName: Name of the variable column
- /// - valueColumnName: Name of the value column
- /// - Returns: A ``DataFrame``.
- public func melt(
- _ ids: [String],
- _ variableColumnName: String,
- _ valueColumnName: String
- ) -> DataFrame {
- return unpivot(ids, variableColumnName, valueColumnName)
- }
-
- /// Unpivot a DataFrame from wide format to long format, optionally leaving identifier columns
- /// set. This is the reverse to `groupBy(...).pivot(...).agg(...)`, except for the aggregation,
- /// which cannot be reversed.
- /// - Parameters:
- /// - ids: ID column names
- /// - values: Value column names to unpivot
- /// - variableColumnName: Name of the variable column
- /// - valueColumnName: Name of the value column
- /// - Returns: A ``DataFrame``.
- public func unpivot(
- _ ids: [String],
- _ values: [String],
- _ variableColumnName: String,
- _ valueColumnName: String
- ) -> DataFrame {
- return buildUnpivot(ids, values, variableColumnName, valueColumnName)
- }
-
- /// Unpivot a DataFrame from wide format to long format, optionally leaving identifier columns
- /// set. This is the reverse to `groupBy(...).pivot(...).agg(...)`, except for the aggregation,
- /// which cannot be reversed.
- /// - Parameters:
- /// - ids: ID column names
- /// - variableColumnName: Name of the variable column
- /// - valueColumnName: Name of the value column
- /// - Returns: A ``DataFrame``.
- public func unpivot(
- _ ids: [String],
- _ variableColumnName: String,
- _ valueColumnName: String
- ) -> DataFrame {
- return buildUnpivot(ids, nil, variableColumnName, valueColumnName)
- }
-
- func buildUnpivot(
- _ ids: [String],
- _ values: [String]?,
- _ variableColumnName: String,
- _ valueColumnName: String
- ) -> DataFrame {
- let plan = SparkConnectClient.getUnpivot(self.plan.root, ids, values, variableColumnName, valueColumnName)
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Transposes a ``DataFrame`` such that the values in the specified index column become the new
- /// columns of the ``DataFrame``.
- /// - Parameter indexColumn: The single column that will be treated as the index for the transpose operation.
- /// This column will be used to pivot the data, transforming the DataFrame such that the values of
- /// the indexColumn become the new columns in the transposed DataFrame.
- /// - Returns: A transposed ``DataFrame``.
- public func transpose(_ indexColumn: String) -> DataFrame {
- return buildTranspose([indexColumn])
- }
-
- func buildTranspose(_ indexColumn: [String]) -> DataFrame {
- let plan = SparkConnectClient.getTranspose(self.plan.root, indexColumn)
- return DataFrame(spark: self.spark, plan: plan)
- }
-
- /// Groups the DataFrame using the specified columns.
- ///
- /// This method is used to perform aggregations on groups of data.
- /// After grouping, you can apply aggregation functions like count(), sum(), avg(), etc.
- ///
- /// ```swift
- /// // Group by single column
- /// let byDept = df.groupBy("department")
- /// .agg(count("*").alias("employee_count"))
- ///
- /// // Group by multiple columns
- /// let byDeptAndLocation = df.groupBy("department", "location")
- /// .agg(
- /// avg("salary").alias("avg_salary"),
- /// max("salary").alias("max_salary")
- /// )
- /// ```
- ///
- /// - Parameter cols: Column names to group by
- /// - Returns: A ``GroupedData`` object for aggregation operations
- public func groupBy(_ cols: String...) -> GroupedData {
- return GroupedData(self, GroupType.groupby, cols)
- }
-
- /// Create a multi-dimensional rollup for the current ``DataFrame`` using the specified columns, so we
- /// can run aggregation on them.
- /// - Parameter cols: Grouping column names.
- /// - Returns: A ``GroupedData``.
- public func rollup(_ cols: String...) -> GroupedData {
- return GroupedData(self, GroupType.rollup, cols)
- }
-
- /// Create a multi-dimensional cube for the current ``DataFrame`` using the specified columns, so we
- /// can run aggregation on them.
- /// - Parameter cols: Grouping column names.
- /// - Returns: A ``GroupedData``.
- public func cube(_ cols: String...) -> GroupedData {
- return GroupedData(self, GroupType.cube, cols)
- }
-
- /// Creates a local temporary view using the given name. The lifetime of this temporary view is
- /// tied to the `SparkSession` that was used to create this ``DataFrame``.
- /// - Parameter viewName: A view name.
- public func createTempView(_ viewName: String) async throws {
- try await createTempView(viewName, replace: false, global: false)
- }
-
- /// Creates a local temporary view using the given name. The lifetime of this temporary view is
- /// tied to the `SparkSession` that was used to create this ``DataFrame``.
- /// - Parameter viewName: A view name.
- public func createOrReplaceTempView(_ viewName: String) async throws {
- try await createTempView(viewName, replace: true, global: false)
- }
-
- /// Creates a global temporary view using the given name. The lifetime of this temporary view is
- /// tied to this Spark application, but is cross-session.
- /// - Parameter viewName: A view name.
- public func createGlobalTempView(_ viewName: String) async throws {
- try await createTempView(viewName, replace: false, global: true)
- }
-
- /// Creates a global temporary view using the given name. The lifetime of this temporary view is
- /// tied to this Spark application, but is cross-session.
- /// - Parameter viewName: A view name.
- public func createOrReplaceGlobalTempView(_ viewName: String) async throws {
- try await createTempView(viewName, replace: true, global: true)
- }
-
- func createTempView(_ viewName: String, replace: Bool, global: Bool) async throws {
- try await spark.client.createTempView(self.plan.root, viewName, replace: replace, isGlobal: global)
- }
-
- /// Returns a ``DataFrameWriter`` that can be used to write non-streaming data.
- public var write: DataFrameWriter {
- get {
- DataFrameWriter(df: self)
- }
- }
-
- /// Create a write configuration builder for v2 sources.
- /// - Parameter table: A table name, e.g., `catalog.db.table`.
- /// - Returns: A ``DataFrameWriterV2`` instance.
- public func writeTo(_ table: String) -> DataFrameWriterV2 {
- return DataFrameWriterV2(table, self)
- }
-
- /// Returns a ``DataStreamWriter`` that can be used to write streaming data.
- public var writeStream: DataStreamWriter {
- get {
- DataStreamWriter(df: self)
- }
- }
-}
diff --git a/Sources/SparkConnect/DataFrameReader.swift b/Sources/SparkConnect/DataFrameReader.swift
deleted file mode 100644
index 58567db..0000000
--- a/Sources/SparkConnect/DataFrameReader.swift
+++ /dev/null
@@ -1,278 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-import Foundation
-
-/// An interface used to load a ``DataFrame`` from external storage systems
-/// (e.g. file systems, key-value stores, etc). Use `SparkSession.read` to access this.
-public actor DataFrameReader: Sendable {
- var source: String = ""
-
- var paths: [String] = []
-
- var extraOptions: CaseInsensitiveDictionary = CaseInsensitiveDictionary([:])
-
- var userSpecifiedSchemaDDL: String? = nil
-
- let sparkSession: SparkSession
-
- init(sparkSession: SparkSession) {
- self.sparkSession = sparkSession
- }
-
- /// Returns the specified table/view as a ``DataFrame``. If it's a table, it must support batch
- /// reading and the returned ``DataFrame`` is the batch scan query plan of this table. If it's a
- /// view, the returned ``DataFrame`` is simply the query plan of the view, which can either be a
- /// batch or streaming query plan.
- ///
- /// - Parameter tableName: a qualified or unqualified name that designates a table or view. If a database is
- /// specified, it identifies the table/view from the database. Otherwise, it first attempts to
- /// find a temporary view with the given name and then match the table/view from the current
- /// database. Note that, the global temporary view database is also valid here.
- /// - Returns: A ``DataFrame`` instance.
- public func table(_ tableName: String) -> DataFrame {
- var namedTable = NamedTable()
- namedTable.unparsedIdentifier = tableName
- namedTable.options = self.extraOptions.toStringDictionary()
-
- var read = Read()
- read.namedTable = namedTable
-
- var relation = Relation()
- relation.read = read
-
- var plan = Plan()
- plan.opType = .root(relation)
-
- return DataFrame(spark: sparkSession, plan: plan)
- }
-
- /// Specifies the input data source format.
- /// - Parameter source: A string.
- /// - Returns: A ``DataFrameReader``.
- public func format(_ source: String) -> DataFrameReader {
- self.source = source
- return self
- }
-
- /// Adds an input option for the underlying data source.
- /// - Parameters:
- /// - key: A key string.
- /// - value: A value string.
- /// - Returns: A ``DataFrameReader``.
- public func option(_ key: String, _ value: String) -> DataFrameReader {
- self.extraOptions[key] = value
- return self
- }
-
- /// Adds an input option for the underlying data source.
- /// - Parameters:
- /// - key: A key string.
- /// - value: A bool value.
- /// - Returns: A ``DataFrameReader``.
- public func option(_ key: String, _ value: Bool) -> DataFrameReader {
- return option(key, String(value))
- }
-
- /// Adds an input option for the underlying data source.
- /// - Parameters:
- /// - key: A key string.
- /// - value: A `Int` value.
- /// - Returns: A ``DataFrameReader``.
- public func option(_ key: String, _ value: Int) -> DataFrameReader {
- return option(key, String(value))
- }
-
- /// Adds an input option for the underlying data source.
- /// - Parameters:
- /// - key: A key string.
- /// - value: A `Int64` value.
- /// - Returns: A ``DataFrameReader``.
- public func option(_ key: String, _ value: Int64) -> DataFrameReader {
- return option(key, String(value))
- }
-
- /// Adds an input option for the underlying data source.
- /// - Parameters:
- /// - key: A key string.
- /// - value: A `Double` value.
- /// - Returns: A ``DataFrameReader``.
- public func option(_ key: String, _ value: Double) -> DataFrameReader {
- return option(key, String(value))
- }
-
- /// Specifies the input schema. Some data sources (e.g. JSON) can infer the input schema
- /// automatically from data. By specifying the schema here, the underlying data source can skip
- /// the schema inference step, and thus speed up data loading.
- /// - Parameter schema: A DDL schema string.
- /// - Returns: A ``DataFrameReader``.
- @discardableResult
- public func schema(_ schema: String) async throws -> DataFrameReader {
- // Validate by parsing.
- do {
- try await sparkSession.client.ddlParse(schema)
- } catch {
- throw SparkConnectError.InvalidTypeException
- }
- self.userSpecifiedSchemaDDL = schema
- return self
- }
-
- /// Loads input in as a ``DataFrame``, for data sources that don't require a path (e.g. external
- /// key-value stores).
- /// - Returns: A ``DataFrame``.
- public func load() -> DataFrame {
- return load([])
- }
-
- /// Loads input in as a ``DataFrame``, for data sources that require a path (e.g. data backed by a
- /// local or distributed file system).
- /// - Parameter path: A path string.
- /// - Returns: A ``DataFrame``.
- public func load(_ path: String) -> DataFrame {
- return load([path])
- }
-
- /// Loads input in as a ``DataFrame``, for data sources that support multiple paths. Only works if
- /// the source is a HadoopFsRelationProvider.
- /// - Parameter paths: An array of path strings.
- /// - Returns: A ``DataFrame``.
- public func load(_ paths: [String]) -> DataFrame {
- self.paths = paths
-
- var dataSource = DataSource()
- dataSource.format = self.source
- dataSource.paths = self.paths
- dataSource.options = self.extraOptions.toStringDictionary()
- if let userSpecifiedSchemaDDL = self.userSpecifiedSchemaDDL {
- dataSource.schema = userSpecifiedSchemaDDL
- }
-
- var read = Read()
- read.dataSource = dataSource
-
- var relation = Relation()
- relation.read = read
-
- var plan = Plan()
- plan.opType = .root(relation)
-
- return DataFrame(spark: sparkSession, plan: plan)
- }
-
- /// Loads a CSV file and returns the result as a ``DataFrame``. See the documentation on the other
- /// overloaded `csv()` method for more details.
- /// - Parameter path: A path string
- /// - Returns: A ``DataFrame``.
- public func csv(_ path: String) -> DataFrame {
- self.source = "csv"
- return load(path)
- }
-
- /// Loads CSV files and returns the result as a ``DataFrame``.
- /// This function will go through the input once to determine the input schema if `inferSchema`
- /// is enabled. To avoid going through the entire data once, disable `inferSchema` option or
- /// specify the schema explicitly using `schema`.
- /// - Parameter paths: Path strings.
- /// - Returns: A ``DataFrame``.
- public func csv(_ paths: String...) -> DataFrame {
- self.source = "csv"
- return load(paths)
- }
-
- /// Loads a JSON file and returns the result as a ``DataFrame``.
- /// - Parameter path: A path string
- /// - Returns: A ``DataFrame``.
- public func json(_ path: String) -> DataFrame {
- self.source = "json"
- return load(path)
- }
-
- /// Loads JSON files and returns the result as a ``DataFrame``.
- /// - Parameter paths: Path strings
- /// - Returns: A ``DataFrame``.
- public func json(_ paths: String...) -> DataFrame {
- self.source = "json"
- return load(paths)
- }
-
- /// Loads an XML file and returns the result as a ``DataFrame``.
- /// - Parameter path: A path string
- /// - Returns: A ``DataFrame``.
- public func xml(_ path: String) -> DataFrame {
- self.source = "xml"
- return load(path)
- }
-
- /// Loads XML files and returns the result as a ``DataFrame``.
- /// - Parameter paths: Path strings
- /// - Returns: A ``DataFrame``.
- public func xml(_ paths: String...) -> DataFrame {
- self.source = "xml"
- return load(paths)
- }
-
- /// Loads an ORC file and returns the result as a ``DataFrame``.
- /// - Parameter path: A path string
- /// - Returns: A ``DataFrame``.
- public func orc(_ path: String) -> DataFrame {
- self.source = "orc"
- return load(path)
- }
-
- /// Loads ORC files and returns the result as a ``DataFrame``.
- /// - Parameter paths: Path strings
- /// - Returns: A ``DataFrame``.
- public func orc(_ paths: String...) -> DataFrame {
- self.source = "orc"
- return load(paths)
- }
-
- /// Loads a Parquet file and returns the result as a ``DataFrame``.
- /// - Parameter path: A path string
- /// - Returns: A ``DataFrame``.
- public func parquet(_ path: String) -> DataFrame {
- self.source = "parquet"
- return load(path)
- }
-
- /// Loads Parquet files, returning the result as a ``DataFrame``.
- /// - Parameter paths: Path strings
- /// - Returns: A ``DataFrame``.
- public func parquet(_ paths: String...) -> DataFrame {
- self.source = "parquet"
- return load(paths)
- }
-
- /// Construct a `DataFrame` representing the database table accessible via JDBC URL url named
- /// table and connection properties.
- /// - Parameters:
- /// - url: The JDBC URL of the form `jdbc:subprotocol:subname` to connect to.
- /// - table: The JDBC table that should be read from or written into.
- /// - properties: A string-string dictionary for connection properties.
- /// - Returns: A `DataFrame`.
- public func jdbc(_ url: String, _ table: String, _ properties: [String: String] = [:]) -> DataFrame {
- for (key, value) in properties {
- self.extraOptions[key] = value
- }
- self.extraOptions["url"] = url
- self.extraOptions["dbtable"] = table
- self.source = "jdbc"
- return load()
- }
-}
diff --git a/Sources/SparkConnect/DataFrameWriter.swift b/Sources/SparkConnect/DataFrameWriter.swift
deleted file mode 100644
index 11a5fa8..0000000
--- a/Sources/SparkConnect/DataFrameWriter.swift
+++ /dev/null
@@ -1,248 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-import Foundation
-
-/// An interface used to write a `DataFrame` to external storage systems
-/// (e.g. file systems, key-value stores, etc). Use `DataFrame.write` to access this.
-public actor DataFrameWriter: Sendable {
- var source: String? = nil
-
- var saveMode: String = "default"
-
- var extraOptions: CaseInsensitiveDictionary = CaseInsensitiveDictionary()
-
- var partitioningColumns: [String]? = nil
-
- var bucketColumnNames: [String]? = nil
-
- var numBuckets: Int32? = nil
-
- var sortColumnNames: [String]? = nil
-
- var clusteringColumns: [String]? = nil
-
- let df: DataFrame
-
- init(df: DataFrame) {
- self.df = df
- }
-
- /// Specifies the output data source format.
- /// - Parameter source: A string.
- /// - Returns: A `DataFrameReader`.
- public func format(_ source: String) -> DataFrameWriter {
- self.source = source
- return self
- }
-
- /// Specifies the behavior when data or table already exists. Options include:
- /// `overwrite`, `append`, `ignore`, `error` or `errorifexists` (default).
- ///
- /// - Parameter saveMode: A string for save mode.
- /// - Returns: A `DataFrameWriter`
- public func mode(_ saveMode: String) -> DataFrameWriter {
- self.saveMode = saveMode
- return self
- }
-
- /// Adds an output option for the underlying data source.
- /// - Parameters:
- /// - key: A key string.
- /// - value: A value string.
- /// - Returns: A `DataFrameWriter`.
- public func option(_ key: String, _ value: String) -> DataFrameWriter {
- self.extraOptions[key] = value
- return self
- }
-
- public func partitionBy(_ columns: String...) -> DataFrameWriter {
- self.partitioningColumns = columns
- return self
- }
-
- public func bucketBy(numBuckets: Int32, _ columnNames: String...) -> DataFrameWriter {
- self.numBuckets = numBuckets
- self.bucketColumnNames = columnNames
- return self
- }
-
- public func sortBy(_ columnNames: String...) -> DataFrameWriter {
- self.sortColumnNames = columnNames
- return self
- }
-
- public func clusterBy(_ columnNames: String...) -> DataFrameWriter {
- self.clusteringColumns = columnNames
- return self
- }
-
- /// Loads input in as a `DataFrame`, for data sources that don't require a path (e.g. external
- /// key-value stores).
- public func save() async throws {
- return try await saveInternal(nil)
- }
-
- /// Loads input in as a `DataFrame`, for data sources that require a path (e.g. data backed by a
- /// local or distributed file system).
- /// - Parameter path: A path string.
- public func save(_ path: String) async throws {
- try await saveInternal(path)
- }
-
- private func saveInternal(_ path: String?) async throws {
- try await executeWriteOperation({
- var write = WriteOperation()
- if let path = path {
- write.path = path
- }
- return write
- })
- }
-
- /// Saves the content of the ``DataFrame`` as the specified table.
- /// - Parameter tableName: A table name.
- public func saveAsTable(_ tableName: String) async throws {
- try await executeWriteOperation({
- var write = WriteOperation()
- write.table.tableName = tableName
- write.table.saveMethod = .saveAsTable
- return write
- })
- }
-
- /// Inserts the content of the ``DataFrame`` to the specified table. It requires that the schema of
- /// the ``DataFrame`` is the same as the schema of the table. Unlike ``saveAsTable``,
- /// ``insertInto`` ignores the column names and just uses position-based resolution.
- /// - Parameter tableName: A table name.
- public func insertInto(_ tableName: String) async throws {
- try await executeWriteOperation({
- var write = WriteOperation()
- write.table.tableName = tableName
- write.table.saveMethod = .insertInto
- return write
- })
- }
-
- private func executeWriteOperation(_ f: () -> WriteOperation) async throws {
- var write = f()
-
- // Cannot both be set
- assert(!(!write.path.isEmpty && !write.table.tableName.isEmpty))
-
- let plan = await self.df.getPlan() as! Plan
- write.input = plan.root
- write.mode = self.saveMode.toSaveMode
-
- if let source = self.source {
- write.source = source
- }
- if let sortColumnNames = self.sortColumnNames {
- write.sortColumnNames = sortColumnNames
- }
- if let partitioningColumns = self.partitioningColumns {
- write.partitioningColumns = partitioningColumns
- }
- if let clusteringColumns = self.clusteringColumns {
- write.clusteringColumns = clusteringColumns
- }
- if let numBuckets = self.numBuckets {
- var bucketBy = WriteOperation.BucketBy()
- bucketBy.numBuckets = numBuckets
- if let bucketColumnNames = self.bucketColumnNames {
- bucketBy.bucketColumnNames = bucketColumnNames
- }
- write.bucketBy = bucketBy
- }
-
- for option in self.extraOptions.toStringDictionary() {
- write.options[option.key] = option.value
- }
-
- var command = Spark_Connect_Command()
- command.writeOperation = write
-
- try await df.spark.client.execute(df.spark.sessionID, command)
- }
-
- /// Saves the content of the `DataFrame` in CSV format at the specified path.
- /// - Parameter path: A path string
- /// - Returns: A `DataFrame`.
- public func csv(_ path: String) async throws {
- self.source = "csv"
- return try await save(path)
- }
-
- /// Saves the content of the `DataFrame` in JSON format at the specified path.
- /// - Parameter path: A path string
- /// - Returns: A `DataFrame`.
- public func json(_ path: String) async throws {
- self.source = "json"
- return try await save(path)
- }
-
- /// Saves the content of the `DataFrame` in XML format at the specified path.
- /// - Parameter path: A path string
- /// - Returns: A `DataFrame`.
- public func xml(_ path: String) async throws {
- self.source = "xml"
- return try await save(path)
- }
-
- /// Saves the content of the `DataFrame` in ORC format at the specified path.
- /// - Parameter path: A path string
- /// - Returns: A `DataFrame`.
- public func orc(_ path: String) async throws {
- self.source = "orc"
- return try await save(path)
- }
-
- /// Saves the content of the `DataFrame` in Parquet format at the specified path.
- /// - Parameter path: A path string
- public func parquet(_ path: String) async throws {
- self.source = "parquet"
- return try await save(path)
- }
-
- /// Saves the content of the `DataFrame` in a text file at the specified path.
- /// The DataFrame must have only one column that is of string type.
- /// Each row becomes a new line in the output file.
- ///
- /// - Parameter path: A path string
- public func text(_ path: String) async throws {
- self.source = "text"
- return try await save(path)
- }
-
- /// Saves the content of the `DataFrame` to an external database table via JDBC. In the case the
- /// table already exists in the external database, behavior of this function depends on the save
- /// mode, specified by the `mode` function (default to throwing an exception).
- /// - Parameters:
- /// - url: The JDBC URL of the form `jdbc:subprotocol:subname` to connect to.
- /// - table: Name of the table in the external database.
- /// - properties:JDBC database connection arguments, a list of arbitrary string tag/value.
- public func jdbc(_ url: String, _ table: String, _ properties: [String: String] = [:]) async throws {
- for (key, value) in properties {
- self.extraOptions[key] = value
- }
- self.extraOptions["url"] = url
- self.extraOptions["dbtable"] = table
- self.source = "jdbc"
- return try await save()
- }
-}
diff --git a/Sources/SparkConnect/DataFrameWriterV2.swift b/Sources/SparkConnect/DataFrameWriterV2.swift
deleted file mode 100644
index b2ad861..0000000
--- a/Sources/SparkConnect/DataFrameWriterV2.swift
+++ /dev/null
@@ -1,152 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-import Foundation
-
-/// Interface used to write a ``DataFrame`` to external storage using the v2 API.
-public actor DataFrameWriterV2: Sendable {
-
- let tableName: String
-
- let df: DataFrame
-
- var provider: String? = nil
-
- var extraOptions: CaseInsensitiveDictionary = CaseInsensitiveDictionary()
-
- var tableProperties: CaseInsensitiveDictionary = CaseInsensitiveDictionary()
-
- var partitioningColumns: [Spark_Connect_Expression] = []
-
- var clusteringColumns: [String]? = nil
-
- init(_ table: String, _ df: DataFrame) {
- self.tableName = table
- self.df = df
- }
-
- /// Specifies a provider for the underlying output data source. Spark's default catalog supports
- /// "orc", "json", etc.
- /// - Parameter provider: <#provider description#>
- public func using(_ provider: String) -> DataFrameWriterV2 {
- self.provider = provider
- return self
- }
-
- /// Adds an output option for the underlying data source.
- /// - Parameters:
- /// - key: A key string.
- /// - value: A value string.
- /// - Returns: A `DataFrameWriter`.
- public func option(_ key: String, _ value: String) -> DataFrameWriterV2 {
- self.extraOptions[key] = value
- return self
- }
-
- /// Add a table property.
- /// - Parameters:
- /// - property: A property name.
- /// - value: A property value.
- public func tableProperty(property: String, value: String) -> DataFrameWriterV2 {
- self.tableProperties[property] = value
- return self
- }
-
- /// Partition the output table created by `create`, `createOrReplace`, or `replace` using the
- /// given columns or transforms.
- /// - Parameter columns: Columns to partition
- /// - Returns: A ``DataFrameWriterV2``.
- public func partitionBy(_ columns: String...) -> DataFrameWriterV2 {
- self.partitioningColumns = columns.map {
- var expr = Spark_Connect_Expression()
- expr.expressionString = $0.toExpressionString
- return expr
- }
- return self
- }
-
- /// Clusters the output by the given columns on the storage. The rows with matching values in the
- /// specified clustering columns will be consolidated within the same group.
- /// - Parameter columns: Columns to cluster
- /// - Returns: A ``DataFrameWriterV2``.
- public func clusterBy(_ columns: String...) -> DataFrameWriterV2 {
- self.clusteringColumns = columns
- return self
- }
-
- /// Create a new table from the contents of the data frame.
- public func create() async throws {
- try await executeWriteOperation(.create)
- }
-
- /// Replace an existing table with the contents of the data frame.
- public func replace() async throws {
- try await executeWriteOperation(.replace)
- }
-
- /// Create a new table or replace an existing table with the contents of the data frame.
- public func createOrReplace() async throws {
- try await executeWriteOperation(.createOrReplace)
- }
-
- /// Append the contents of the data frame to the output table.
- public func append() async throws {
- try await executeWriteOperation(.append)
- }
-
- /// Overwrite rows matching the given filter condition with the contents of the ``DataFrame`` in the
- /// output table.
- /// - Parameter condition: A filter condition.
- public func overwrite(condition: String) async throws {
- try await executeWriteOperation(.overwrite)
- }
-
- /// Overwrite all partition for which the ``DataFrame`` contains at least one row with the contents
- /// of the data frame in the output table.
- /// This operation is equivalent to Hive's `INSERT OVERWRITE ... PARTITION`, which replaces
- /// partitions dynamically depending on the contents of the ``DataFrame``.
- public func overwritePartitions() async throws {
- try await executeWriteOperation(.overwritePartitions)
- }
-
- private func executeWriteOperation(_ mode: WriteOperationV2.Mode) async throws {
- var write = WriteOperationV2()
-
- let plan = await self.df.getPlan() as! Plan
- write.input = plan.root
- write.tableName = self.tableName
- if let provider = self.provider {
- write.provider = provider
- }
- write.partitioningColumns = self.partitioningColumns
- if let clusteringColumns = self.clusteringColumns {
- write.clusteringColumns = clusteringColumns
- }
- for option in self.extraOptions.toStringDictionary() {
- write.options[option.key] = option.value
- }
- for property in self.tableProperties.toStringDictionary() {
- write.tableProperties[property.key] = property.value
- }
- write.mode = mode
-
- var command = Spark_Connect_Command()
- command.writeOperationV2 = write
- try await df.spark.client.execute(df.spark.sessionID, command)
- }
-}
diff --git a/Sources/SparkConnect/DataStreamReader.swift b/Sources/SparkConnect/DataStreamReader.swift
deleted file mode 100644
index e90a1c1..0000000
--- a/Sources/SparkConnect/DataStreamReader.swift
+++ /dev/null
@@ -1,227 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-import Foundation
-
-/// An actor to load a streaming ``DataFrame`` from external storage systems
-/// (e.g. file systems, key-value stores, etc). Use `SparkSession.readStream` to access this.
-public actor DataStreamReader: Sendable {
- var source: String = ""
-
- var paths: [String] = []
-
- var extraOptions: CaseInsensitiveDictionary = CaseInsensitiveDictionary([:])
-
- var userSpecifiedSchemaDDL: String? = nil
-
- let sparkSession: SparkSession
-
- init(sparkSession: SparkSession) {
- self.sparkSession = sparkSession
- }
-
- /// Specifies the input data source format.
- /// - Parameter source: A string.
- /// - Returns: A ``DataStreamReader``.
- public func format(_ source: String) -> DataStreamReader {
- self.source = source
- return self
- }
-
- /// Specifies the input schema. Some data sources (e.g. JSON) can infer the input schema
- /// automatically from data. By specifying the schema here, the underlying data source can skip
- /// the schema inference step, and thus speed up data loading.
- /// - Parameter schema: A DDL schema string.
- /// - Returns: A ``DataStreamReader``.
- @discardableResult
- public func schema(_ schema: String) async throws -> DataStreamReader {
- // Validate by parsing.
- do {
- try await sparkSession.client.ddlParse(schema)
- } catch {
- throw SparkConnectError.InvalidTypeException
- }
- self.userSpecifiedSchemaDDL = schema
- return self
- }
-
- /// Adds an input option for the underlying data source.
- /// - Parameters:
- /// - key: A key string.
- /// - value: A value string.
- /// - Returns: A ``DataStreamReader``.
- public func option(_ key: String, _ value: String) -> DataStreamReader {
- self.extraOptions[key] = value
- return self
- }
-
- /// Adds an input option for the underlying data source.
- /// - Parameters:
- /// - key: A key string.
- /// - value: A `Bool` value.
- /// - Returns: A ``DataStreamReader``.
- public func option(_ key: String, _ value: Bool) -> DataStreamReader {
- return option(key, String(value))
- }
-
- /// Adds an input option for the underlying data source.
- /// - Parameters:
- /// - key: A key string.
- /// - value: A `Int` value.
- /// - Returns: A ``DataStreamReader``.
- public func option(_ key: String, _ value: Int) -> DataStreamReader {
- return option(key, String(value))
- }
-
- /// Adds an input option for the underlying data source.
- /// - Parameters:
- /// - key: A key string.
- /// - value: A `Int64` value.
- /// - Returns: A ``DataStreamReader``.
- public func option(_ key: String, _ value: Int64) -> DataStreamReader {
- return option(key, String(value))
- }
-
- /// Adds an input option for the underlying data source.
- /// - Parameters:
- /// - key: A key string.
- /// - value: A `Double` value.
- /// - Returns: A ``DataStreamReader``.
- public func option(_ key: String, _ value: Double) -> DataStreamReader {
- return option(key, String(value))
- }
-
- /// Adds input options for the underlying data source.
- /// - Parameter options: A string-string dictionary.
- /// - Returns: A ``DataStreamReader``.
- public func options(_ options: [String: String]) -> DataStreamReader {
- for (key, value) in options {
- self.extraOptions[key] = value
- }
- return self
- }
-
- /// Loads input data stream in as a ``DataFrame``, for data streams that don't require a path
- /// (e.g. external key-value stores).
- /// - Returns: A ``DataFrame``.
- public func load() -> DataFrame {
- return load([])
- }
-
- /// Loads input data stream in as a ``DataFrame``, for data streams that require a path
- /// (e.g. data backed by a local or distributed file system).
- /// - Parameter path: A path string.
- /// - Returns: A ``DataFrame``.
- public func load(_ path: String) -> DataFrame {
- return load([path])
- }
-
- func load(_ paths: [String]) -> DataFrame {
- self.paths = paths
-
- var dataSource = DataSource()
- dataSource.format = self.source
- dataSource.paths = self.paths
- dataSource.options = self.extraOptions.toStringDictionary()
- if let userSpecifiedSchemaDDL = self.userSpecifiedSchemaDDL {
- dataSource.schema = userSpecifiedSchemaDDL
- }
-
- var read = Read()
- read.dataSource = dataSource
- read.isStreaming = true
-
- var relation = Relation()
- relation.read = read
-
- var plan = Plan()
- plan.opType = .root(relation)
-
- return DataFrame(spark: sparkSession, plan: plan)
- }
-
- /// Define a Streaming ``DataFrame`` on a Table. The DataSource corresponding to the table should
- /// support streaming mode.
- /// - Parameter tableName: The name of the table.
- /// - Returns: A ``DataFrame``.
- public func table(_ tableName: String) -> DataFrame {
- var namedTable = NamedTable()
- namedTable.unparsedIdentifier = tableName
- namedTable.options = self.extraOptions.toStringDictionary()
-
- var read = Read()
- read.namedTable = namedTable
- read.isStreaming = true
-
- var relation = Relation()
- relation.read = read
-
- var plan = Plan()
- plan.opType = .root(relation)
-
- return DataFrame(spark: sparkSession, plan: plan)
- }
-
- /// Loads a text file stream and returns the result as a ``DataFrame``.
- /// - Parameter path: A path string
- /// - Returns: A ``DataFrame``.
- public func text(_ path: String) -> DataFrame {
- self.source = "text"
- return load(path)
- }
-
- /// Loads a CSV file stream and returns the result as a ``DataFrame``.
- /// - Parameter path: A path string
- /// - Returns: A ``DataFrame``.
- public func csv(_ path: String) -> DataFrame {
- self.source = "csv"
- return load(path)
- }
-
- /// Loads a JSON file stream and returns the result as a ``DataFrame``.
- /// - Parameter path: A path string
- /// - Returns: A ``DataFrame``.
- public func json(_ path: String) -> DataFrame {
- self.source = "json"
- return load(path)
- }
-
- /// Loads an XML file stream and returns the result as a ``DataFrame``.
- /// - Parameter path: A path string
- /// - Returns: A ``DataFrame``.
- public func xml(_ path: String) -> DataFrame {
- self.source = "xml"
- return load(path)
- }
-
- /// Loads an ORC file stream and returns the result as a ``DataFrame``.
- /// - Parameter path: A path string
- /// - Returns: A ``DataFrame``.
- public func orc(_ path: String) -> DataFrame {
- self.source = "orc"
- return load(path)
- }
-
- /// Loads a Parquet file stream and returns the result as a ``DataFrame``.
- /// - Parameter path: A path string
- /// - Returns: A ``DataFrame``.
- public func parquet(_ path: String) -> DataFrame {
- self.source = "parquet"
- return load(path)
- }
-}
diff --git a/Sources/SparkConnect/DataStreamWriter.swift b/Sources/SparkConnect/DataStreamWriter.swift
deleted file mode 100644
index 08fbd0e..0000000
--- a/Sources/SparkConnect/DataStreamWriter.swift
+++ /dev/null
@@ -1,208 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-import Foundation
-
-public enum Trigger {
- case OneTime
- case AvailableNow
- case ProcessingTime(_ intervalMs: Int64)
- case Continuous(_ intervalMs: Int64)
-}
-
-/// An actor used to write a streaming `DataFrame` to external storage systems
-/// (e.g. file systems, key-value stores, etc). Use `DataFrame.writeStream` to access this.
-public actor DataStreamWriter: Sendable {
- var queryName: String? = nil
-
- var source: String? = nil
-
- var trigger: Trigger = Trigger.ProcessingTime(0)
-
- var path: String? = nil
-
- var tableName: String? = nil
-
- var outputMode: String? = nil
-
- var extraOptions: CaseInsensitiveDictionary = CaseInsensitiveDictionary()
-
- var partitioningColumns: [String]? = nil
-
- var clusteringColumns: [String]? = nil
-
- let df: DataFrame
-
- init(df: DataFrame) {
- self.df = df
- }
-
- /// Specifies the name of the ``StreamingQuery`` that can be
- /// started with `start()`. This name must be unique among all the currently active queries in
- /// the associated SparkSession.
- /// - Parameter queryName: A string name.
- /// - Returns: A ``DataStreamWriter``.
- public func queryName(_ queryName: String) -> DataStreamWriter {
- self.queryName = queryName
- return self
- }
-
- /// Specifies the underlying output data source.
- /// - Parameter source: A string.
- /// - Returns: A `DataStreamWriter`.
- public func format(_ source: String) -> DataStreamWriter {
- self.source = source
- return self
- }
-
- /// Specifies how data of a streaming ``DataFrame`` is written to a streaming sink.
- ///
- /// - `append`: only the new rows in the streaming ``DataFrame`` will be written to the sink.
- /// - `complete`: all the rows in the streaming ``DataFrame`` will be written to the sink
- /// every time there are some updates.
- /// - `update`: only the rows that were updated in the streaming ``DataFrame`` will be
- /// written to the sink every time there are some updates. If the query doesn't contain
- /// aggregations, it will be equivalent to `append` mode.
- ///
- /// - Parameter outputMode: A string for outputMode.
- /// - Returns: A ``DataStreamWriter``.
- public func outputMode(_ outputMode: String) -> DataStreamWriter {
- self.outputMode = outputMode
- return self
- }
-
- /// Adds an output option for the underlying data source.
- /// - Parameters:
- /// - key: A key string.
- /// - value: A value string.
- /// - Returns: A `DataStreamWriter`.
- public func option(_ key: String, _ value: String) -> DataStreamWriter {
- self.extraOptions[key] = value
- return self
- }
-
- /// Partitions the output by the given columns on the file system. If specified, the output is
- /// laid out on the file system similar to Hive's partitioning scheme.
- /// - Parameter colNames: Column names to partition.
- /// - Returns: A ``DataStreamWriter``.
- public func partitionBy(_ colNames: String...) -> DataStreamWriter {
- self.partitioningColumns = colNames
- return self
- }
-
- /// Clusters the output by the given columns. If specified, the output is laid out such that
- /// records with similar values on the clustering column are grouped together in the same file.
- /// - Parameter colNames: Column names to cluster.
- /// - Returns: A ``DataStreamWriter``.
- public func clusterBy(_ colNames: String...) -> DataStreamWriter {
- self.clusteringColumns = colNames
- return self
- }
-
- /// Loads input in as a `DataFrame`, for data sources that don't require a path (e.g. external
- /// key-value stores).
- public func trigger(_ trigger: Trigger) async throws -> DataStreamWriter {
- self.trigger = trigger
- return self
- }
-
- /// Starts the execution of the streaming query, which will continually output results to the
- /// given path as new data arrives. The returned ``StreamingQuery`` object can be used to interact
- /// with the stream.
- /// - Parameter path: A path to write.
- /// - Returns: A ``StreamingQuery``.
- public func start(_ path: String) async throws -> StreamingQuery {
- self.path = path
- return try await start()
- }
-
- /// Starts the execution of the streaming query, which will continually output results to the
- /// given path as new data arrives. The returned ``StreamingQuery`` object can be used to interact
- /// with the stream. Throws exceptions if the following conditions are met:
- /// - Another run of the same streaming query, that is a streaming query sharing the same
- /// checkpoint location, is already active on the same Spark Driver
- /// - The SQL configuration `spark.sql.streaming.stopActiveRunOnRestart` is enabled
- /// - The active run cannot be stopped within the timeout controlled by the SQL configuration `spark.sql.streaming.stopTimeout`
- ///
- /// - Returns: A ``StreamingQuery``.
- public func start() async throws -> StreamingQuery {
- var writeStreamOperationStart = WriteStreamOperationStart()
- writeStreamOperationStart.input = (await df.getPlan() as! Plan).root
- if let source = self.source {
- writeStreamOperationStart.format = source
- }
- writeStreamOperationStart.options = self.extraOptions.toStringDictionary()
- if let partitioningColumns = self.partitioningColumns {
- writeStreamOperationStart.partitioningColumnNames = partitioningColumns
- }
- if let clusteringColumns = self.clusteringColumns {
- writeStreamOperationStart.clusteringColumnNames = clusteringColumns
- }
- writeStreamOperationStart.trigger =
- switch self.trigger {
- case .ProcessingTime(let intervalMs):
- .processingTimeInterval("INTERVAL \(intervalMs) MILLISECOND")
- case .OneTime:
- .once(true)
- case .AvailableNow:
- .availableNow(true)
- case .Continuous(let intervalMs):
- .continuousCheckpointInterval("INTERVAL \(intervalMs) MILLISECOND")
- }
- if let outputMode = self.outputMode {
- writeStreamOperationStart.outputMode = outputMode
- }
- if let queryName = self.queryName {
- writeStreamOperationStart.queryName = queryName
- }
- if let path = self.path {
- writeStreamOperationStart.sinkDestination = .path(path)
- }
- if let tableName = self.tableName {
- writeStreamOperationStart.sinkDestination = .tableName(tableName)
- }
-
- var command = Spark_Connect_Command()
- command.writeStreamOperationStart = writeStreamOperationStart
-
- let response = try await df.spark.client.execute(df.spark.sessionID, command)
- let result = response.first!.writeStreamOperationStartResult
- if result.hasQueryStartedEventJson {
- // TODO: post
- }
-
- let query = try await StreamingQuery(
- UUID(uuidString: result.queryID.id)!,
- UUID(uuidString: result.queryID.runID)!,
- result.name,
- self.df.sparkSession
- )
-
- return query
- }
-
- /// Starts the execution of the streaming query, which will continually output results to the
- /// given table as new data arrives. The returned ``StreamingQuery`` object can be used to interact
- /// with the stream.
- /// - Parameter tableName: A table name.
- /// - Returns: A ``StreamingQuery``.
- public func toTable(tableName: String) async throws -> StreamingQuery {
- self.tableName = tableName
- return try await start()
- }
-}
diff --git a/Sources/SparkConnect/Documentation.docc/Examples.md b/Sources/SparkConnect/Documentation.docc/Examples.md
deleted file mode 100644
index 2f87e70..0000000
--- a/Sources/SparkConnect/Documentation.docc/Examples.md
+++ /dev/null
@@ -1,137 +0,0 @@
-# Spark Connect Swift Examples
-
-This document provides an overview of the example applications inside [Examples](https://github.com/apache/spark-connect-swift/tree/main/Examples) directory. These examples demonstrate how to use Spark Connect Swift to interact with Apache Spark in different contexts.
-
-## Prepare Spark Connect Server
-
-Start a Spark Connect Server:
-
-```bash
-docker run -it --rm -p 15002:15002 apache/spark:4.0.0-preview2 bash -c "/opt/spark/sbin/start-connect-server.sh --wait -c spark.log.level=ERROR"
-```
-
-## Basic Application Example
-
-The basic application example demonstrates fundamental operations with Apache Spark Connect, including:
-- Connecting to a Spark server
-- Creating and manipulating tables with SQL
-- Using DataFrame operations
-- Reading and writing data in the ORC format
-
-### Key Features
-- SQL execution for table operations
-- DataFrame transformations with filter operations
-- Data persistence with ORC format
-- Basic session management
-
-### How to Run
-
-Build and run the application:
-
-```bash
-# Using Docker
-docker build -t apache/spark-connect-swift:app .
-docker run -it --rm -e SPARK_REMOTE=sc://host.docker.internal:15002 apache/spark-connect-swift:app
-
-# From source code
-swift run
-```
-
-## Pi Calculation Example
-
-The Pi calculation example shows how to use Spark Connect Swift for computational tasks by calculating an approximation of π (pi) using the Monte Carlo method.
-
-### Key Features
-- Command-line argument handling
-- Mathematical computations with Spark
-- Random number generation
-- Filtering and counting operations
-
-### How to Run
-
-Build and run the application:
-
-```bash
-# Using Docker
-docker build -t apache/spark-connect-swift:pi .
-docker run --rm -e SPARK_REMOTE=sc://host.docker.internal:15002 apache/spark-connect-swift:pi
-
-# From source code
-swift run
-```
-
-## Streaming Example
-
-The streaming example demonstrates how to process streaming data using Spark Connect Swift client, specifically for counting words from a network socket stream.
-
-### Key Features
-- Stream processing with Spark Connect
-- Network socket data source
-- Word counting with string operations
-- Real-time console output
-
-### How to Run
-
-Start a Netcat server as the data source:
-
-```bash
-nc -lk 9999
-```
-
-Build and run the application:
-
-```bash
-# Using Docker
-docker build -t apache/spark-connect-swift:stream .
-docker run --rm -e SPARK_REMOTE=sc://host.docker.internal:15002 -e TARGET_HOST=host.docker.internal apache/spark-connect-swift:stream
-
-# From source code
-swift run
-```
-
-Type text into the Netcat terminal to see real-time word counting from `Spark Connect Server` container.
-
-## Web Application Example
-
-The web application example showcases how to integrate Spark Connect Swift with a web server using the Vapor framework.
-
-### Key Features
-- HTTP server integration with Vapor
-- REST API endpoints
-- Spark session management within web requests
-- Version information retrieval
-
-### How to Run
-
-Build and run the application:
-
-```bash
-# Using Docker
-docker build -t apache/spark-connect-swift:web .
-docker run -it --rm -p 8080:8080 -e SPARK_REMOTE=sc://host.docker.internal:15002 apache/spark-connect-swift:web
-
-# From source code
-swift run
-```
-
-Access the web application:
-
-```bash
-# Root endpoint
-$ curl http://127.0.0.1:8080/
-Welcome to the Swift world. Say hello!%
-
-# Spark-powered endpoint
-curl http://127.0.0.1:8080/hello
-Hi, this is powered by the Apache Spark 4.0.0-preview2.%
-```
-
-## Development Environment
-
-All examples include:
-- A Dockerfile for containerized execution
-- A Package.swift file for Swift Package Manager configuration
-- A README.md with detailed instructions
-- Source code in the Sources directory
-
-These examples are designed to be used with Apache Spark 4.0.0 or newer, using the Spark Connect protocol for client-server interaction.
diff --git a/Sources/SparkConnect/Documentation.docc/GettingStarted.md b/Sources/SparkConnect/Documentation.docc/GettingStarted.md
deleted file mode 100644
index 9aed405..0000000
--- a/Sources/SparkConnect/Documentation.docc/GettingStarted.md
+++ /dev/null
@@ -1,100 +0,0 @@
-# Getting Started with SparkConnect
-
-A step-by-step guide to get started with SparkConnect for Swift.
-
-## Installation
-
-Add SparkConnect to your Swift package dependencies:
-
-```swift
-dependencies: [
- .package(url: "https://github.com/apache/spark-connect-swift.git", from: "main")
-]
-```
-
-Then, add it to your target dependencies:
-
-```swift
-targets: [
- .target(
- name: "YourApp",
- dependencies: ["SparkConnect"]
- )
-]
-```
-
-## Prerequisites
-
-- Swift 6.0 or later
-- macOS 15+, iOS 18+, watchOS 11+, or tvOS 18+
-- A running Apache Spark cluster with Spark Connect enabled
-
-## Basic Usage
-
-### 1. Create a SparkSession
-
-```swift
-import SparkConnect
-
-let spark = SparkSession.builder()
- .appName("MySwiftApp")
- .remote("sc://localhost:15002")
- .build()
-```
-
-### 2. DataFrame Operations
-
-```swift
-// From a range
-let df1 = try await spark.range(1, 10)
-
-// Show data
-try await df1.show()
-
-// Select columns
-try await df1.select("id").show()
-
-let df2 = await df1.selectExpr("id", "id % 3 as value")
-try await df2.show()
-
-// Filter data
-try await df2.filter("value == 0").show()
-
-// Group and aggregate
-try await df2
- .groupBy("value")
- .agg("count(*)", "sum(value)")
- .show()
-```
-
-### 3. SQL Queries
-
-```swift
-// Register a temporary view
-try await df2.createOrReplaceTempView("v1")
-
-// Run SQL Queries
-let result = try await spark.sql("""
- SELECT id, sum(value) as value_sum
- FROM v1
- GROUP BY id
- ORDER BY value_sum DESC
-""")
-
-result.show()
-```
-
-### 4. Reading and Writing Data
-
-```swift
-// Read CSV
-let csvDf = spark.read
- .option("header", "true")
- .option("inferSchema", "true")
- .csv("path/to/data.csv")
-
-// Write ORC
-csvDf.write
- .mode("overwrite")
- .orc("path/to/output")
-```
diff --git a/Sources/SparkConnect/Documentation.docc/Info.plist b/Sources/SparkConnect/Documentation.docc/Info.plist
deleted file mode 100644
index 7137f3f..0000000
--- a/Sources/SparkConnect/Documentation.docc/Info.plist
+++ /dev/null
@@ -1,32 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-<dict>
- <key>CFBundleDisplayName</key>
- <string>SparkConnect</string>
- <key>CFBundleIdentifier</key>
- <string>org.apache.spark.connect.swift.SparkConnect</string>
- <key>CFBundleName</key>
- <string>SparkConnect</string>
- <key>CFBundleVersion</key>
- <string>0.1.0</string>
- <key>NSHumanReadableCopyright</key>
- <string>© 2025 Apache Software Foundation</string>
-</dict>
-</plist>
diff --git a/Sources/SparkConnect/Documentation.docc/SparkConnect.md b/Sources/SparkConnect/Documentation.docc/SparkConnect.md
deleted file mode 100644
index 6c1f49d..0000000
--- a/Sources/SparkConnect/Documentation.docc/SparkConnect.md
+++ /dev/null
@@ -1,25 +0,0 @@
-# ``SparkConnect``
-
-Swift implementation of Apache Spark Connect client for distributed data processing.
-
-## Overview
-
-SparkConnect is a modern Swift library that provides a native interface to Apache Spark clusters using the Spark Connect protocol. This library enables Swift developers to leverage the power of Apache Spark for distributed data processing, machine learning, and analytical workloads directly from their Swift applications.
-
-### Key Features
-
-- Native Swift API for Apache Spark operations
-- Support for DataFrame and SQL operations
-- Support for grouped data operations and aggregations
-- Efficient data serialization using Arrow format
-
-## Topics
-
-### Getting Started
-
-- <doc:GettingStarted>
-- ``SparkSession``
-
-### DataFrame Operations
-
-- ``DataFrame``
diff --git a/Sources/SparkConnect/Documentation.docc/SparkSession.md b/Sources/SparkConnect/Documentation.docc/SparkSession.md
deleted file mode 100644
index 2fa4425..0000000
--- a/Sources/SparkConnect/Documentation.docc/SparkSession.md
+++ /dev/null
@@ -1,64 +0,0 @@
-# ``SparkConnect/SparkSession``
-
-The entry point for SparkConnect functionality.
-
-## Overview
-
-`SparkSession` is the primary interaction point with Apache Spark. It provides an interface to create DataFrames, execute SQL queries, and manage cluster configurations.
-
-### Creating a SparkSession
-
-```swift
-let spark = SparkSession.builder()
- .appName("My Swift Spark App")
- .remote("sc://localhost:15002")
- .build()
-```
-
-### Basic Usage
-
-```swift
-// Create a DataFrame from a range
-let df = spark.range(1, 10)
-
-// Execute SQL query
-let result = spark.sql("SELECT * FROM table")
-
-// Read data from files
-let csvDf = spark.read.csv("path/to/file.csv")
-```
-
-## Topics
-
-### Creating Sessions
-
-- ``builder``
-- ``stop()``
-
-### DataFrame Operations
-
-- ``emptyDataFrame``
-- ``range(_:_:_:)``
-- ``sql(_:)``
-
-### Data I/O
-
-- ``read``
-
-### Configuration
-
-- ``conf``
-
-### Catalog Operations
-
-- ``catalog``
-
-### Managing Operations
-
-- ``addTag(_:)``
-- ``removeTag(_:)``
-- ``getTags()``
-- ``clearTags()``
-- ``interruptAll()``
-- ``interruptTag(_:)``
-- ``interruptOperation(_:)``
diff --git a/Sources/SparkConnect/ErrorUtils.swift b/Sources/SparkConnect/ErrorUtils.swift
deleted file mode 100644
index 483e781..0000000
--- a/Sources/SparkConnect/ErrorUtils.swift
+++ /dev/null
@@ -1,36 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-import Foundation
-
-/// Utility functions like `org.apache.spark.util.SparkErrorUtils`.
-public enum ErrorUtils {
- public static func tryWithSafeFinally<T>(
- _ block: () async throws -> T, _ finallyBlock: () async throws -> Void
- ) async rethrows -> T {
- let result: T
- do {
- result = try await block()
- try await finallyBlock()
- } catch {
- try? await finallyBlock()
- throw error
- }
- return result
- }
-}
diff --git a/Sources/SparkConnect/Extension.swift b/Sources/SparkConnect/Extension.swift
deleted file mode 100644
index e841fa4..0000000
--- a/Sources/SparkConnect/Extension.swift
+++ /dev/null
@@ -1,324 +0,0 @@
-//
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-//
-
-import Foundation
-
-extension String {
- /// Get a `Plan` instance from a string.
- var toSparkConnectPlan: Plan {
- var sql = Spark_Connect_SQL()
- sql.query = self
- var relation = Relation()
- relation.sql = sql
- var plan = Plan()
- plan.opType = Plan.OneOf_OpType.root(relation)
- return plan
- }
-
- func toSparkConnectPlan(_ posArguments: [Sendable]) -> Plan {
- var sql = Spark_Connect_SQL()
- sql.query = self
- sql.posArguments = posArguments.map {
- var literal = ExpressionLiteral()
- switch $0 {
- case let value as Bool:
- literal.boolean = value
- case let value as Int8:
- literal.byte = Int32(value)
- case let value as Int16:
- literal.short = Int32(value)
- case let value as Int32:
- literal.integer = value
- case let value as Int64:
- literal.long = value
- case let value as Int:
- literal.long = Int64(value)
- case let value as String:
- literal.string = value
- default:
- literal.string = $0 as! String
- }
- var expr = Spark_Connect_Expression()
- expr.literal = literal
- return expr
- }
- var relation = Relation()
- relation.sql = sql
- var plan = Plan()
- plan.opType = Plan.OneOf_OpType.root(relation)
- return plan
- }
-
- func toSparkConnectPlan(_ namedArguments: [String: Sendable]) -> Plan {
- var sql = Spark_Connect_SQL()
- sql.query = self
- sql.namedArguments = namedArguments.mapValues { value in
- var literal = ExpressionLiteral()
- switch value {
- case let value as Bool:
- literal.boolean = value
- case let value as Int8:
- literal.byte = Int32(value)
- case let value as Int16:
- literal.short = Int32(value)
- case let value as Int32:
- literal.integer = value
- case let value as Int64:
- literal.long = value
- case let value as Int:
- literal.long = Int64(value)
- case let value as String:
- literal.string = value
- default:
- literal.string = value as! String
- }
- var expr = Spark_Connect_Expression()
- expr.literal = literal
- return expr
- }
- var relation = Relation()
- relation.sql = sql
- var plan = Plan()
- plan.opType = Plan.OneOf_OpType.root(relation)
- return plan
- }
-
- /// Get a `UserContext` instance from a string.
- var toUserContext: UserContext {
- var context = UserContext()
- context.userID = self
- context.userName = self
- return context
- }
-
... 50659 lines suppressed ...
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org