You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2022/11/11 09:18:43 UTC
[incubator-sedona] branch master updated: [SEDONA-190] Prepare Sedona for 1.3.0 release (#709)
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sedona.git
The following commit(s) were added to refs/heads/master by this push:
new c455a835 [SEDONA-190] Prepare Sedona for 1.3.0 release (#709)
c455a835 is described below
commit c455a8358f7a72156f5fae54725d3f68ce01274b
Author: Jia Yu <ji...@apache.org>
AuthorDate: Fri Nov 11 02:18:37 2022 -0700
[SEDONA-190] Prepare Sedona for 1.3.0 release (#709)
---
README.md | 7 +-
docs-overrides/main.html | 2 +-
docs/community/contact.md | 4 +-
docs/community/publish.md | 144 ++++++--------
docs/setup/compile.md | 10 +-
docs/setup/maven-coordinates.md | 48 +----
docs/setup/release-notes.md | 98 ++++++++-
flink/pom.xml | 5 -
mkdocs.yml | 12 +-
sql/pom.xml | 5 -
.../scala/org/apache/sedona/sql/UDF/Catalog.scala | 16 +-
.../expressions_udaf/AggregateFunctions.scala | 218 ---------------------
viz/pom.xml | 5 -
13 files changed, 179 insertions(+), 395 deletions(-)
diff --git a/README.md b/README.md
index 32cd0928..ebfa3077 100644
--- a/README.md
+++ b/README.md
@@ -1,12 +1,10 @@
<img src="https://www.apache.org/logos/res/sedona/sedona.png" width="400">
-**Are you a user / contributor / PPMC of Apache Sedona? Please fill in <a href="https://www.surveymonkey.com/r/CKP9RR7">this form</a> to participate in the <b>first ever Sedona online community call</b> on October 22, 2022!**
-
-
[](https://github.com/apache/incubator-sedona/actions/workflows/java.yml) [](https://github.com/apache/incubator-sedona/actions/workflows/python.yml) [](https://github.com/apache/incubator-sedona/action [...]
Click [](https://mybinder.org/v2/gh/apache/incubator-sedona/HEAD?filepath=binder) and play the interactive Sedona Python Jupyter Notebook immediately!
+[](https://discord.gg/9A3k5dEBsY)
Apache Sedona™(incubating) is a cluster computing system for processing large-scale spatial data. Sedona equips cluster computing systems such as Apache Spark and Apache Flink with a set of out-of-the-box distributed Spatial Datasets and Spatial SQL that efficiently load, process, and analyze large-scale spatial data across machines.
@@ -47,7 +45,8 @@ Feedback to improve Apache Sedona: [Google Form](https://docs.google.com/forms/d
Twitter: [Sedona@Twitter](https://twitter.com/ApacheSedona)
-Gitter chat: [](https://gitter.im/apache/sedona?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
+[](https://discord.gg/9A3k5dEBsY)
+
[Sedona JIRA](https://issues.apache.org/jira/projects/SEDONA): Bugs, Pull Requests, and other similar issues
diff --git a/docs-overrides/main.html b/docs-overrides/main.html
index 885b88b7..e7ee8b6c 100644
--- a/docs-overrides/main.html
+++ b/docs-overrides/main.html
@@ -28,7 +28,7 @@
<a href="{{ config.repo_url }}" title="{{ lang.t('source.link.title') }}" class="md-button">
Go to GitHub
</a>
- <p>Are you a user / contributor / PPMC of Apache Sedona? Please fill in <a href="https://www.surveymonkey.com/r/CKP9RR7">this form</a> to participate in the <b>first ever Sedona online community call</b> on October 22, 2022!</p>
+ <p><a href="https://discord.gg/9A3k5dEBsY" target="_blank"><img src="https://dcbadge.vercel.app/api/server/9A3k5dEBsY" alt="Discord Shield"/></a></p>
</div>
<br><br><br>
diff --git a/docs/community/contact.md b/docs/community/contact.md
index 0f1da784..f05ce6fb 100644
--- a/docs/community/contact.md
+++ b/docs/community/contact.md
@@ -18,9 +18,9 @@ Feedback to improve Apache Sedona: [Google Form](https://docs.google.com/forms/d
[Apache Sedona@Twitter](https://twitter.com/ApacheSedona)
-## Gitter chat
+## Discord Server
-[](https://gitter.im/apache/sedona?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge)
+[](https://discord.gg/9A3k5dEBsY)
## Mailing list
diff --git a/docs/community/publish.md b/docs/community/publish.md
index b9a72df3..adfeaa47 100644
--- a/docs/community/publish.md
+++ b/docs/community/publish.md
@@ -15,7 +15,20 @@ This page is for Sedona PPMC to publish Sedona releases.
## Prepare Secret GPG key
1. Install GNUGPG if it was not installed before. On Mac: `brew install gnupg gnupg2`
-2. Generate a secret key. It must be RSA4096 (4096 bits long). Please follow Step 1 - 13 as listed here: https://docs.github.com/en/authentication/managing-commit-signature-verification/generating-a-new-gpg-key#generating-a-gpg-key
+2. Generate a secret key. It must be RSA4096 (4096 bits long).
+ * Run `gpg --full-generate-key`. If not work, run `gpg --default-new-key-algo rsa4096 --gen-key`
+ * At the prompt, specify the kind of key you want: Select `RSA`, then press `enter`
+ * At the prompt, specify the key size you want: Enter `4096`
+ * At the prompt, enter the length of time the key should be valid: Press `enter` to make the key never expire.
+ * Verify that your selections are correct.
+ * Enter your user ID information: use your real name and Apache email address.
+ * Type a secure passphrase.
+ * Use the `gpg --list-secret-keys --keyid-format=long` command to list the long form of the GPG keys.
+ * From the list of GPG keys, copy the long form of the GPG key ID you'd like to use (e.g., `3AA5C34371567BD2`)
+ * Run `gpg --export --armor 3AA5C34371567BD2`, substituting in the GPG key ID you'd like to use.
+ * Copy your GPG key, beginning with `-----BEGIN PGP PUBLIC KEY BLOCK-----` and ending with `-----END PGP PUBLIC KEY BLOCK-----`.
+3. Publish your armored key in major key servers: https://keyserver.pgp.com/
+4. Append your armored PGP public key to the `KEYS` file: https://dist.apache.org/repos/dist/dev/incubator/sedona/KEYS
## Set up ASF username for Maven
@@ -57,26 +70,22 @@ mvn clean -Darguments="-DskipTests" release:prepare -DdryRun=true -DautoVersionS
mvn deploy -DskipTests
```
-#### Prepare for Spark 2.4 and Scala 2.11
+If you are using Mac and see `sign artifacts failed`, please enter the following command in your terminal and re-compile again.
-1. Prepare the SNAPSHOTs
-```
-mvn clean release:prepare -DdryRun=true -DautoVersionSubmodules=true -Dresume=false -DcheckModificationExcludeList=sql/src/main/scala/org/apache/sedona/sql/UDF/UdfRegistrator.scala,sql/src/main/scala/org/apache/spark/sql/sedona_sql/strategy/join/JoinQueryDetector.scala,sql/src/main/scala/org/apache/spark/sql/sedona_sql/strategy/join/TraitJoinQueryExec.scala -Darguments="-DskipTests -Dscala=2.11 -Dspark=2.4"
-```
-2. Deploy the SNAPSHOTs
-```
-mvn deploy -DskipTests -Dscala=2.11 -Dspark=2.4
+```bash
+GPG_TTY=$(tty)
+export GPG_TTY
```
-#### Prepare for Spark 2.4 and Scala 2.12
+#### Prepare for Spark 3.0 and Scala 2.13
1. Prepare the SNAPSHOTs
```
-mvn clean release:prepare -DdryRun=true -DautoVersionSubmodules=true -Dresume=false -DcheckModificationExcludeList=sql/src/main/scala/org/apache/sedona/sql/UDF/UdfRegistrator.scala,sql/src/main/scala/org/apache/spark/sql/sedona_sql/strategy/join/JoinQueryDetector.scala,sql/src/main/scala/org/apache/spark/sql/sedona_sql/strategy/join/TraitJoinQueryExec.scala -Darguments="-DskipTests -Dscala=2.12 -Dspark=2.4"
+mvn clean -Darguments="-DskipTests -Dscala=2.13" release:prepare -DdryRun=true -DautoVersionSubmodules=true -Dresume=false
```
2. Deploy the SNAPSHOTs
```
-mvn deploy -DskipTests -Dscala=2.12 -Dspark=2.4
+mvn deploy -DskipTests -Dscala=2.13
```
## Check ASF copyright in all file headers
@@ -130,16 +139,10 @@ mvn clean release:perform -DautoVersionSubmodules=true -Dresume=false -Dargument
Now let's repeat the process to other Sedona modules. Make sure you use the correct SCM Git tag id `{{ sedona.current_git_tag }}` (see below).
-#### For Spark 2.4 and Scala 2.11
-
-```
-mvn org.apache.maven.plugins:maven-release-plugin:2.3.2:perform -DconnectionUrl=scm:git:https://github.com/apache/incubator-sedona.git -Dtag={{ sedona.current_git_tag }} -DautoVersionSubmodules=true -Dresume=false -Darguments="-DskipTests -Dscala=2.11 -Dspark=2.4"
-```
-
-#### For Spark 2.4 and Scala 2.12
+#### For Spark 3.0 and Scala 2.13
```
-mvn org.apache.maven.plugins:maven-release-plugin:2.3.2:perform -DconnectionUrl=scm:git:https://github.com/apache/incubator-sedona.git -Dtag={{ sedona.current_git_tag }} -DautoVersionSubmodules=true -Dresume=false -Darguments="-DskipTests -Dscala=2.12 -Dspark=2.4"
+mvn org.apache.maven.plugins:maven-release-plugin:2.3.2:perform -DconnectionUrl=scm:git:https://github.com/apache/incubator-sedona.git -Dtag={{ sedona.current_git_tag }} -DautoVersionSubmodules=true -Dresume=false -Darguments="-DskipTests -Dscala=2.13"
```
@@ -147,7 +150,7 @@ mvn org.apache.maven.plugins:maven-release-plugin:2.3.2:perform -DconnectionUrl=
All release candidates must be first placed in ASF Dist Dev SVN before vote: https://dist.apache.org/repos/dist/dev/incubator/sedona
-1. Make sure your armored PGP public key (must be encrypted by RSA-4096) is included in the `KEYS` file: https://dist.apache.org/repos/dist/dev/incubator/sedona/KEYS, and publish in major key servers: https://pgp.mit.edu/, https://keyserver.ubuntu.com/
+1. Make sure your armored PGP public key (must be encrypted by RSA-4096) is included in the `KEYS` file: https://dist.apache.org/repos/dist/dev/incubator/sedona/KEYS, and publish in major key servers: https://keyserver.pgp.com/
2. Create a folder on SVN, such as `{{ sedona.current_git_tag }}`
```bash
#!/bin/bash
@@ -160,23 +163,18 @@ git clone --shared --branch {{ sedona.current_git_tag}} https://github.com/apach
rm -rf apache-sedona-{{ sedona.current_version }}-src/.git
tar czf apache-sedona-{{ sedona.current_version }}-src.tar.gz apache-sedona-{{ sedona.current_version }}-src
mkdir apache-sedona-{{ sedona.current_version }}-bin
-cd apache-sedona-{{ sedona.current_version }}-src && mvn clean install -DskipTests -Dscala=2.12 -Dspark=3.0 && cd ..
+cd apache-sedona-{{ sedona.current_version }}-src && mvn clean install -DskipTests -Dscala=2.12 && cd ..
cp apache-sedona-{{ sedona.current_version }}-src/core/target/sedona-*{{ sedona.current_version}}.jar apache-sedona-{{ sedona.current_version }}-bin/
cp apache-sedona-{{ sedona.current_version }}-src/sql/target/sedona-*{{ sedona.current_version}}.jar apache-sedona-{{ sedona.current_version }}-bin/
cp apache-sedona-{{ sedona.current_version }}-src/viz/target/sedona-*{{ sedona.current_version}}.jar apache-sedona-{{ sedona.current_version }}-bin/
cp apache-sedona-{{ sedona.current_version }}-src/python-adapter/target/sedona-*{{ sedona.current_version}}.jar apache-sedona-{{ sedona.current_version }}-bin/
cp apache-sedona-{{ sedona.current_version }}-src/flink/target/sedona-*{{ sedona.current_version}}.jar apache-sedona-{{ sedona.current_version }}-bin/
-cd apache-sedona-{{ sedona.current_version }}-src && mvn clean install -DskipTests -Dscala=2.11 -Dspark=2.4 && cd ..
+cd apache-sedona-{{ sedona.current_version }}-src && mvn clean install -DskipTests -Dscala=2.13 && cd ..
cp apache-sedona-{{ sedona.current_version }}-src/core/target/sedona-*{{ sedona.current_version}}.jar apache-sedona-{{ sedona.current_version }}-bin/
cp apache-sedona-{{ sedona.current_version }}-src/sql/target/sedona-*{{ sedona.current_version}}.jar apache-sedona-{{ sedona.current_version }}-bin/
cp apache-sedona-{{ sedona.current_version }}-src/viz/target/sedona-*{{ sedona.current_version}}.jar apache-sedona-{{ sedona.current_version }}-bin/
cp apache-sedona-{{ sedona.current_version }}-src/python-adapter/target/sedona-*{{ sedona.current_version}}.jar apache-sedona-{{ sedona.current_version }}-bin/
cp apache-sedona-{{ sedona.current_version }}-src/flink/target/sedona-*{{ sedona.current_version}}.jar apache-sedona-{{ sedona.current_version }}-bin/
-cd apache-sedona-{{ sedona.current_version }}-src && mvn clean install -DskipTests -Dscala=2.12 -Dspark=2.4 && cd ..
-cp apache-sedona-{{ sedona.current_version }}-src/core/target/sedona-*{{ sedona.current_version}}.jar apache-sedona-{{ sedona.current_version }}-bin/
-cp apache-sedona-{{ sedona.current_version }}-src/sql/target/sedona-*{{ sedona.current_version}}.jar apache-sedona-{{ sedona.current_version }}-bin/
-cp apache-sedona-{{ sedona.current_version }}-src/viz/target/sedona-*{{ sedona.current_version}}.jar apache-sedona-{{ sedona.current_version }}-bin/
-cp apache-sedona-{{ sedona.current_version }}-src/python-adapter/target/sedona-*{{ sedona.current_version}}.jar apache-sedona-{{ sedona.current_version }}-bin/
tar czf apache-sedona-{{ sedona.current_version }}-bin.tar.gz apache-sedona-{{ sedona.current_version }}-bin
shasum -a 512 apache-sedona-{{ sedona.current_version }}-src.tar.gz > apache-sedona-{{ sedona.current_version }}-src.tar.gz.sha512
shasum -a 512 apache-sedona-{{ sedona.current_version }}-bin.tar.gz > apache-sedona-{{ sedona.current_version }}-bin.tar.gz.sha512
@@ -235,7 +233,12 @@ https://dist.apache.org/repos/dist/dev/incubator/sedona/{{ sedona.current_rc }}/
The vote will be open for at least 72 hours or until at least 3 "+1" PMC votes are cast
-Please vote accordingly:
+Instruction for checking items on the checklist: https://sedona.apache.org/community/vote/
+
+We recommend you use this Jupyter notebook on MyBinder to perform this task: https://mybinder.org/v2/gh/jiayuasu/sedona-tools/HEAD?labpath=binder%2Fverify-release.ipynb
+
+**Please vote accordingly and you must provide your checklist for your vote**.
+
[ ] +1 approve
@@ -243,7 +246,7 @@ Please vote accordingly:
[ ] -1 disapprove with the reason
-Checklist for reference (because of DISCLAIMER-WIP, other checklist items are not blockers):
+Checklist:
[ ] Download links are valid.
@@ -461,29 +464,19 @@ The staging repo on repository.apache.org is usually automatically closed before
mvn org.apache.maven.plugins:maven-release-plugin:2.3.2:perform -DconnectionUrl=scm:git:https://github.com/apache/incubator-sedona.git -Dtag={{ sedona.current_git_tag }} -DautoVersionSubmodules=true -Dresume=false -Darguments="-DskipTests"
```
-##### For Spark 2.4 and Scala 2.11
-
-```
-mvn org.apache.maven.plugins:maven-release-plugin:2.3.2:perform -DconnectionUrl=scm:git:https://github.com/apache/incubator-sedona.git -Dtag={{ sedona.current_git_tag }} -DautoVersionSubmodules=true -Dresume=false -Darguments="-DskipTests -Dscala=2.11 -Dspark=2.4"
-```
-
-##### For Spark 2.4 and Scala 2.12
+##### For Spark 3.0 and Scala 2.13
```
-mvn org.apache.maven.plugins:maven-release-plugin:2.3.2:perform -DconnectionUrl=scm:git:https://github.com/apache/incubator-sedona.git -Dtag={{ sedona.current_git_tag }} -DautoVersionSubmodules=true -Dresume=false -Darguments="-DskipTests -Dscala=2.12 -Dspark=2.4"
+mvn org.apache.maven.plugins:maven-release-plugin:2.3.2:perform -DconnectionUrl=scm:git:https://github.com/apache/incubator-sedona.git -Dtag={{ sedona.current_git_tag }} -DautoVersionSubmodules=true -Dresume=false -Darguments="-DskipTests -Dscala=2.13"
```
#### Fix the error when close the staged repo
-In the last step, you may see 6 errors similar to the following:
+In the last step, you may see many errors similar to the following:
```
typeId signature-staging
-failureMessage Invalid Signature: '/org/apache/sedona/sedona-python-adapter-2.4_2.12/{{ sedona.current_version }}/sedona-python-adapter-2.4_2.12-{{ sedona.current_version }}.pom.asc' is not a valid signature for 'sedona-python-adapter-2.4_2.12-{{ sedona.current_version }}.pom'.
-failureMessage Invalid Signature: '/org/apache/sedona/sedona-python-adapter-2.4_2.11/{{ sedona.current_version }}/sedona-python-adapter-2.4_2.11-{{ sedona.current_version }}.pom.asc' is not a valid signature for 'sedona-python-adapter-2.4_2.11-{{ sedona.current_version }}.pom'.
failureMessage Invalid Signature: '/org/apache/sedona/sedona-python-adapter-3.0_2.12/{{ sedona.current_version }}/sedona-python-adapter-3.0_2.12-{{ sedona.current_version }}.pom.asc' is not a valid signature for 'sedona-python-adapter-3.0_2.12-{{ sedona.current_version }}.pom'.
-failureMessage Invalid Signature: '/org/apache/sedona/sedona-viz-2.4_2.12/{{ sedona.current_version }}/sedona-viz-2.4_2.12-{{ sedona.current_version }}.pom.asc' is not a valid signature for 'sedona-viz-2.4_2.12-{{ sedona.current_version }}.pom'.
failureMessage Invalid Signature: '/org/apache/sedona/sedona-viz-3.0_2.12/{{ sedona.current_version }}/sedona-viz-3.0_2.12-{{ sedona.current_version }}.pom.asc' is not a valid signature for 'sedona-viz-3.0_2.12-{{ sedona.current_version }}.pom'.
-failureMessage Invalid Signature: '/org/apache/sedona/sedona-viz-2.4_2.11/{{ sedona.current_version }}/sedona-viz-2.4_2.11-{{ sedona.current_version }}.pom.asc' is not a valid signature for 'sedona-viz-2.4_2.11-{{ sedona.current_version }}.pom'.
```
This is caused by a bug in the resolved-pom-maven-plugin in POM.xml. You will have to upload the signatures of the POM files mannualy.
@@ -496,78 +489,59 @@ username=admin
password=admin123
stagingid=1016
-wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-core-2.4_2.11/{{ sedona.current_version }}/sedona-core-2.4_2.11-{{ sedona.current_version }}.pom
-wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-core-2.4_2.12/{{ sedona.current_version }}/sedona-core-2.4_2.12-{{ sedona.current_version }}.pom
+wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-common/{{ sedona.current_version }}/sedona-common-{{ sedona.current_version }}.pom
wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-core-3.0_2.12/{{ sedona.current_version }}/sedona-core-3.0_2.12-{{ sedona.current_version }}.pom
-wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-sql-2.4_2.11/{{ sedona.current_version }}/sedona-sql-2.4_2.11-{{ sedona.current_version }}.pom
-wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-sql-2.4_2.12/{{ sedona.current_version }}/sedona-sql-2.4_2.12-{{ sedona.current_version }}.pom
+wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-core-3.0_2.13/{{ sedona.current_version }}/sedona-core-3.0_2.13-{{ sedona.current_version }}.pom
wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-sql-3.0_2.12/{{ sedona.current_version }}/sedona-sql-3.0_2.12-{{ sedona.current_version }}.pom
-wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-viz-2.4_2.11/{{ sedona.current_version }}/sedona-viz-2.4_2.11-{{ sedona.current_version }}.pom
-wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-viz-2.4_2.12/{{ sedona.current_version }}/sedona-viz-2.4_2.12-{{ sedona.current_version }}.pom
+wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-sql-3.0_2.13/{{ sedona.current_version }}/sedona-sql-3.0_2.13-{{ sedona.current_version }}.pom
wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-viz-3.0_2.12/{{ sedona.current_version }}/sedona-viz-3.0_2.12-{{ sedona.current_version }}.pom
-wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-python-adapter-2.4_2.11/{{ sedona.current_version }}/sedona-python-adapter-2.4_2.11-{{ sedona.current_version }}.pom
-wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-python-adapter-2.4_2.12/{{ sedona.current_version }}/sedona-python-adapter-2.4_2.12-{{ sedona.current_version }}.pom
+wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-viz-3.0_2.13/{{ sedona.current_version }}/sedona-viz-3.0_2.13-{{ sedona.current_version }}.pom
wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-python-adapter-3.0_2.12/{{ sedona.current_version }}/sedona-python-adapter-3.0_2.12-{{ sedona.current_version }}.pom
+wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-python-adapter-3.0_2.13/{{ sedona.current_version }}/sedona-python-adapter-3.0_2.13-{{ sedona.current_version }}.pom
wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-flink_2.12/{{ sedona.current_version }}/sedona-flink_2.12-{{ sedona.current_version }}.pom
-wget https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-flink_2.11/{{ sedona.current_version }}/sedona-flink_2.11-{{ sedona.current_version }}.pom
-gpg -ab sedona-core-2.4_2.11-{{ sedona.current_version }}.pom
-gpg -ab sedona-core-2.4_2.12-{{ sedona.current_version }}.pom
+gpg -ab sedona-common-{{ sedona.current_version }}.pom
gpg -ab sedona-core-3.0_2.12-{{ sedona.current_version }}.pom
-gpg -ab sedona-sql-2.4_2.11-{{ sedona.current_version }}.pom
-gpg -ab sedona-sql-2.4_2.12-{{ sedona.current_version }}.pom
gpg -ab sedona-sql-3.0_2.12-{{ sedona.current_version }}.pom
-gpg -ab sedona-viz-2.4_2.11-{{ sedona.current_version }}.pom
-gpg -ab sedona-viz-2.4_2.12-{{ sedona.current_version }}.pom
gpg -ab sedona-viz-3.0_2.12-{{ sedona.current_version }}.pom
-gpg -ab sedona-python-adapter-2.4_2.11-{{ sedona.current_version }}.pom
-gpg -ab sedona-python-adapter-2.4_2.12-{{ sedona.current_version }}.pom
gpg -ab sedona-python-adapter-3.0_2.12-{{ sedona.current_version }}.pom
gpg -ab sedona-flink_2.12-{{ sedona.current_version }}.pom
-gpg -ab sedona-flink_2.11-{{ sedona.current_version }}.pom
+gpg -ab sedona-core-3.0_2.13-{{ sedona.current_version }}.pom
+gpg -ab sedona-sql-3.0_2.13-{{ sedona.current_version }}.pom
+gpg -ab sedona-viz-3.0_2.13-{{ sedona.current_version }}.pom
+gpg -ab sedona-python-adapter-3.0_2.13-{{ sedona.current_version }}.pom
-curl -v -u $username:$password --upload-file sedona-python-adapter-2.4_2.11-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-python-adapter-2.4_2.11/{{ sedona.current_version }}/sedona-python-adapter-2.4_2.11-{{ sedona.current_version }}.pom.asc
-curl -v -u $username:$password --upload-file sedona-python-adapter-2.4_2.12-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-python-adapter-2.4_2.12/{{ sedona.current_version }}/sedona-python-adapter-2.4_2.12-{{ sedona.current_version }}.pom.asc
+curl -v -u $username:$password --upload-file sedona-common-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-common/{{ sedona.current_version }}/sedona-common-{{ sedona.current_version }}.pom.asc
curl -v -u $username:$password --upload-file sedona-python-adapter-3.0_2.12-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-python-adapter-3.0_2.12/{{ sedona.current_version }}/sedona-python-adapter-3.0_2.12-{{ sedona.current_version }}.pom.asc
-curl -v -u $username:$password --upload-file sedona-viz-2.4_2.11-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-viz-2.4_2.11/{{ sedona.current_version }}/sedona-viz-2.4_2.11-{{ sedona.current_version }}.pom.asc
-
-curl -v -u $username:$password --upload-file sedona-viz-2.4_2.12-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-viz-2.4_2.12/{{ sedona.current_version }}/sedona-viz-2.4_2.12-{{ sedona.current_version }}.pom.asc
-
curl -v -u $username:$password --upload-file sedona-viz-3.0_2.12-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-viz-3.0_2.12/{{ sedona.current_version }}/sedona-viz-3.0_2.12-{{ sedona.current_version }}.pom.asc
-curl -v -u $username:$password --upload-file sedona-core-2.4_2.11-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-core-2.4_2.11/{{ sedona.current_version }}/sedona-core-2.4_2.11-{{ sedona.current_version }}.pom.asc
-
-curl -v -u $username:$password --upload-file sedona-core-2.4_2.12-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-core-2.4_2.12/{{ sedona.current_version }}/sedona-core-2.4_2.12-{{ sedona.current_version }}.pom.asc
-
curl -v -u $username:$password --upload-file sedona-core-3.0_2.12-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-core-3.0_2.12/{{ sedona.current_version }}/sedona-core-3.0_2.12-{{ sedona.current_version }}.pom.asc
-curl -v -u $username:$password --upload-file sedona-sql-2.4_2.11-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-sql-2.4_2.11/{{ sedona.current_version }}/sedona-sql-2.4_2.11-{{ sedona.current_version }}.pom.asc
-
-curl -v -u $username:$password --upload-file sedona-sql-2.4_2.12-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-sql-2.4_2.12/{{ sedona.current_version }}/sedona-sql-2.4_2.12-{{ sedona.current_version }}.pom.asc
-
curl -v -u $username:$password --upload-file sedona-sql-3.0_2.12-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-sql-3.0_2.12/{{ sedona.current_version }}/sedona-sql-3.0_2.12-{{ sedona.current_version }}.pom.asc
curl -v -u $username:$password --upload-file sedona-flink_2.12-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-flink_2.12/{{ sedona.current_version }}/sedona-flink_2.12-{{ sedona.current_version }}.pom.asc
-curl -v -u $username:$password --upload-file sedona-flink_2.11-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-flink_2.11/{{ sedona.current_version }}/sedona-flink_2.11-{{ sedona.current_version }}.pom.asc
+curl -v -u $username:$password --upload-file sedona-python-adapter-3.0_2.13-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-python-adapter-3.0_2.13/{{ sedona.current_version }}/sedona-python-adapter-3.0_2.12-{{ sedona.current_version }}.pom.asc
+
+curl -v -u $username:$password --upload-file sedona-viz-3.0_2.13-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-viz-3.0_2.13/{{ sedona.current_version }}/sedona-viz-3.0_2.13-{{ sedona.current_version }}.pom.asc
+
+curl -v -u $username:$password --upload-file sedona-core-3.0_2.13-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-core-3.0_2.13/{{ sedona.current_version }}/sedona-core-3.0_2.13-{{ sedona.current_version }}.pom.asc
+
+curl -v -u $username:$password --upload-file sedona-sql-3.0_2.13-{{ sedona.current_version }}.pom.asc https://repository.apache.org/service/local/repositories/orgapachesedona-$stagingid/content/org/apache/sedona/sedona-sql-3.0_2.13/{{ sedona.current_version }}/sedona-sql-3.0_2.13-{{ sedona.current_version }}.pom.asc
-rm sedona-core-2.4_2.11-{{ sedona.current_version }}.pom.asc
-rm sedona-core-2.4_2.12-{{ sedona.current_version }}.pom.asc
+rm sedona-common-{{ sedona.current_version }}.pom.asc
rm sedona-core-3.0_2.12-{{ sedona.current_version }}.pom.asc
-rm sedona-sql-2.4_2.11-{{ sedona.current_version }}.pom.asc
-rm sedona-sql-2.4_2.12-{{ sedona.current_version }}.pom.asc
rm sedona-sql-3.0_2.12-{{ sedona.current_version }}.pom.asc
-rm sedona-viz-2.4_2.11-{{ sedona.current_version }}.pom.asc
-rm sedona-viz-2.4_2.12-{{ sedona.current_version }}.pom.asc
rm sedona-viz-3.0_2.12-{{ sedona.current_version }}.pom.asc
-rm sedona-python-adapter-2.4_2.11-{{ sedona.current_version }}.pom.asc
-rm sedona-python-adapter-2.4_2.12-{{ sedona.current_version }}.pom.asc
rm sedona-python-adapter-3.0_2.12-{{ sedona.current_version }}.pom.asc
rm sedona-flink_2.12-{{ sedona.current_version }}.pom.asc
-rm sedona-flink_2.11-{{ sedona.current_version }}.pom.asc
+rm sedona-core-3.0_2.13-{{ sedona.current_version }}.pom.asc
+rm sedona-sql-3.0_2.13-{{ sedona.current_version }}.pom.asc
+rm sedona-viz-3.0_2.13-{{ sedona.current_version }}.pom.asc
+rm sedona-python-adapter-3.0_2.13-{{ sedona.current_version }}.pom.asc
```
admin is your Apache ID username and admin123 is your Apache ID password. You can find the correct upload path from the web interface.
diff --git a/docs/setup/compile.md b/docs/setup/compile.md
index 95fc1f99..d02da42b 100644
--- a/docs/setup/compile.md
+++ b/docs/setup/compile.md
@@ -35,15 +35,11 @@ To compile all modules, please make sure you are in the root folder of all modul
=== "Spark 3.0 + Scala 2.12"
```
- mvn clean install -DskipTests -Dscala=2.12 -Dspark=3.0
+ mvn clean install -DskipTests -Dscala=2.12
```
-=== "Spark 2.4 + Scala 2.11"
+=== "Spark 3.0 + Scala 2.13"
```
- mvn clean install -DskipTests -Dscala=2.11 -Dspark=2.4
- ```
-=== "Spark 2.4 + Scala 2.12"
- ```
- mvn clean install -DskipTests -Dscala=2.12 -Dspark=2.4
+ mvn clean install -DskipTests -Dscala=2.13
```
!!!tip
diff --git a/docs/setup/maven-coordinates.md b/docs/setup/maven-coordinates.md
index 41940a5e..1b4fd6b6 100644
--- a/docs/setup/maven-coordinates.md
+++ b/docs/setup/maven-coordinates.md
@@ -35,38 +35,17 @@ The optional GeoTools library is required only if you want to use CRS transforma
</dependency>
```
- === "Spark 2.4 and Scala 2.11"
+ === "Spark 3.0 and Scala 2.13"
```xml
<dependency>
<groupId>org.apache.sedona</groupId>
- <artifactId>sedona-python-adapter-2.4_2.11</artifactId>
+ <artifactId>sedona-python-adapter-3.0_2.13</artifactId>
<version>{{ sedona.current_version }}</version>
</dependency>
<dependency>
<groupId>org.apache.sedona</groupId>
- <artifactId>sedona-viz-2.4_2.11</artifactId>
- <version>{{ sedona.current_version }}</version>
- </dependency>
- <!-- Optional: https://mvnrepository.com/artifact/org.datasyslab/geotools-wrapper -->
- <dependency>
- <groupId>org.datasyslab</groupId>
- <artifactId>geotools-wrapper</artifactId>
- <version>{{ sedona.current_geotools }}</version>
- </dependency>
- ```
-
- === "Spark 2.4 and Scala 2.12"
-
- ```xml
- <dependency>
- <groupId>org.apache.sedona</groupId>
- <artifactId>sedona-python-adapter-2.4_2.12</artifactId>
- <version>{{ sedona.current_version }}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.sedona</groupId>
- <artifactId>sedona-viz-2.4_2.12</artifactId>
+ <artifactId>sedona-viz-3.0_2.13</artifactId>
<version>{{ sedona.current_version }}</version>
</dependency>
<!-- Optional: https://mvnrepository.com/artifact/org.datasyslab/geotools-wrapper -->
@@ -77,6 +56,7 @@ The optional GeoTools library is required only if you want to use CRS transforma
</dependency>
```
+
!!! abstract "Sedona with Apache Flink"
=== "Flink 1.12+ and Scala 2.12"
@@ -99,26 +79,6 @@ The optional GeoTools library is required only if you want to use CRS transforma
<version>{{ sedona.current_geotools }}</version>
</dependency>
```
- === "Flink 1.12+ and Scala 2.11"
-
- ```xml
- <dependency>
- <groupId>org.apache.sedona</groupId>
- <artifactId>sedona-python-adapter-3.0_2.11</artifactId>
- <version>{{ sedona.current_version }}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.sedona</groupId>
- <artifactId>sedona-flink_2.11</artifactId>
- <version>{{ sedona.current_version }}</version>
- </dependency>
- <!-- Optional: https://mvnrepository.com/artifact/org.datasyslab/geotools-wrapper -->
- <dependency>
- <groupId>org.datasyslab</groupId>
- <artifactId>geotools-wrapper</artifactId>
- <version>{{ sedona.current_geotools }}</version>
- </dependency>
- ```
#### SernetCDF 0.1.0
diff --git a/docs/setup/release-notes.md b/docs/setup/release-notes.md
index 1f4d75ef..8072aba3 100644
--- a/docs/setup/release-notes.md
+++ b/docs/setup/release-notes.md
@@ -1,11 +1,107 @@
!!!warning
Support of Spark 2.X and Scala 2.11 will be removed in Sedona 1.3.0+ although some parts of the source code might still be compatible. Sedona 1.3.0+ will release binary for both Scala 2.12 and 2.13.
+## Sedona 1.3.0
+
+This version is a major release on Sedona 1.3.0 line and consists of 46 PRs. It includes many new functions, optimization and bug fixes.
+
+### Highlights
+
+- [X] Sedona on Spark in this release is compiled against Spark 3.3.
+- [X] Sedona on Flink in this release is compiled against Flink 1.14.
+- [X] Scala 2.11 support is removed.
+- [X] Spark 2.X support is removed.
+- [X] Python 3.10 support is added.
+- [X] Aggregators in Flink are added
+- [X] Correctness fixes for corner cases in range join and distance join.
+- [X] Native GeoParquet read and write.
+ * `df = spark.read.format("geoparquet").option("fieldGeometry", "myGeometryColumn").load("PATH/TO/MYFILE.parquet")`
+ * `df.write.format("geoparquet").save("PATH/TO/MYFILE.parquet")`
+- [X] DataFrame style API
+ * `df.select(ST_Point(min_value, max_value).as("point"))`
+- [X] Allow WKT format CRS in ST_Transform
+ * `ST_Transform(geom, "srcWktString", "tgtWktString")`
+
+```yaml
+
+GEOGCS["WGS 84",
+ DATUM["WGS_1984",
+ SPHEROID["WGS 84",6378137,298.257223563,
+ AUTHORITY["EPSG","7030"]],
+ AUTHORITY["EPSG","6326"]],
+ PRIMEM["Greenwich",0,
+ AUTHORITY["EPSG","8901"]],
+ UNIT["degree",0.0174532925199433,
+ AUTHORITY["EPSG","9122"]],
+ AUTHORITY["EPSG","4326"]]
+
+```
+
+### Bug fixes
+
+ * [SEDONA-119](https://issues.apache.org/jira/browse/SEDONA-119) - ST_Touches join query returns true for polygons whose interiors intersect
+ * [SEDONA-136](https://issues.apache.org/jira/browse/SEDONA-136) - Enable testAsEWKT for Flink
+ * [SEDONA-137](https://issues.apache.org/jira/browse/SEDONA-137) - Fix ST_Buffer for Flink to work
+ * [SEDONA-138](https://issues.apache.org/jira/browse/SEDONA-138) - Fix ST_GeoHash for Flink to work
+ * [SEDONA-153](https://issues.apache.org/jira/browse/SEDONA-153) - Python Serialization Fails with Nulls
+ * [SEDONA-158](https://issues.apache.org/jira/browse/SEDONA-158) - Fix wrong description about ST_GeometryN in the API docs
+ * [SEDONA-169](https://issues.apache.org/jira/browse/SEDONA-169) - Fix ST_RemovePoint in accordance with the API document
+ * [SEDONA-178](https://issues.apache.org/jira/browse/SEDONA-178) - Correctness issue in distance join queries
+ * [SEDONA-182](https://issues.apache.org/jira/browse/SEDONA-182) - ST_AsText should not return SRID
+ * [SEDONA-186](https://issues.apache.org/jira/browse/SEDONA-186) - collecting result rows of a spatial join query with SELECT * fails with serde error
+ * [SEDONA-188](https://issues.apache.org/jira/browse/SEDONA-188) - Python warns about missing `jars` even when some are found
+
+### New Features
+
+ * [SEDONA-94](https://issues.apache.org/jira/browse/SEDONA-94) - GeoParquet Support For Sedona
+ * [SEDONA-125](https://issues.apache.org/jira/browse/SEDONA-125) - Allows customized CRS in ST_Transform
+ * [SEDONA-166](https://issues.apache.org/jira/browse/SEDONA-166) - Provide Type-safe DataFrame Style API
+ * [SEDONA-168](https://issues.apache.org/jira/browse/SEDONA-168) - Add ST_Normalize to Apache Sedona
+ * [SEDONA-171](https://issues.apache.org/jira/browse/SEDONA-171) - Add ST_SetPoint to Apache Sedona
+
+
+### Improvement
+
+ * [SEDONA-121](https://issues.apache.org/jira/browse/SEDONA-121) - Add equivalent constructors left over from Spark to Flink
+ * [SEDONA-132](https://issues.apache.org/jira/browse/SEDONA-132) - Create common module for SQL functions
+ * [SEDONA-133](https://issues.apache.org/jira/browse/SEDONA-133) - Allow user-defined schemas in Adapter.toDf()
+ * [SEDONA-139](https://issues.apache.org/jira/browse/SEDONA-139) - Fix wrong argument order in Flink unit tests
+ * [SEDONA-140](https://issues.apache.org/jira/browse/SEDONA-140) - Update Sedona Dependencies in R Package
+ * [SEDONA-143](https://issues.apache.org/jira/browse/SEDONA-143) - Add missing unit tests for the Flink predicates
+ * [SEDONA-144](https://issues.apache.org/jira/browse/SEDONA-144) - Add ST_AsGeoJSON to the Flink API
+ * [SEDONA-145](https://issues.apache.org/jira/browse/SEDONA-145) - Fix ST_AsEWKT to reserve the Z coordinate
+ * [SEDONA-146](https://issues.apache.org/jira/browse/SEDONA-146) - Add missing output funtions to the Flink API
+ * [SEDONA-147](https://issues.apache.org/jira/browse/SEDONA-147) - Add SRID functions to the Flink API
+ * [SEDONA-148](https://issues.apache.org/jira/browse/SEDONA-148) - Add boolean functions to the Flink API
+ * [SEDONA-149](https://issues.apache.org/jira/browse/SEDONA-149) - Add Python 3.10 support
+ * [SEDONA-151](https://issues.apache.org/jira/browse/SEDONA-151) - Add ST aggregators to Sedona Flink
+ * [SEDONA-152](https://issues.apache.org/jira/browse/SEDONA-152) - Add reader/writer functions for GML and KML
+ * [SEDONA-154](https://issues.apache.org/jira/browse/SEDONA-154) - Add measurement functions to the Flink API
+ * [SEDONA-157](https://issues.apache.org/jira/browse/SEDONA-157) - Add coordinate accessors to the Flink API
+ * [SEDONA-159](https://issues.apache.org/jira/browse/SEDONA-159) - Add Nth accessor functions to the Flink API
+ * [SEDONA-160](https://issues.apache.org/jira/browse/SEDONA-160) - Fix geoparquetIOTests.scala to cleanup after test
+ * [SEDONA-161](https://issues.apache.org/jira/browse/SEDONA-161) - Add ST_Boundary to the Flink API
+ * [SEDONA-162](https://issues.apache.org/jira/browse/SEDONA-162) - Add ST_Envelope to the Flink API
+ * [SEDONA-163](https://issues.apache.org/jira/browse/SEDONA-163) - Better handle of unsupported types in shapefile reader
+ * [SEDONA-164](https://issues.apache.org/jira/browse/SEDONA-164) - Add geometry count functions to the Flink API
+ * [SEDONA-165](https://issues.apache.org/jira/browse/SEDONA-165) - Upgrade Apache Rat to 0.14
+ * [SEDONA-170](https://issues.apache.org/jira/browse/SEDONA-170) - Add ST_AddPoint and ST_RemovePoint to the Flink API
+ * [SEDONA-172](https://issues.apache.org/jira/browse/SEDONA-172) - Add ST_LineFromMultiPoint to Apache Sedona
+ * [SEDONA-176](https://issues.apache.org/jira/browse/SEDONA-176) - Make ST_Contains conform with OGC standard, and add ST_Covers and ST_CoveredBy functions.
+ * [SEDONA-177](https://issues.apache.org/jira/browse/SEDONA-177) - Support spatial predicates other than INTERSECTS and COVERS/COVERED_BY in RangeQuery.SpatialRangeQuery and JoinQuery.SpatialJoinQuery
+ * [SEDONA-181](https://issues.apache.org/jira/browse/SEDONA-181) - Build fails with java.lang.IllegalAccessError: class org.apache.spark.storage.StorageUtils$
+ * [SEDONA-189](https://issues.apache.org/jira/browse/SEDONA-189) - Prepare geometries in broadcast join
+
+### Task
+
+* [SEDONA-150](https://issues.apache.org/jira/browse/SEDONA-150) - Drop Spark 2.4 and Scala 2.11 support
+
+
## Sedona 1.2.1
This version is a maintenance release on Sedona 1.2.0 line. It includes bug fixes.
-Sedona on Spark is now compiled against Spark 3.3, instead of Spark 2.2.
+Sedona on Spark is now compiled against Spark 3.3, instead of Spark 3.2.
### SQL (for Spark)
diff --git a/flink/pom.xml b/flink/pom.xml
index 7bdbd409..abd838b1 100644
--- a/flink/pom.xml
+++ b/flink/pom.xml
@@ -38,11 +38,6 @@
</properties>
<dependencies>
- <dependency>
- <groupId>org.apache.sedona</groupId>
- <artifactId>sedona-common</artifactId>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.sedona</groupId>
<artifactId>sedona-core-${spark.compat.version}_${scala.compat.version}</artifactId>
diff --git a/mkdocs.yml b/mkdocs.yml
index d2995f57..36b710ee 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -161,12 +161,12 @@ extra:
- icon: fontawesome/brands/twitter
link: 'https://twitter.com/ApacheSedona'
sedona:
- current_version: 1.2.1-incubating
- current_git_tag: sedona-1.2.1-incubating-rc1
- current_rc: 1.2.1-incubating-rc1
- current_snapshot: 1.2.1-incubating-SNAPSHOT
- next_version: 1.3.0-incubating
- current_geotools: 1.1.0-25.2
+ current_version: 1.3.0-incubating
+ current_git_tag: sedona-1.3.0-incubating-rc1
+ current_rc: 1.3.0-incubating-rc1
+ current_snapshot: 1.3.0-incubating-SNAPSHOT
+ next_version: 1.3.1-incubating
+ current_geotools: 1.3.0-27.2
copyright: Copyright © 2022 The Apache Software Foundation
markdown_extensions:
- admonition
diff --git a/sql/pom.xml b/sql/pom.xml
index c3de7f99..781939d6 100644
--- a/sql/pom.xml
+++ b/sql/pom.xml
@@ -37,11 +37,6 @@
</properties>
<dependencies>
- <dependency>
- <groupId>org.apache.sedona</groupId>
- <artifactId>sedona-common</artifactId>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.sedona</groupId>
<artifactId>sedona-core-${spark.compat.version}_${scala.compat.version}</artifactId>
diff --git a/sql/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala b/sql/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala
index 5e7e2359..15a6239b 100644
--- a/sql/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala
+++ b/sql/src/main/scala/org/apache/sedona/sql/UDF/Catalog.scala
@@ -19,13 +19,12 @@
package org.apache.sedona.sql.UDF
import org.apache.spark.sql.catalyst.FunctionIdentifier
-import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, ExpressionInfo}
-import org.apache.spark.sql.catalyst.expressions.Literal
import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder
-import org.apache.spark.sql.expressions.{Aggregator, UserDefinedAggregateFunction}
-import org.apache.spark.sql.sedona_sql.expressions.{ST_YMax, ST_YMin, _}
+import org.apache.spark.sql.catalyst.expressions.{ExpectsInputTypes, Expression, ExpressionInfo, Literal}
+import org.apache.spark.sql.expressions.Aggregator
import org.apache.spark.sql.sedona_sql.expressions.collect.{ST_Collect, ST_CollectionExtract}
-import org.apache.spark.sql.sedona_sql.expressions.raster.{RS_Add, RS_Append, RS_Array, RS_Base64, RS_BitwiseAnd, RS_BitwiseOr, RS_Count, RS_Divide, RS_FetchRegion, RS_GetBand, RS_GreaterThan, RS_GreaterThanEqual, RS_HTML, RS_LessThan, RS_LessThanEqual, RS_LogicalDifference, RS_LogicalOver, RS_Mean, RS_Mode, RS_Modulo, RS_Multiply, RS_MultiplyFactor, RS_Normalize, RS_NormalizedDifference, RS_SquareRoot, RS_Subtract}
+import org.apache.spark.sql.sedona_sql.expressions.raster._
+import org.apache.spark.sql.sedona_sql.expressions._
import org.locationtech.jts.geom.Geometry
import org.locationtech.jts.operation.buffer.BufferParameters
@@ -169,13 +168,6 @@ object Catalog {
new ST_Intersection_Aggr
)
- import org.apache.spark.sql.sedona_sql.expressions_udaf
- val aggregateExpressions_UDAF: Seq[UserDefinedAggregateFunction] = Seq(
- new expressions_udaf.ST_Union_Aggr,
- new expressions_udaf.ST_Envelope_Aggr,
- new expressions_udaf.ST_Intersection_Aggr
- )
-
private def function[T <: Expression : ClassTag](defaultArgs: Any *): FunctionDescription = {
val classTag = implicitly[ClassTag[T]]
val constructor = classTag.runtimeClass.getConstructor(classOf[Seq[Expression]])
diff --git a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions_udaf/AggregateFunctions.scala b/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions_udaf/AggregateFunctions.scala
deleted file mode 100644
index f527d0b7..00000000
--- a/sql/src/main/scala/org/apache/spark/sql/sedona_sql/expressions_udaf/AggregateFunctions.scala
+++ /dev/null
@@ -1,218 +0,0 @@
-/**
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.spark.sql.sedona_sql.expressions_udaf
-
-import org.apache.spark.sql.Row
-import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction}
-import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT
-import org.apache.spark.sql.types.{DataType, StructField, StructType}
-import org.locationtech.jts.geom.{Coordinate, Geometry, GeometryFactory}
-
-/**
- * Return the polygon union of all Polygon in the given column
- */
-
-class ST_Union_Aggr extends UserDefinedAggregateFunction {
- override def inputSchema: StructType = StructType(StructField("Union", GeometryUDT) :: Nil)
-
- override def bufferSchema: StructType = StructType(
- StructField("Union", GeometryUDT) :: Nil
- )
-
- override def dataType: DataType = GeometryUDT
-
- override def deterministic: Boolean = true
-
- override def initialize(buffer: MutableAggregationBuffer): Unit = {
- val coordinates: Array[Coordinate] = new Array[Coordinate](5)
- coordinates(0) = new Coordinate(-999999999, -999999999)
- coordinates(1) = new Coordinate(-999999999, -999999999)
- coordinates(2) = new Coordinate(-999999999, -999999999)
- coordinates(3) = new Coordinate(-999999999, -999999999)
- coordinates(4) = coordinates(0)
- val geometryFactory = new GeometryFactory()
- buffer(0) = geometryFactory.createPolygon(coordinates)
- }
-
- override def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
- val accumulateUnion = buffer.getAs[Geometry](0)
- val newPolygon = input.getAs[Geometry](0)
- if (accumulateUnion.getArea == 0) buffer(0) = newPolygon
- else buffer(0) = accumulateUnion.union(newPolygon)
- }
-
- override def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
- val leftPolygon = buffer1.getAs[Geometry](0)
- val rightPolygon = buffer2.getAs[Geometry](0)
- if (leftPolygon.getCoordinates()(0).x == -999999999) buffer1(0) = rightPolygon
- else if (rightPolygon.getCoordinates()(0).x == -999999999) buffer1(0) = leftPolygon
- else buffer1(0) = leftPolygon.union(rightPolygon)
- }
-
- override def evaluate(buffer: Row): Any = {
- return buffer.getAs[Geometry](0)
- }
-}
-
-/**
- * Return the envelope boundary of the entire column
- */
-class ST_Envelope_Aggr extends UserDefinedAggregateFunction {
- // This is the input fields for your aggregate function.
- override def inputSchema: org.apache.spark.sql.types.StructType =
- StructType(StructField("Envelope", GeometryUDT) :: Nil)
-
- // This is the internal fields you keep for computing your aggregate.
- override def bufferSchema: StructType = StructType(
- StructField("Envelope", GeometryUDT) :: Nil
- )
-
- // This is the output type of your aggregatation function.
- override def dataType: DataType = GeometryUDT
-
- override def deterministic: Boolean = true
-
- // This is the initial value for your buffer schema.
- override def initialize(buffer: MutableAggregationBuffer): Unit = {
- val coordinates: Array[Coordinate] = new Array[Coordinate](5)
- coordinates(0) = new Coordinate(-999999999, -999999999)
- coordinates(1) = new Coordinate(-999999999, -999999999)
- coordinates(2) = new Coordinate(-999999999, -999999999)
- coordinates(3) = new Coordinate(-999999999, -999999999)
- coordinates(4) = new Coordinate(-999999999, -999999999)
- val geometryFactory = new GeometryFactory()
- buffer(0) = geometryFactory.createPolygon(coordinates)
- //buffer(0) = new GenericArrayData(GeometrySerializer.serialize(geometryFactory.createPolygon(coordinates)))
- }
-
- // This is how to update your buffer schema given an input.
- override def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
- val accumulateEnvelope = buffer.getAs[Geometry](0).getEnvelopeInternal
- val newEnvelope = input.getAs[Geometry](0).getEnvelopeInternal
- val coordinates: Array[Coordinate] = new Array[Coordinate](5)
- var minX = 0.0
- var minY = 0.0
- var maxX = 0.0
- var maxY = 0.0
- if (accumulateEnvelope.getMinX == -999999999) {
- // Found the accumulateEnvelope is the initial value
- minX = newEnvelope.getMinX
- minY = newEnvelope.getMinY
- maxX = newEnvelope.getMaxX
- maxY = newEnvelope.getMaxY
- }
- else {
- minX = Math.min(accumulateEnvelope.getMinX, newEnvelope.getMinX)
- minY = Math.min(accumulateEnvelope.getMinY, newEnvelope.getMinY)
- maxX = Math.max(accumulateEnvelope.getMaxX, newEnvelope.getMaxX)
- maxY = Math.max(accumulateEnvelope.getMaxY, newEnvelope.getMaxY)
- }
- coordinates(0) = new Coordinate(minX, minY)
- coordinates(1) = new Coordinate(minX, maxY)
- coordinates(2) = new Coordinate(maxX, maxY)
- coordinates(3) = new Coordinate(maxX, minY)
- coordinates(4) = coordinates(0)
- val geometryFactory = new GeometryFactory()
- buffer(0) = geometryFactory.createPolygon(coordinates)
- }
-
- // This is how to merge two objects with the bufferSchema type.
- override def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
- val leftEnvelope = buffer1.getAs[Geometry](0).getEnvelopeInternal
- val rightEnvelope = buffer2.getAs[Geometry](0).getEnvelopeInternal
- val coordinates: Array[Coordinate] = new Array[Coordinate](5)
- var minX = 0.0
- var minY = 0.0
- var maxX = 0.0
- var maxY = 0.0
- if (leftEnvelope.getMinX == -999999999) {
- // Found the leftEnvelope is the initial value
- minX = rightEnvelope.getMinX
- minY = rightEnvelope.getMinY
- maxX = rightEnvelope.getMaxX
- maxY = rightEnvelope.getMaxY
- }
- else if (rightEnvelope.getMinX == -999999999) {
- // Found the rightEnvelope is the initial value
- minX = leftEnvelope.getMinX
- minY = leftEnvelope.getMinY
- maxX = leftEnvelope.getMaxX
- maxY = leftEnvelope.getMaxY
- }
- else {
- minX = Math.min(leftEnvelope.getMinX, rightEnvelope.getMinX)
- minY = Math.min(leftEnvelope.getMinY, rightEnvelope.getMinY)
- maxX = Math.max(leftEnvelope.getMaxX, rightEnvelope.getMaxX)
- maxY = Math.max(leftEnvelope.getMaxY, rightEnvelope.getMaxY)
- }
- coordinates(0) = new Coordinate(minX, minY)
- coordinates(1) = new Coordinate(minX, maxY)
- coordinates(2) = new Coordinate(maxX, maxY)
- coordinates(3) = new Coordinate(maxX, minY)
- coordinates(4) = coordinates(0)
- val geometryFactory = new GeometryFactory()
- buffer1(0) = geometryFactory.createPolygon(coordinates)
- }
-
- // This is where you output the final value, given the final value of your bufferSchema.
- override def evaluate(buffer: Row): Any = {
- return buffer.getAs[Geometry](0)
- }
-}
-
-/**
- * Return the polygon intersection of all Polygon in the given column
- */
-class ST_Intersection_Aggr extends UserDefinedAggregateFunction {
- override def inputSchema: StructType = StructType(StructField("Intersection", GeometryUDT) :: Nil)
-
- override def bufferSchema: StructType = StructType(
- StructField("Intersection", GeometryUDT) :: Nil
- )
-
- override def dataType: DataType = GeometryUDT
-
- override def deterministic: Boolean = true
-
- override def initialize(buffer: MutableAggregationBuffer): Unit = {
- val coordinates: Array[Coordinate] = new Array[Coordinate](5)
- coordinates(0) = new Coordinate(-999999999, -999999999)
- coordinates(1) = new Coordinate(-999999999, -999999999)
- coordinates(2) = new Coordinate(-999999999, -999999999)
- coordinates(3) = new Coordinate(-999999999, -999999999)
- coordinates(4) = new Coordinate(-999999999, -999999999)
- val geometryFactory = new GeometryFactory()
- buffer(0) = geometryFactory.createPolygon(coordinates)
- }
-
- override def update(buffer: MutableAggregationBuffer, input: Row): Unit = {
- val accumulateIntersection = buffer.getAs[Geometry](0)
- val newPolygon = input.getAs[Geometry](0)
- if (accumulateIntersection.getArea == 0) buffer(0) = newPolygon
- else buffer(0) = accumulateIntersection.intersection(newPolygon)
- }
-
- override def merge(buffer1: MutableAggregationBuffer, buffer2: Row): Unit = {
- val leftPolygon = buffer1.getAs[Geometry](0)
- val rightPolygon = buffer2.getAs[Geometry](0)
- if (leftPolygon.getCoordinates()(0).x == -999999999) buffer1(0) = rightPolygon
- else if (rightPolygon.getCoordinates()(0).x == -999999999) buffer1(0) = leftPolygon
- else buffer1(0) = leftPolygon.intersection(rightPolygon)
- }
-
- override def evaluate(buffer: Row): Any = {
- buffer.getAs[Geometry](0)
- }
-}
\ No newline at end of file
diff --git a/viz/pom.xml b/viz/pom.xml
index 4a4530f8..b427a8c4 100644
--- a/viz/pom.xml
+++ b/viz/pom.xml
@@ -37,11 +37,6 @@
</properties>
<dependencies>
- <dependency>
- <groupId>org.apache.sedona</groupId>
- <artifactId>sedona-common</artifactId>
- <version>${project.version}</version>
- </dependency>
<dependency>
<groupId>org.apache.sedona</groupId>
<artifactId>sedona-core-${spark.compat.version}_${scala.compat.version}</artifactId>