You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2023/01/02 06:21:21 UTC
[incubator-sedona] branch master updated: [SEDONA-212] Rework dependency management (#735)
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sedona.git
The following commit(s) were added to refs/heads/master by this push:
new 91af7113 [SEDONA-212] Rework dependency management (#735)
91af7113 is described below
commit 91af71136d8bc96684ade11615f67ade08b8be64
Author: Adam Binford <ad...@gmail.com>
AuthorDate: Mon Jan 2 01:20:23 2023 -0500
[SEDONA-212] Rework dependency management (#735)
---
.github/workflows/java.yml | 11 +-
.github/workflows/python.yml | 2 +-
common/pom.xml | 21 +-
core/pom.xml | 106 +++++++--
flink-shaded/.gitignore | 12 +
flink-shaded/pom.xml | 105 +++++++++
flink/pom.xml | 96 ++++----
pom.xml | 475 ++++++++++++++++++++-------------------
python-adapter/pom.xml | 118 ++++------
python/sedona/core/jvm/config.py | 2 +-
spark-shaded/.gitignore | 12 +
spark-shaded/pom.xml | 163 ++++++++++++++
sql/pom.xml | 105 ++++++---
viz/pom.xml | 89 ++++++--
14 files changed, 891 insertions(+), 426 deletions(-)
diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
index 7c53d67c..e6ee2d9a 100644
--- a/.github/workflows/java.yml
+++ b/.github/workflows/java.yml
@@ -18,32 +18,26 @@ jobs:
include:
- spark: 3.3.0
scala: 2.13.8
- jackson: 2.13.3
jdk: '8'
skipTests: ''
- spark: 3.3.0
scala: 2.12.15
- jackson: 2.13.3
jdk: '8'
skipTests: ''
- spark: 3.3.0
scala: 2.12.15
- jackson: 2.13.3
jdk: '11'
skipTests: '-DskipTests'
- spark: 3.2.0
scala: 2.12.15
- jackson: 2.12.3
jdk: '8'
skipTests: ''
- spark: 3.1.2
scala: 2.12.15
- jackson: 2.10.0
jdk: '8'
skipTests: ''
- spark: 3.0.3
scala: 2.12.15
- jackson: 2.10.0
jdk: '8'
skipTests: ''
@@ -65,9 +59,8 @@ jobs:
- env:
SPARK_VERSION: ${{ matrix.spark }}
SCALA_VERSION: ${{ matrix.scala }}
- JACKSON_VERSION: ${{ matrix.jackson }}
- SKIP_TESTS: ${{ matrix.skipTests}}
- run: mvn -q clean install -Dscala=${SCALA_VERSION:0:4} -Dspark.version=${SPARK_VERSION} -Dsedona.jackson.version=${JACKSON_VERSION} ${SKIP_TESTS}
+ SKIP_TESTS: ${{ matrix.skipTests }}
+ run: mvn -q clean install -Dscala=${SCALA_VERSION:0:4} -Dspark.version=${SPARK_VERSION} ${SKIP_TESTS}
- run: mkdir staging
- run: cp core/target/sedona-*.jar staging
- run: cp sql/target/sedona-*.jar staging
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index f9804082..0d5cc9b4 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -83,7 +83,7 @@ jobs:
- env:
SPARK_VERSION: ${{ matrix.spark }}
HADOOP_VERSION: ${{ matrix.hadoop }}
- run: find python-adapter/target -name sedona-* -exec cp {} spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}/jars/ \;
+ run: find spark-shaded/target -name sedona-*.jar -exec cp {} spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}/jars/ \;
- env:
SPARK_VERSION: ${{ matrix.spark }}
HADOOP_VERSION: ${{ matrix.hadoop }}
diff --git a/common/pom.xml b/common/pom.xml
index cac05ad5..3a796d1a 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -33,6 +33,26 @@
<packaging>jar</packaging>
<dependencies>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-main</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-referencing</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-epsg-hsql</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.locationtech.jts</groupId>
+ <artifactId>jts-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.wololo</groupId>
+ <artifactId>jts2geojson</artifactId>
+ </dependency>
</dependencies>
<build>
<sourceDirectory>src/main/java</sourceDirectory>
@@ -63,4 +83,3 @@
</profile>
</profiles>
</project>
-
diff --git a/core/pom.xml b/core/pom.xml
index 91aba13c..71578019 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -41,46 +41,118 @@
<groupId>org.apache.sedona</groupId>
<artifactId>sedona-common</artifactId>
<version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
- <!-- Test -->
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.compat.version}</artifactId>
+ <exclusions>
+ <!-- Make sure Hadoop's guava version is used for older versions of Spark tests -->
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <!-- These break the tests involving the mini HDFS cluster -->
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client-runtime</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.compat.version}</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-minicluster</artifactId>
- <version>${hadoop.version}</version>
- <scope>test</scope>
+ <artifactId>hadoop-client</artifactId>
</dependency>
<dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
- <version>1.8.5</version>
- <scope>test</scope>
+ <groupId>edu.ucar</groupId>
+ <artifactId>cdm-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-main</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-referencing</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-epsg-hsql</artifactId>
</dependency>
<dependency>
<groupId>org.geotools</groupId>
<artifactId>gt-shapefile</artifactId>
- <version>${geotools.version}</version>
- <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.locationtech.jts</groupId>
+ <artifactId>jts-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.wololo</groupId>
+ <artifactId>jts2geojson</artifactId>
<exclusions>
- <exclusion>
- <groupId>org.locationtech.jts</groupId>
- <artifactId>jts-core</artifactId>
- </exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>*</artifactId>
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ </dependency>
+
+ <!-- Test -->
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.compat.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <version>1.8.5</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
<sourceDirectory>src/main/java</sourceDirectory>
<plugins>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.scalastyle</groupId>
+ <artifactId>scalastyle-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
-
diff --git a/flink-shaded/.gitignore b/flink-shaded/.gitignore
new file mode 100644
index 00000000..1cc6c4a1
--- /dev/null
+++ b/flink-shaded/.gitignore
@@ -0,0 +1,12 @@
+/target/
+/.settings/
+/.classpath
+/.project
+/dependency-reduced-pom.xml
+/doc/
+/.idea/
+*.iml
+/latest/
+/spark-warehouse/
+/metastore_db/
+*.log
diff --git a/flink-shaded/pom.xml b/flink-shaded/pom.xml
new file mode 100644
index 00000000..5c8844ce
--- /dev/null
+++ b/flink-shaded/pom.xml
@@ -0,0 +1,105 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>sedona-parent</artifactId>
+ <groupId>org.apache.sedona</groupId>
+ <version>1.3.2-incubating-SNAPSHOT</version>
+ </parent>
+ <artifactId>sedona-flink-shaded_${scala.compat.version}</artifactId>
+
+ <name>${project.groupId}:${project.artifactId}</name>
+ <description>A cluster computing system for processing large-scale spatial data: Flink Shaded. Apache Sedona is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necess [...]
+ <url>http://sedona.apache.org/</url>
+ <packaging>jar</packaging>
+
+ <properties>
+ <maven.deploy.skip>false</maven.deploy.skip>
+ <cdm.scope>compile</cdm.scope>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.sedona</groupId>
+ <artifactId>sedona-flink_${scala.compat.version}</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <!-- Define this to be shaded since it is provided in other modules -->
+ <dependency>
+ <groupId>edu.ucar</groupId>
+ <artifactId>cdm-core</artifactId>
+ </dependency>
+ </dependencies>
+ <build>
+ <sourceDirectory>src/main/scala</sourceDirectory>
+ <plugins>
+ <plugin>
+ <groupId>io.paradoxical</groupId>
+ <artifactId>resolved-pom-maven-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>resolve-my-pom</id>
+ <phase>none</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <transformers>
+ <!-- use transformer to handle merge of META-INF/services - see http://java.net/jira/browse/JERSEY-440?focusedCommentId=14822&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#action_14822 -->
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+ <resource>reference.conf</resource>
+ </transformer>
+ </transformers>
+ <artifactSet>
+ <excludes>
+ <exclude>org.scala-lang:scala-library</exclude>
+ </excludes>
+ </artifactSet>
+ <filters>
+ <!-- filter to address "Invalid signature file" issue - see http://stackoverflow.com/a/6743609/589215 -->
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/flink/pom.xml b/flink/pom.xml
index c8238803..8814f658 100644
--- a/flink/pom.xml
+++ b/flink/pom.xml
@@ -35,73 +35,77 @@
<properties>
<maven.deploy.skip>false</maven.deploy.skip>
<flink.version>1.14.3</flink.version>
+ <flink.scope>provided</flink.scope>
</properties>
<dependencies>
+ <dependency>
+ <groupId>org.apache.sedona</groupId>
+ <artifactId>sedona-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.sedona</groupId>
<artifactId>sedona-core-${spark.compat.version}_${scala.compat.version}</artifactId>
<version>${project.version}</version>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.sedona</groupId>
<artifactId>sedona-sql-${spark.compat.version}_${scala.compat.version}</artifactId>
<version>${project.version}</version>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-core</artifactId>
<version>${flink.version}</version>
- <scope>${dependency.scope}</scope>
+ <scope>${flink.scope}</scope>
</dependency>
<!-- For Flink DataStream API-->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-streaming-java_${scala.compat.version}</artifactId>
<version>${flink.version}</version>
- <scope>${dependency.scope}</scope>
+ <scope>${flink.scope}</scope>
</dependency>
<!-- Flink Kafka connector-->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-connector-kafka_${scala.compat.version}</artifactId>
<version>${flink.version}</version>
- <scope>${dependency.scope}</scope>
+ <scope>${flink.scope}</scope>
</dependency>
<!-- For playing flink in IDE-->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-clients_${scala.compat.version}</artifactId>
<version>${flink.version}</version>
- <scope>${dependency.scope}</scope>
+ <scope>${flink.scope}</scope>
</dependency>
<!-- For Flink flink api, planner, udf/udt, csv-->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-api-java-bridge_${scala.compat.version}</artifactId>
<version>${flink.version}</version>
- <scope>${dependency.scope}</scope>
+ <scope>${flink.scope}</scope>
</dependency>
<!-- Starting Flink 14, Blink planner has been renamed to the official Flink planner-->
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-planner_${scala.compat.version}</artifactId>
<version>${flink.version}</version>
- <scope>${dependency.scope}</scope>
+ <scope>${flink.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-table-common</artifactId>
<version>${flink.version}</version>
- <scope>${dependency.scope}</scope>
+ <scope>${flink.scope}</scope>
</dependency>
<dependency>
<groupId>org.apache.flink</groupId>
<artifactId>flink-csv</artifactId>
<version>${flink.version}</version>
- <scope>${dependency.scope}</scope>
+ <scope>${flink.scope}</scope>
</dependency>
<!-- For Flink Web Ui in test-->
<dependency>
@@ -110,6 +114,38 @@
<version>${flink.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-main</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-referencing</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-epsg-hsql</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.compat.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.reload4j</groupId>
+ <artifactId>reload4j</artifactId>
+ <version>1.2.21</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ <version>1.15</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.jheaps</groupId>
<artifactId>jheaps</artifactId>
@@ -120,38 +156,18 @@
<build>
<plugins>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.scalastyle</groupId>
+ <artifactId>scalastyle-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
-<!-- <build>-->
-<!-- <sourceDirectory>src/main/scala</sourceDirectory>-->
-<!-- <plugins>-->
-<!-- <plugin>-->
-<!-- <groupId>org.scalastyle</groupId>-->
-<!-- <artifactId>scalastyle-maven-plugin</artifactId>-->
-<!-- <version>1.0.0</version>-->
-<!-- <configuration>-->
-<!-- <verbose>false</verbose>-->
-<!-- <failOnViolation>true</failOnViolation>-->
-<!-- <includeTestSourceDirectory>true</includeTestSourceDirectory>-->
-<!-- <failOnWarning>false</failOnWarning>-->
-<!-- <sourceDirectory>${project.basedir}/src/main/scala</sourceDirectory>-->
-<!-- <testSourceDirectory>${project.basedir}/src/test/scala</testSourceDirectory>-->
-<!-- <configLocation>${project.basedir}/../scalastyle_config.xml</configLocation>-->
-<!-- <outputFile>${project.basedir}/target/scalastyle-output.xml</outputFile>-->
-<!-- <outputEncoding>UTF-8</outputEncoding>-->
-<!-- </configuration>-->
-<!-- <executions>-->
-<!-- <execution>-->
-<!-- <goals>-->
-<!-- <goal>check</goal>-->
-<!-- </goals>-->
-<!-- </execution>-->
-<!-- </executions>-->
-<!-- </plugin>-->
-<!-- </plugins>-->
-<!-- </build>-->
</project>
diff --git a/pom.xml b/pom.xml
index 90a2db44..5b360b35 100644
--- a/pom.xml
+++ b/pom.xml
@@ -59,184 +59,199 @@
<properties>
<project.scm.id>github</project.scm.id>
- <geotools.version>24.0</geotools.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <dependency.scope>provided</dependency.scope>
+ <java.version>1.8</java.version>
+ <maven.compiler.source>${java.version}</maven.compiler.source>
+ <maven.compiler.target>${java.version}</maven.compiler.target>
+ <maven.deploy.skip>false</maven.deploy.skip>
+ <maven.compiler.plugin.version>3.10.1</maven.compiler.plugin.version>
+
+ <cdm.version>5.4.2</cdm.version>
+ <geotools.version>24.0</geotools.version>
+ <hadoop.version>3.2.4</hadoop.version>
+ <jackson.version>2.13.4</jackson.version>
<jts.version>1.19.0</jts.version>
<jts2geojson.version>0.16.1</jts2geojson.version>
- <netcdf.version>5.4.2</netcdf.version>
<spark.version>3.3.0</spark.version>
<spark.compat.version>3.0</spark.compat.version>
- <sedona.jackson.version>2.13.3</sedona.jackson.version>
- <hadoop.version>3.2.4</hadoop.version>
- <maven.deploy.skip>false</maven.deploy.skip>
- <maven.compiler.plugin.version>3.10.1</maven.compiler.plugin.version>
+
<!-- Actual scala version will be set by a profile.
Setting a default value helps IDE:s that can't make sense of profiles. -->
<scala.compat.version>2.12</scala.compat.version>
+
+ <geotools.scope>provided</geotools.scope>
+ <!-- Because it's not in Maven central, make it provided by default -->
+ <cdm.scope>provided</cdm.scope>
</properties>
<dependencies>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-databind</artifactId>
- <version>${sedona.jackson.version}</version>
- <scope>${dependency.scope}</scope>
- </dependency>
- <dependency>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>jackson-core</artifactId>
- <version>${sedona.jackson.version}</version>
- <scope>${dependency.scope}</scope>
- </dependency>
- <dependency>
- <groupId>org.wololo</groupId>
- <artifactId>jts2geojson</artifactId>
- <version>${jts2geojson.version}</version>
- <exclusions>
- <exclusion>
- <groupId>org.locationtech.jts</groupId>
- <artifactId>jts-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- <scope>${dependency.scope}</scope>
- </dependency>
- <dependency>
- <groupId>org.locationtech.jts</groupId>
- <artifactId>jts-core</artifactId>
- <version>${jts.version}</version>
- <scope>${dependency.scope}</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- <version>${hadoop.version}</version>
- <scope>${dependency.scope}</scope>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>edu.ucar</groupId>
- <artifactId>cdm-core</artifactId>
- <version>${netcdf.version}</version>
- <scope>${dependency.scope}</scope>
- </dependency>
- <!--The following GeoTools dependencies use GNU Lesser General Public License and thus are excluded from the binary distribution-->
- <!-- Users have to include them by themselves manually -->
- <!-- See https://www.apache.org/legal/resolved.html#category-x -->
- <!-- See https://github.com/geotools/geotools#license -->
- <!--for CRS transformation-->
- <dependency>
- <groupId>org.geotools</groupId>
- <artifactId>gt-main</artifactId>
- <version>${geotools.version}</version>
- <scope>${dependency.scope}</scope>
- <exclusions>
- <exclusion>
- <groupId>org.locationtech.jts</groupId>
- <artifactId>jts-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <!--for CRS transformation-->
- <dependency>
- <groupId>org.geotools</groupId>
- <artifactId>gt-referencing</artifactId>
- <version>${geotools.version}</version>
- <scope>${dependency.scope}</scope>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <!--for CRS transformation-->
- <dependency>
- <groupId>org.geotools</groupId>
- <artifactId>gt-epsg-hsql</artifactId>
- <version>${geotools.version}</version>
- <scope>${dependency.scope}</scope>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <!--for GeoTiff Reader-->
- <dependency>
- <groupId>org.geotools</groupId>
- <artifactId>gt-geotiff</artifactId>
- <version>${geotools.version}</version>
- <scope>${dependency.scope}</scope>
- </dependency>
- <dependency>
- <groupId>org.geotools</groupId>
- <artifactId>gt-coverage</artifactId>
- <version>${geotools.version}</version>
- <scope>${dependency.scope}</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.compat.version}</artifactId>
- <version>${spark.version}</version>
- <scope>${dependency.scope}</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.compat.version}</artifactId>
- <version>${spark.version}</version>
- <scope>${dependency.scope}</scope>
- </dependency>
- <!-- Test -->
+ <!-- Include junit automatically, projects must include scalatest themselves since not all use -->
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <version>4.13.1</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest_${scala.compat.version}</artifactId>
- <version>3.1.1</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-compiler</artifactId>
- <version>${scala.version}</version>
- <scope>test</scope>
</dependency>
</dependencies>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.wololo</groupId>
+ <artifactId>jts2geojson</artifactId>
+ <version>${jts2geojson.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.locationtech.jts</groupId>
+ <artifactId>jts-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.locationtech.jts</groupId>
+ <artifactId>jts-core</artifactId>
+ <version>${jts.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>edu.ucar</groupId>
+ <artifactId>cdm-core</artifactId>
+ <version>${cdm.version}</version>
+ <scope>${cdm.scope}</scope>
+ </dependency>
+ <!--The following GeoTools dependencies use GNU Lesser General Public License and thus are excluded from the binary distribution-->
+ <!-- Users have to include them by themselves manually -->
+ <!-- See https://www.apache.org/legal/resolved.html#category-x -->
+ <!-- See https://github.com/geotools/geotools#license -->
+ <!--for CRS transformation-->
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-main</artifactId>
+ <version>${geotools.version}</version>
+ <scope>${geotools.scope}</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.locationtech.jts</groupId>
+ <artifactId>jts-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!--for CRS transformation-->
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-referencing</artifactId>
+ <version>${geotools.version}</version>
+ <scope>${geotools.scope}</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!--for CRS transformation-->
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-epsg-hsql</artifactId>
+ <version>${geotools.version}</version>
+ <scope>${geotools.scope}</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-shapefile</artifactId>
+ <version>${geotools.version}</version>
+ <scope>${geotools.scope}</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <!--for GeoTiff Reader-->
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-geotiff</artifactId>
+ <version>${geotools.version}</version>
+ <scope>${geotools.scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-coverage</artifactId>
+ <version>${geotools.version}</version>
+ <scope>${geotools.scope}</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.compat.version}</artifactId>
+ <version>${spark.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.compat.version}</artifactId>
+ <version>${spark.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>2.6</version>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ <version>${scala.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-compiler</artifactId>
+ <version>${scala.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+ <artifactId>scala-collection-compat_${scala.compat.version}</artifactId>
+ <version>2.5.0</version>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.13.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.compat.version}</artifactId>
+ <version>3.1.1</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
<repositories>
- <repository>
- <id>maven2-repository.dev.java.net</id>
- <name>Java.net repository</name>
- <url>https://download.java.net/maven/2</url>
- </repository>
<repository>
<id>osgeo</id>
<name>OSGeo Release Repository</name>
@@ -325,6 +340,52 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>3.3.0</version>
+ </plugin>
+ <plugin>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest-maven-plugin</artifactId>
+ <version>2.2.0</version>
+ <configuration>
+ <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
+ <junitxml>.</junitxml>
+ <filereports>TestSuiteReport.txt</filereports>
+ </configuration>
+ <executions>
+ <execution>
+ <id>test</id>
+ <goals>
+ <goal>test</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.scalastyle</groupId>
+ <artifactId>scalastyle-maven-plugin</artifactId>
+ <version>1.0.0</version>
+ <configuration>
+ <verbose>false</verbose>
+ <failOnViolation>true</failOnViolation>
+ <includeTestSourceDirectory>true</includeTestSourceDirectory>
+ <failOnWarning>false</failOnWarning>
+ <sourceDirectory>${project.basedir}/src/main/scala</sourceDirectory>
+ <testSourceDirectory>${project.basedir}/src/test/scala</testSourceDirectory>
+ <configLocation>${project.basedir}/../scalastyle_config.xml</configLocation>
+ <outputFile>${project.basedir}/target/scalastyle-output.xml</outputFile>
+ <outputEncoding>UTF-8</outputEncoding>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</pluginManagement>
<plugins>
@@ -359,52 +420,28 @@
</algorithms>
</configuration>
</plugin>
- <plugin>
- <!-- see http://davidb.github.com/scala-maven-plugin -->
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <version>3.2.1</version>
- </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
+ <!-- This plugin is to replace property values in artifact names when publish -->
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <version>2.1</version>
+ <groupId>io.paradoxical</groupId>
+ <artifactId>resolved-pom-maven-plugin</artifactId>
+ <version>1.0</version>
+ <configuration>
+ <properties>
+ <spark.compat.version>${spark.compat.version}</spark.compat.version>
+ <scala.compat.version>${scala.compat.version}</scala.compat.version>
+ </properties>
+ </configuration>
<executions>
<execution>
+ <id>resolve-my-pom</id>
<phase>package</phase>
<goals>
- <goal>shade</goal>
+ <goal>resolve-pom</goal>
</goals>
- <configuration>
- <transformers>
- <!-- use transformer to handle merge of META-INF/services - see http://java.net/jira/browse/JERSEY-440?focusedCommentId=14822&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#action_14822 -->
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
- <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>reference.conf</resource>
- </transformer>
- </transformers>
- <filters>
- <!-- filter to address "Invalid signature file" issue - see http://stackoverflow.com/a/6743609/589215 -->
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>edu.ucar:cdm-core</artifact>
- <excludes>
- <exclude>com/fasterxml/**</exclude>
- </excludes>
- </filter>
- </filters>
- </configuration>
</execution>
</executions>
</plugin>
@@ -413,24 +450,6 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>2.18.1</version>
</plugin>
- <plugin>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest-maven-plugin</artifactId>
- <version>2.2.0</version>
- <configuration>
- <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
- <junitxml>.</junitxml>
- <filereports>TestSuiteReport.txt</filereports>
- </configuration>
- <executions>
- <execution>
- <id>test</id>
- <goals>
- <goal>test</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
@@ -448,27 +467,6 @@
<additionalparam>-Xdoclint:none</additionalparam>
</configuration>
</plugin>
- <!-- This plugin is to replace property values in artifact names when publish -->
- <plugin>
- <groupId>io.paradoxical</groupId>
- <artifactId>resolved-pom-maven-plugin</artifactId>
- <version>1.0</version>
- <configuration>
- <properties>
- <spark.compat.version>${spark.compat.version}</spark.compat.version>
- <scala.compat.version>${scala.compat.version}</scala.compat.version>
- </properties>
- </configuration>
- <executions>
- <execution>
- <id>resolve-my-pom</id>
- <phase>test</phase>
- <goals>
- <goal>resolve-pom</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
@@ -508,6 +506,8 @@
<scala.version>2.13.8</scala.version>
<scala.compat.version>2.13</scala.compat.version>
<scaladoc.arg>-no-java-comments</scaladoc.arg>
+ <!-- Skip deploying parent module for Scala 2.13 profile, it will be deployed with 2.12 -->
+ <maven.deploy.skip>true</maven.deploy.skip>
</properties>
<modules>
<module>common</module>
@@ -515,6 +515,7 @@
<module>sql</module>
<module>viz</module>
<module>python-adapter</module>
+ <module>spark-shaded</module>
</modules>
</profile>
<profile>
@@ -537,7 +538,9 @@
<module>sql</module>
<module>viz</module>
<module>python-adapter</module>
+ <module>spark-shaded</module>
<module>flink</module>
+ <module>flink-shaded</module>
</modules>
</profile>
</profiles>
diff --git a/python-adapter/pom.xml b/python-adapter/pom.xml
index c96646af..23622e56 100644
--- a/python-adapter/pom.xml
+++ b/python-adapter/pom.xml
@@ -38,14 +38,15 @@
<dependencies>
<dependency>
- <groupId>org.locationtech.jts</groupId>
- <artifactId>jts-core</artifactId>
- <version>${jts.version}</version>
- </dependency>
- <dependency>
- <groupId>org.wololo</groupId>
- <artifactId>jts2geojson</artifactId>
- <version>${jts2geojson.version}</version>
+ <groupId>org.apache.sedona</groupId>
+ <artifactId>sedona-common</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.sedona</groupId>
@@ -57,90 +58,65 @@
<artifactId>sedona-sql-${spark.compat.version}_${scala.compat.version}</artifactId>
<version>${project.version}</version>
</dependency>
- <!--for CRS transformation-->
- <dependency>
- <groupId>org.datasyslab</groupId>
- <artifactId>geotools-wrapper</artifactId>
- <version>geotools-${geotools.version}</version>
- <scope>${geotools.scope}</scope>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.compat.version}</artifactId>
<exclusions>
+ <!-- Make sure Hadoop's guava version is used for older versions of Spark tests -->
<exclusion>
- <groupId>org.locationtech.jts</groupId>
- <artifactId>jts-core</artifactId>
- </exclusion>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>*</artifactId>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
</exclusion>
</exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.compat.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.locationtech.jts</groupId>
+ <artifactId>jts-core</artifactId>
</dependency>
- <!--for CRS transformation-->
<dependency>
<groupId>org.geotools</groupId>
- <artifactId>gt-epsg-extension</artifactId>
- <version>${geotools.version}</version>
- <scope>${geotools.scope}</scope>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
+ <artifactId>gt-main</artifactId>
</dependency>
- <!--for GeoTiff Reader-->
<dependency>
<groupId>org.geotools</groupId>
- <artifactId>gt-geotiff</artifactId>
- <version>${geotools.version}</version>
- <scope>${geotools.scope}</scope>
+ <artifactId>gt-epsg-hsql</artifactId>
</dependency>
<dependency>
- <groupId>org.geotools</groupId>
- <artifactId>gt-coverage</artifactId>
- <version>${geotools.version}</version>
- <scope>${geotools.scope}</scope>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+ <artifactId>scala-collection-compat_${scala.compat.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.compat.version}</artifactId>
</dependency>
</dependencies>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<plugins>
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest-maven-plugin</artifactId>
+ </plugin>
<plugin>
<groupId>org.scalastyle</groupId>
<artifactId>scalastyle-maven-plugin</artifactId>
- <version>1.0.0</version>
- <configuration>
- <verbose>false</verbose>
- <failOnViolation>true</failOnViolation>
- <includeTestSourceDirectory>true</includeTestSourceDirectory>
- <failOnWarning>false</failOnWarning>
- <sourceDirectory>${project.basedir}/src/main/scala</sourceDirectory>
- <testSourceDirectory>${project.basedir}/src/test/scala</testSourceDirectory>
- <configLocation>${project.basedir}/../scalastyle_config.xml</configLocation>
- <outputFile>${project.basedir}/target/scalastyle-output.xml</outputFile>
- <outputEncoding>UTF-8</outputEncoding>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
</plugin>
</plugins>
</build>
- <profiles>
- <profile>
- <id>geotools</id>
- <activation>
- <property>
- <name>geotools</name>
- </property>
- <activeByDefault>false</activeByDefault>
- </activation>
- <properties>
- <geotools.scope>compile</geotools.scope>
- </properties>
- </profile>
- </profiles>
</project>
diff --git a/python/sedona/core/jvm/config.py b/python/sedona/core/jvm/config.py
index 0359be2a..64669590 100644
--- a/python/sedona/core/jvm/config.py
+++ b/python/sedona/core/jvm/config.py
@@ -143,7 +143,7 @@ class SedonaMeta:
def get_version(cls, spark_jars: str) -> Optional[str]:
# Find Spark version, Scala version and Sedona version.
versions = findall(
- r"sedona-python-adapter-([^,\n]+)_([^,\n]+)-([^,\n]+)-incubating",
+ r"sedona-(?:python-adapter|spark-shaded)-([^,\n]+)_([^,\n]+)-([^,\n]+)-incubating",
spark_jars,
)
try:
diff --git a/spark-shaded/.gitignore b/spark-shaded/.gitignore
new file mode 100644
index 00000000..1cc6c4a1
--- /dev/null
+++ b/spark-shaded/.gitignore
@@ -0,0 +1,12 @@
+/target/
+/.settings/
+/.classpath
+/.project
+/dependency-reduced-pom.xml
+/doc/
+/.idea/
+*.iml
+/latest/
+/spark-warehouse/
+/metastore_db/
+*.log
diff --git a/spark-shaded/pom.xml b/spark-shaded/pom.xml
new file mode 100644
index 00000000..9c6786f7
--- /dev/null
+++ b/spark-shaded/pom.xml
@@ -0,0 +1,163 @@
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>sedona-parent</artifactId>
+ <groupId>org.apache.sedona</groupId>
+ <version>1.3.2-incubating-SNAPSHOT</version>
+ </parent>
+ <artifactId>sedona-spark-shaded-${spark.compat.version}_${scala.compat.version}</artifactId>
+
+ <name>${project.groupId}:${project.artifactId}</name>
+ <description>A cluster computing system for processing large-scale spatial data: Spark Shaded. Apache Sedona is an effort undergoing incubation at The Apache Software Foundation (ASF), sponsored by the Apache Incubator. Incubation is required of all newly accepted projects until a further review indicates that the infrastructure, communications, and decision making process have stabilized in a manner consistent with other successful ASF projects. While incubation status is not necess [...]
+ <url>http://sedona.apache.org/</url>
+ <packaging>jar</packaging>
+
+ <properties>
+ <maven.deploy.skip>false</maven.deploy.skip>
+ <geotools.scope>provided</geotools.scope>
+ <cdm.scope>compile</cdm.scope>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.sedona</groupId>
+ <artifactId>sedona-common</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.sedona</groupId>
+ <artifactId>sedona-core-${spark.compat.version}_${scala.compat.version}</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.sedona</groupId>
+ <artifactId>sedona-sql-${spark.compat.version}_${scala.compat.version}</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.sedona</groupId>
+ <artifactId>sedona-python-adapter-${spark.compat.version}_${scala.compat.version}</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <!-- Define this to be shaded since it is provided in other modules -->
+ <dependency>
+ <groupId>edu.ucar</groupId>
+ <artifactId>cdm-core</artifactId>
+ </dependency>
+ <!-- Define these so they get included in shaded jar for python testing purposes -->
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-main</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-referencing</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-epsg-hsql</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-geotiff</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-coverage</artifactId>
+ </dependency>
+ </dependencies>
+ <build>
+ <sourceDirectory>src/main/scala</sourceDirectory>
+ <plugins>
+ <plugin>
+ <groupId>io.paradoxical</groupId>
+ <artifactId>resolved-pom-maven-plugin</artifactId>
+ <version>1.0</version>
+ <executions>
+ <execution>
+ <id>resolve-my-pom</id>
+ <phase>none</phase>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.1</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <transformers>
+ <!-- use transformer to handle merge of META-INF/services - see http://java.net/jira/browse/JERSEY-440?focusedCommentId=14822&page=com.atlassian.jira.plugin.system.issuetabpanels%3Acomment-tabpanel#action_14822 -->
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+ <resource>reference.conf</resource>
+ </transformer>
+ </transformers>
+ <artifactSet>
+ <excludes>
+ <exclude>org.scala-lang:scala-library</exclude>
+ </excludes>
+ </artifactSet>
+ <filters>
+ <!-- filter to address "Invalid signature file" issue - see http://stackoverflow.com/a/6743609/589215 -->
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <profiles>
+ <!-- Used for Python tests to bundle geotools in the shaded jar -->
+ <profile>
+ <id>geotools</id>
+ <activation>
+ <property>
+ <name>geotools</name>
+ </property>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <geotools.scope>compile</geotools.scope>
+ </properties>
+ </profile>
+ </profiles>
+</project>
diff --git a/sql/pom.xml b/sql/pom.xml
index dd7d17ed..f77d3a14 100644
--- a/sql/pom.xml
+++ b/sql/pom.xml
@@ -37,51 +37,104 @@
</properties>
<dependencies>
+ <dependency>
+ <groupId>org.apache.sedona</groupId>
+ <artifactId>sedona-common</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>org.apache.sedona</groupId>
<artifactId>sedona-core-${spark.compat.version}_${scala.compat.version}</artifactId>
<version>${project.version}</version>
- <scope>provided</scope>
</dependency>
+
<dependency>
- <groupId>org.scala-lang.modules</groupId>
- <artifactId>scala-collection-compat_${scala.compat.version}</artifactId>
- <version>2.5.0</version>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.compat.version}</artifactId>
<exclusions>
+ <!-- Make sure Hadoop's guava version is used for older versions of Spark tests -->
<exclusion>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.compat.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-main</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-referencing</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-epsg-hsql</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-geotiff</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-coverage</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.locationtech.jts</groupId>
+ <artifactId>jts-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.wololo</groupId>
+ <artifactId>jts2geojson</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+ <artifactId>scala-collection-compat_${scala.compat.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.compat.version}</artifactId>
+ </dependency>
</dependencies>
<build>
<sourceDirectory>src/main/scala</sourceDirectory>
<plugins>
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest-maven-plugin</artifactId>
+ </plugin>
<plugin>
<groupId>org.scalastyle</groupId>
<artifactId>scalastyle-maven-plugin</artifactId>
- <version>1.0.0</version>
- <configuration>
- <verbose>false</verbose>
- <failOnViolation>true</failOnViolation>
- <includeTestSourceDirectory>true</includeTestSourceDirectory>
- <failOnWarning>false</failOnWarning>
- <sourceDirectory>${project.basedir}/src/main/scala</sourceDirectory>
- <testSourceDirectory>${project.basedir}/src/test/scala</testSourceDirectory>
- <configLocation>${project.basedir}/../scalastyle_config.xml</configLocation>
- <outputFile>${project.basedir}/target/scalastyle-output.xml</outputFile>
- <outputEncoding>UTF-8</outputEncoding>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
</plugin>
</plugins>
</build>
</project>
-
diff --git a/viz/pom.xml b/viz/pom.xml
index 25855bb9..fe9c0630 100644
--- a/viz/pom.xml
+++ b/viz/pom.xml
@@ -37,17 +37,69 @@
</properties>
<dependencies>
+ <dependency>
+ <groupId>org.apache.sedona</groupId>
+ <artifactId>sedona-common</artifactId>
+ <version>${project.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>org.apache.sedona</groupId>
<artifactId>sedona-core-${spark.compat.version}_${scala.compat.version}</artifactId>
<version>${project.version}</version>
- <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.sedona</groupId>
<artifactId>sedona-sql-${spark.compat.version}_${scala.compat.version}</artifactId>
<version>${project.version}</version>
- <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.compat.version}</artifactId>
+ <exclusions>
+ <!-- Make sure Hadoop's guava version is used for older versions of Spark tests -->
+ <exclusion>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.compat.version}</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.locationtech.jts</groupId>
+ <artifactId>jts-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-main</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-referencing</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.geotools</groupId>
+ <artifactId>gt-epsg-hsql</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.scala-lang.modules</groupId>
+ <artifactId>scala-collection-compat_${scala.compat.version}</artifactId>
</dependency>
<dependency>
<groupId>org.beryx</groupId>
@@ -58,7 +110,7 @@
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-s3</artifactId>
<version>1.12.362</version>
- <scope>${dependency.scope}</scope>
+ <scope>provided</scope>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
@@ -66,36 +118,25 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest_${scala.compat.version}</artifactId>
+ </dependency>
</dependencies>
<build>
<sourceDirectory>src/main/java</sourceDirectory>
<plugins>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-compiler-plugin</artifactId>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.scalatest</groupId>
+ <artifactId>scalatest-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.scalastyle</groupId>
<artifactId>scalastyle-maven-plugin</artifactId>
- <version>1.0.0</version>
- <configuration>
- <verbose>false</verbose>
- <failOnViolation>true</failOnViolation>
- <includeTestSourceDirectory>true</includeTestSourceDirectory>
- <failOnWarning>false</failOnWarning>
- <sourceDirectory>${project.basedir}/src/main/scala</sourceDirectory>
- <testSourceDirectory>${project.basedir}/src/test/scala</testSourceDirectory>
- <configLocation>${project.basedir}/../scalastyle_config.xml</configLocation>
- <outputFile>${project.basedir}/target/scalastyle-output.xml</outputFile>
- <outputEncoding>UTF-8</outputEncoding>
- </configuration>
- <executions>
- <execution>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
</plugin>
</plugins>
</build>