You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2022/08/22 03:54:39 UTC
[incubator-sedona] branch drop-spark-2.4 updated: Drop Spark 2.4 and Scala 2.11 support
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch drop-spark-2.4
in repository https://gitbox.apache.org/repos/asf/incubator-sedona.git
The following commit(s) were added to refs/heads/drop-spark-2.4 by this push:
new fa0c20c4 Drop Spark 2.4 and Scala 2.11 support
fa0c20c4 is described below
commit fa0c20c46842c75958fb265fbbc5e3612a6ab2a8
Author: Jia Yu <ji...@apache.org>
AuthorDate: Sun Aug 21 20:54:33 2022 -0700
Drop Spark 2.4 and Scala 2.11 support
---
.github/workflows/java.yml | 14 +-------
.github/workflows/python.yml | 4 ---
.github/workflows/r.yml | 30 ++--------------
common/pom.xml | 19 +++++++---
pom.xml | 86 +++-----------------------------------------
5 files changed, 24 insertions(+), 129 deletions(-)
diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
index e088f312..c8712a95 100644
--- a/.github/workflows/java.yml
+++ b/.github/workflows/java.yml
@@ -18,9 +18,6 @@ jobs:
- spark: 3.3.0
scala: 2.13.8
jackson: 2.13.3
- - spark: 3.2.0
- scala: 2.13.5
- jackson: 2.12.3
- spark: 3.3.0
scala: 2.12.15
jackson: 2.13.3
@@ -33,12 +30,6 @@ jobs:
- spark: 3.0.3
scala: 2.12.15
jackson: 2.10.0
- - spark: 2.4.8
- scala: 2.12.15
- jackson: 2.6.7
- - spark: 2.4.8
- scala: 2.11.8
- jackson: 2.6.7
steps:
- uses: actions/checkout@v2
@@ -59,10 +50,7 @@ jobs:
SPARK_VERSION: ${{ matrix.spark }}
SCALA_VERSION: ${{ matrix.scala }}
JACKSON_VERSION: ${{ matrix.jackson }}
- run: |
- if [ ${SPARK_VERSION:0:1} == "3" ]; \
- then mvn -q clean install -Dscala=${SCALA_VERSION:0:4} -Dspark=3.0 -Dspark.version=${SPARK_VERSION} -Dsedona.jackson.version=${JACKSON_VERSION}; \
- else mvn -q clean install -Dscala=${SCALA_VERSION:0:4} -Dspark=2.4 -Dspark.version=${SPARK_VERSION} -Dsedona.jackson.version=${JACKSON_VERSION}; fi \
+ run: mvn -q clean install -Dscala=${SCALA_VERSION:0:4} -Dspark.version=${SPARK_VERSION} -Dsedona.jackson.version=${JACKSON_VERSION}; \
- run: mkdir staging
- run: cp core/target/sedona-*.jar staging
- run: cp sql/target/sedona-*.jar staging
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index b7ea8ab5..f9804082 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -43,10 +43,6 @@ jobs:
scala: '2.12.8'
python: '3.7'
hadoop: '2.7'
- - spark: '2.4.8'
- scala: '2.11.8'
- python: '3.7'
- hadoop: '2.7'
steps:
- uses: actions/checkout@v2
diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml
index 71e97d37..507aea9f 100644
--- a/.github/workflows/r.yml
+++ b/.github/workflows/r.yml
@@ -15,34 +15,10 @@ jobs:
strategy:
fail-fast: true
matrix:
- spark: [2.4.8, 3.0.3, 3.1.2, 3.2.1]
- scala: [2.11.8, 2.12.15]
+ spark: [3.0.3, 3.1.2, 3.2.1, 3.3.0]
+ scala: [2.12.15]
r: [oldrel, release]
- exclude:
- - spark: 3.2.1
- scala: 2.11.8
- r: oldrel
- - spark: 3.2.1
- scala: 2.11.8
- r: release
- - spark: 3.1.2
- scala: 2.11.8
- r: oldrel
- - spark: 3.1.2
- scala: 2.11.8
- r: release
- - spark: 3.0.3
- scala: 2.11.8
- r: oldrel
- - spark: 3.0.3
- scala: 2.11.8
- r: release
- - spark: 2.4.8
- scala: 2.12.15
- r: oldrel
- - spark: 2.4.8
- scala: 2.12.15
- r: release
+
env:
SPARK_VERSION: ${{ matrix.spark }}
SCALA_VERSION: ${{ matrix.scala }}
diff --git a/common/pom.xml b/common/pom.xml
index b85b885a..2babab2a 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -32,10 +32,6 @@
<url>http://sedona.apache.org/</url>
<packaging>jar</packaging>
- <properties>
- <maven.deploy.skip>false</maven.deploy.skip>
- </properties>
-
<dependencies>
</dependencies>
<build>
@@ -51,5 +47,20 @@
</plugin>
</plugins>
</build>
+ <profiles>
+ <profile>
+ <id>common-scala2.13</id>
+ <activation>
+ <property>
+ <name>scala</name>
+ <value>2.13</value>
+ </property>
+ <activeByDefault>false</activeByDefault>
+ </activation>
+ <properties>
+ <maven.deploy.skip>true</maven.deploy.skip>
+ </properties>
+ </profile>
+ </profiles>
</project>
diff --git a/pom.xml b/pom.xml
index aae83e16..680e227d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -63,6 +63,11 @@
<dependency.scope>provided</dependency.scope>
<jts.version>1.18.2</jts.version>
<jts2geojson.version>0.16.1</jts2geojson.version>
+ <spark.version>3.3.0</spark.version>
+ <spark.compat.version>3.0</spark.compat.version>
+ <sedona.jackson.version>2.13.3</sedona.jackson.version>
+ <hadoop.version>3.2.4</hadoop.version>
+ <maven.deploy.skip>false</maven.deploy.skip>
</properties>
<dependencies>
@@ -464,28 +469,6 @@
<artifactId>apache-rat-plugin</artifactId>
<version>0.13</version>
</plugin>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>exec-maven-plugin</artifactId>
- <version>3.0.0</version>
- <executions>
- <execution>
- <configuration>
- <executable>python3</executable>
- <workingDirectory>${maven.multiModuleProjectDirectory}</workingDirectory>
- <arguments>
- <argument>spark-version-converter.py</argument>
- <argument>${spark.converter.version}</argument>
- </arguments>
- </configuration>
- <id>python_build</id>
- <phase>generate-resources</phase>
- <goals>
- <goal>exec</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
</plugins>
</build>
<profiles>
@@ -507,42 +490,6 @@
</repository>
</repositories>
</profile>
- <profile>
- <id>spark3.0</id>
- <activation>
- <property>
- <name>spark</name>
- <value>3.0</value>
- </property>
- <activeByDefault>true</activeByDefault>
- </activation>
- <properties>
- <spark.version>3.3.0</spark.version>
- <spark.compat.version>3.0</spark.compat.version>
- <spark.converter.version>spark3</spark.converter.version>
- <sedona.jackson.version>2.13.3</sedona.jackson.version>
- <hadoop.version>3.2.4</hadoop.version>
- <maven.deploy.skip>false</maven.deploy.skip>
- </properties>
- </profile>
- <profile>
- <id>spark2.4</id>
- <activation>
- <property>
- <name>spark</name>
- <value>2.4</value>
- </property>
- <activeByDefault>false</activeByDefault>
- </activation>
- <properties>
- <spark.version>2.4.7</spark.version>
- <spark.compat.version>2.4</spark.compat.version>
- <spark.converter.version>spark2</spark.converter.version>
- <sedona.jackson.version>2.6.7</sedona.jackson.version>
- <hadoop.version>2.10.2</hadoop.version>
- <maven.deploy.skip>true</maven.deploy.skip>
- </properties>
- </profile>
<profile>
<id>scala2.13</id>
<activation>
@@ -588,28 +535,5 @@
<module>flink</module>
</modules>
</profile>
- <profile>
- <id>scala2.11</id>
- <activation>
- <property>
- <name>scala</name>
- <value>2.11</value>
- </property>
- <activeByDefault>false</activeByDefault>
- </activation>
- <properties>
- <scala.version>2.11.8</scala.version>
- <scala.compat.version>2.11</scala.compat.version>
- <scaladoc.arg />
- </properties>
- <modules>
- <module>common</module>
- <module>core</module>
- <module>sql</module>
- <module>viz</module>
- <module>python-adapter</module>
- <module>flink</module>
- </modules>
- </profile>
</profiles>
</project>