You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2023/11/13 08:56:00 UTC
(sedona) branch fix-examples created (now b6be1a5b9)
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a change to branch fix-examples
in repository https://gitbox.apache.org/repos/asf/sedona.git
at b6be1a5b9 Temp commit
This branch includes the following new commits:
new b6be1a5b9 Temp commit
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(sedona) 01/01: Temp commit
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch fix-examples
in repository https://gitbox.apache.org/repos/asf/sedona.git
commit b6be1a5b98caa7c22e821238c9a060c9291c7333
Author: Jia Yu <ji...@apache.org>
AuthorDate: Mon Nov 13 00:55:45 2023 -0800
Temp commit
---
.github/workflows/example.yml | 6 ++-
examples/flink-sql/pom.xml | 4 +-
examples/spark-rdd-colocation-mining/build.sbt | 13 +++---
.../project/build.properties | 2 +-
.../project/plugins.sbt | 2 +-
.../src/main/scala/ScalaExample.scala | 52 +++++++++-------------
examples/spark-sql/build.sbt | 11 +++--
examples/spark-viz/build.sbt | 11 +++--
pom.xml | 2 +-
9 files changed, 47 insertions(+), 56 deletions(-)
diff --git a/.github/workflows/example.yml b/.github/workflows/example.yml
index 41fab7cd1..2c95d5b71 100644
--- a/.github/workflows/example.yml
+++ b/.github/workflows/example.yml
@@ -3,10 +3,14 @@ on:
push:
branches:
- master
+ paths:
+ - 'examples/**'
pull_request:
branches:
- '*'
-
+ paths:
+ - 'examples/**'
+
jobs:
build:
diff --git a/examples/flink-sql/pom.xml b/examples/flink-sql/pom.xml
index 13e62ab3c..665e3275c 100644
--- a/examples/flink-sql/pom.xml
+++ b/examples/flink-sql/pom.xml
@@ -9,10 +9,10 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
- <geotools.version>1.4.0-28.2</geotools.version>
+ <geotools.version>1.5.0-28.2</geotools.version>
<geotools.scope>compile</geotools.scope>
<scala.compat.version>2.12</scala.compat.version>
- <sedona.version>1.4.0</sedona.version>
+ <sedona.version>1.5.0</sedona.version>
<flink.version>1.14.3</flink.version>
<flink.scope>compile</flink.scope>
</properties>
diff --git a/examples/spark-rdd-colocation-mining/build.sbt b/examples/spark-rdd-colocation-mining/build.sbt
index e8df972ff..0a138fa66 100644
--- a/examples/spark-rdd-colocation-mining/build.sbt
+++ b/examples/spark-rdd-colocation-mining/build.sbt
@@ -26,27 +26,27 @@ lazy val root = (project in file(".")).
version := "0.1.0",
- scalaVersion := "2.12.11",
+ scalaVersion := "2.12.12",
organization := "org.apache.sedona",
publishMavenStyle := true
)
-val SparkVersion = "3.3.1"
+val SparkVersion = "3.4.1"
-val SparkCompatibleVersion = "3.0"
+val SparkCompatibleVersion = "3.4"
val HadoopVersion = "2.7.2"
-val SedonaVersion = "1.4.0"
+val SedonaVersion = "1.5.0"
val ScalaCompatibleVersion = "2.12"
// Change the dependency scope to "provided" when you run "sbt assembly"
val dependencyScope = "compile"
-val geotoolsVersion = "1.4.0-28.2"
+val geotoolsVersion = "1.5.0-28.2"
//val jacksonVersion = "2.10.0"
@@ -62,8 +62,7 @@ libraryDependencies ++= Seq(
"org.apache.hadoop" % "hadoop-mapreduce-client-core" % HadoopVersion % dependencyScope,
"org.apache.hadoop" % "hadoop-common" % HadoopVersion % dependencyScope,
"org.apache.hadoop" % "hadoop-hdfs" % HadoopVersion % dependencyScope,
- "org.apache.sedona" % "sedona-spark-shaded-".concat(SparkCompatibleVersion).concat("_").concat(ScalaCompatibleVersion) % SedonaVersion changing(),
- "org.apache.sedona" % "sedona-viz-".concat(SparkCompatibleVersion).concat("_").concat(ScalaCompatibleVersion) % SedonaVersion changing(),
+ "org.apache.sedona" % "sedona-spark-".concat(SparkCompatibleVersion).concat("_").concat(ScalaCompatibleVersion) % SedonaVersion changing(),
// The following GeoTools packages are only required if you need CRS transformation. Under GNU Lesser General Public License (LGPL) license
"org.datasyslab" % "geotools-wrapper" % geotoolsVersion % "compile"
)
diff --git a/examples/spark-rdd-colocation-mining/project/build.properties b/examples/spark-rdd-colocation-mining/project/build.properties
index 215ddd2b3..331a838c9 100644
--- a/examples/spark-rdd-colocation-mining/project/build.properties
+++ b/examples/spark-rdd-colocation-mining/project/build.properties
@@ -1 +1 @@
-sbt.version = 1.5.5
\ No newline at end of file
+sbt.version = 1.9.7
\ No newline at end of file
diff --git a/examples/spark-rdd-colocation-mining/project/plugins.sbt b/examples/spark-rdd-colocation-mining/project/plugins.sbt
index 415991121..b64685617 100644
--- a/examples/spark-rdd-colocation-mining/project/plugins.sbt
+++ b/examples/spark-rdd-colocation-mining/project/plugins.sbt
@@ -1 +1 @@
-addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.1.0")
\ No newline at end of file
+addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "2.1.4")
\ No newline at end of file
diff --git a/examples/spark-rdd-colocation-mining/src/main/scala/ScalaExample.scala b/examples/spark-rdd-colocation-mining/src/main/scala/ScalaExample.scala
index f69a4e98a..504924251 100644
--- a/examples/spark-rdd-colocation-mining/src/main/scala/ScalaExample.scala
+++ b/examples/spark-rdd-colocation-mining/src/main/scala/ScalaExample.scala
@@ -17,22 +17,21 @@
* under the License.
*/
-import java.awt.Color
import org.apache.log4j.{Level, Logger}
import org.apache.sedona.core.enums.{GridType, IndexType}
import org.apache.sedona.core.formatMapper.shapefileParser.ShapefileReader
import org.apache.sedona.core.spatialOperator.JoinQuery
import org.apache.sedona.core.spatialRDD.{CircleRDD, SpatialRDD}
-import org.apache.sedona.sql.utils.{Adapter, SedonaSQLRegistrator}
-import org.apache.sedona.viz.core.{ImageGenerator, RasterOverlayOperator}
+import org.apache.sedona.spark.SedonaContext
+import org.apache.sedona.sql.utils.Adapter
import org.apache.sedona.viz.core.Serde.SedonaVizKryoRegistrator
+import org.apache.sedona.viz.core.{ImageGenerator, RasterOverlayOperator}
import org.apache.sedona.viz.extension.visualizationEffect.{HeatMap, ScatterPlot}
-import org.apache.sedona.viz.sql.utils.SedonaVizRegistrator
import org.apache.sedona.viz.utils.ImageType
-import org.apache.spark.serializer.KryoSerializer
-import org.apache.spark.sql.SparkSession
import org.locationtech.jts.geom.Geometry
+import java.awt.Color
+
object ScalaExample extends App{
@@ -49,6 +48,13 @@ object ScalaExample extends App{
Logger.getLogger("org").setLevel(Level.WARN)
Logger.getLogger("akka").setLevel(Level.WARN)
+ val config = SedonaContext.builder()
+ .config("spark.kryo.registrator", classOf[SedonaVizKryoRegistrator].getName) // org.apache.sedona.viz.core.Serde.SedonaVizKryoRegistrator
+ .master("local[*]") // Delete this if run in cluster mode
+ .appName("sedona-analysis") // Change this to a proper name
+ .getOrCreate()
+ val sedona = SedonaContext.create(config)
+
visualizeSpatialColocation()
calculateSpatialColocation()
@@ -57,21 +63,14 @@ object ScalaExample extends App{
def visualizeSpatialColocation(): Unit =
{
- val sparkSession:SparkSession = SparkSession.builder().config("spark.serializer",classOf[KryoSerializer].getName).
- config("spark.kryo.registrator", classOf[SedonaVizKryoRegistrator].getName)
- .master("local[*]").appName("Sedona-Analysis").getOrCreate()
-
- SedonaSQLRegistrator.registerAll(sparkSession)
- SedonaVizRegistrator.registerAll(sparkSession)
-
// Prepare NYC area landmarks which includes airports, museums, colleges, hospitals
- var arealmRDD = ShapefileReader.readToPolygonRDD(sparkSession.sparkContext, nycArealandmarkShapefileLocation)
+ var arealmRDD = ShapefileReader.readToPolygonRDD(sedona.sparkContext, nycArealandmarkShapefileLocation)
// Prepare NYC taxi trips. Only use the taxi trips' pickup points
- var tripDf = sparkSession.read.format("csv").option("delimiter",",").option("header","false").load(nyctripCSVLocation)
+ var tripDf = sedona.read.format("csv").option("delimiter",",").option("header","false").load(nyctripCSVLocation)
// Convert from DataFrame to RDD. This can also be done directly through Sedona RDD API.
tripDf.createOrReplaceTempView("tripdf")
- var tripRDD = Adapter.toSpatialRdd(sparkSession.sql("select ST_Point(cast(tripdf._c0 as Decimal(24, 14)), cast(tripdf._c1 as Decimal(24, 14))) as point from tripdf")
+ var tripRDD = Adapter.toSpatialRdd(sedona.sql("select ST_Point(cast(tripdf._c0 as Decimal(24, 14)), cast(tripdf._c1 as Decimal(24, 14))) as point from tripdf")
, "point")
// Convert the Coordinate Reference System from degree-based to meter-based. This returns the accurate distance calculate.
@@ -87,32 +86,24 @@ object ScalaExample extends App{
val frontImage = new ScatterPlot(imageResolutionX, imageResolutionY, arealmRDD.boundaryEnvelope, true)
frontImage.CustomizeColor(0, 0, 0, 255, Color.GREEN, true)
- frontImage.Visualize(sparkSession.sparkContext, arealmRDD)
+ frontImage.Visualize(sedona.sparkContext, arealmRDD)
val backImage = new HeatMap(imageResolutionX, imageResolutionY, arealmRDD.boundaryEnvelope, true, 1)
- backImage.Visualize(sparkSession.sparkContext, tripRDD)
+ backImage.Visualize(sedona.sparkContext, tripRDD)
val overlayOperator = new RasterOverlayOperator(backImage.rasterImage)
overlayOperator.JoinImage(frontImage.rasterImage)
val imageGenerator = new ImageGenerator
imageGenerator.SaveRasterImageAsLocalFile(overlayOperator.backRasterImage, colocationMapLocation, ImageType.PNG)
-
- sparkSession.stop()
}
def calculateSpatialColocation(): Unit =
{
- val sparkSession:SparkSession = SparkSession.builder().config("spark.serializer",classOf[KryoSerializer].getName).
- config("spark.kryo.registrator", classOf[SedonaVizKryoRegistrator].getName).
- master("local[*]").appName("Sedona-Analysis").getOrCreate()
-
- SedonaSQLRegistrator.registerAll(sparkSession)
- SedonaVizRegistrator.registerAll(sparkSession)
// Prepare NYC area landmarks which includes airports, museums, colleges, hospitals
var arealmRDD = new SpatialRDD[Geometry]()
- arealmRDD = ShapefileReader.readToGeometryRDD(sparkSession.sparkContext, nycArealandmarkShapefileLocation)
+ arealmRDD = ShapefileReader.readToGeometryRDD(sedona.sparkContext, nycArealandmarkShapefileLocation)
// Use the center point of area landmarks to check co-location. This is required by Ripley's K function.
arealmRDD.rawSpatialRDD = arealmRDD.rawSpatialRDD.rdd.map[Geometry](f=>
{
@@ -123,15 +114,15 @@ object ScalaExample extends App{
})
// The following two lines are optional. The purpose is to show the structure of the shapefile.
- var arealmDf = Adapter.toDf(arealmRDD, sparkSession)
+ var arealmDf = Adapter.toDf(arealmRDD, sedona)
arealmDf.show()
// Prepare NYC taxi trips. Only use the taxi trips' pickup points
- var tripDf = sparkSession.read.format("csv").option("delimiter",",").option("header","false").load(nyctripCSVLocation)
+ var tripDf = sedona.read.format("csv").option("delimiter",",").option("header","false").load(nyctripCSVLocation)
tripDf.show() // Optional
// Convert from DataFrame to RDD. This can also be done directly through Sedona RDD API.
tripDf.createOrReplaceTempView("tripdf")
- var tripRDD = Adapter.toSpatialRdd(sparkSession.sql("select ST_Point(cast(tripdf._c0 as Decimal(24, 14)), cast(tripdf._c1 as Decimal(24, 14))) as point, 'def' as trip_attr from tripdf")
+ var tripRDD = Adapter.toSpatialRdd(sedona.sql("select ST_Point(cast(tripdf._c0 as Decimal(24, 14)), cast(tripdf._c1 as Decimal(24, 14))) as point, 'def' as trip_attr from tripdf")
, "point")
// Convert the Coordinate Reference System from degree-based to meter-based. This returns the accurate distance calculate.
@@ -179,7 +170,6 @@ object ScalaExample extends App{
println(s"""$currentDistance,$observedL,$colocationDifference,$colocationStatus""")
}
- sparkSession.stop()
}
}
diff --git a/examples/spark-sql/build.sbt b/examples/spark-sql/build.sbt
index aeb45a6c4..68b1c1c6c 100644
--- a/examples/spark-sql/build.sbt
+++ b/examples/spark-sql/build.sbt
@@ -33,20 +33,20 @@ lazy val root = (project in file(".")).
publishMavenStyle := true
)
-val SparkVersion = "3.3.1"
+val SparkVersion = "3.4.1"
-val SparkCompatibleVersion = "3.0"
+val SparkCompatibleVersion = "3.4"
val HadoopVersion = "2.7.2"
-val SedonaVersion = "1.4.0"
+val SedonaVersion = "1.5.0"
val ScalaCompatibleVersion = "2.12"
// Change the dependency scope to "provided" when you run "sbt assembly"
val dependencyScope = "compile"
-val geotoolsVersion = "1.4.0-28.2"
+val geotoolsVersion = "1.5.0-28.2"
//val jacksonVersion = "2.10.0"
@@ -62,8 +62,7 @@ libraryDependencies ++= Seq(
"org.apache.hadoop" % "hadoop-mapreduce-client-core" % HadoopVersion % dependencyScope,
"org.apache.hadoop" % "hadoop-common" % HadoopVersion % dependencyScope,
"org.apache.hadoop" % "hadoop-hdfs" % HadoopVersion % dependencyScope,
- "org.apache.sedona" % "sedona-spark-shaded-".concat(SparkCompatibleVersion).concat("_").concat(ScalaCompatibleVersion) % SedonaVersion changing(),
- "org.apache.sedona" % "sedona-viz-".concat(SparkCompatibleVersion).concat("_").concat(ScalaCompatibleVersion) % SedonaVersion changing(),
+ "org.apache.sedona" % "sedona-spark-".concat(SparkCompatibleVersion).concat("_").concat(ScalaCompatibleVersion) % SedonaVersion changing(),
// The following GeoTools packages are only required if you need CRS transformation. Under GNU Lesser General Public License (LGPL) license
"org.datasyslab" % "geotools-wrapper" % geotoolsVersion % "compile"
)
diff --git a/examples/spark-viz/build.sbt b/examples/spark-viz/build.sbt
index bc2260a89..7f6d15eb9 100644
--- a/examples/spark-viz/build.sbt
+++ b/examples/spark-viz/build.sbt
@@ -33,20 +33,20 @@ lazy val root = (project in file(".")).
publishMavenStyle := true
)
-val SparkVersion = "3.3.1"
+val SparkVersion = "3.4.1"
-val SparkCompatibleVersion = "3.0"
+val SparkCompatibleVersion = "3.4"
val HadoopVersion = "2.7.2"
-val SedonaVersion = "1.4.0"
+val SedonaVersion = "1.5.0"
val ScalaCompatibleVersion = "2.12"
// Change the dependency scope to "provided" when you run "sbt assembly"
val dependencyScope = "compile"
-val geotoolsVersion = "1.4.0-28.2"
+val geotoolsVersion = "1.5.0-28.2"
//val jacksonVersion = "2.10.0"
@@ -62,8 +62,7 @@ libraryDependencies ++= Seq(
"org.apache.hadoop" % "hadoop-mapreduce-client-core" % HadoopVersion % dependencyScope,
"org.apache.hadoop" % "hadoop-common" % HadoopVersion % dependencyScope,
"org.apache.hadoop" % "hadoop-hdfs" % HadoopVersion % dependencyScope,
- "org.apache.sedona" % "sedona-spark-shaded-".concat(SparkCompatibleVersion).concat("_").concat(ScalaCompatibleVersion) % SedonaVersion changing(),
- "org.apache.sedona" % "sedona-viz-".concat(SparkCompatibleVersion).concat("_").concat(ScalaCompatibleVersion) % SedonaVersion changing(),
+ "org.apache.sedona" % "sedona-spark-".concat(SparkCompatibleVersion).concat("_").concat(ScalaCompatibleVersion) % SedonaVersion changing(),
// The following GeoTools packages are only required if you need CRS transformation. Under GNU Lesser General Public License (LGPL) license
"org.datasyslab" % "geotools-wrapper" % geotoolsVersion % "compile"
)
diff --git a/pom.xml b/pom.xml
index cf0ac5366..45a2737dc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -659,7 +659,7 @@
</property>
</activation>
<properties>
- <spark.version>3.4.0</spark.version>
+ <spark.version>3.5.0</spark.version>
<spark.compat.version>3.4</spark.compat.version>
<log4j.version>2.19.0</log4j.version>
<!-- Skip deploying parent module for sedona-spark-3.0 profile, it will be deployed with sedona-spark-3.0 -->