You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2023/09/07 00:41:21 UTC
[sedona] branch viz-to-spark created (now df7ce237f)
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a change to branch viz-to-spark
in repository https://gitbox.apache.org/repos/asf/sedona.git
at df7ce237f Finalize
This branch includes the following new commits:
new 03826df93 Add the initial change
new df7ce237f Finalize
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[sedona] 01/02: Add the initial change
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch viz-to-spark
in repository https://gitbox.apache.org/repos/asf/sedona.git
commit 03826df93ffbd5224e288706bf880442f00c5c7f
Author: Jia Yu <ji...@apache.org>
AuthorDate: Wed Sep 6 13:32:51 2023 -0700
Add the initial change
---
pom.xml | 2 -
spark/common/pom.xml | 17 ++
.../apache/sedona/viz/core/BigBufferedImage.java | 0
.../org/apache/sedona/viz/core/ColoringRule.java | 0
.../apache/sedona/viz/core/GlobalParameter.java | 0
.../org/apache/sedona/viz/core/ImageGenerator.java | 0
.../sedona/viz/core/ImageSerializableWrapper.java | 0
.../org/apache/sedona/viz/core/ImageStitcher.java | 0
.../org/apache/sedona/viz/core/PhotoFilter.java | 0
.../sedona/viz/core/RasterOverlayOperator.java | 0
.../viz/core/Serde/ImageWrapperSerializer.java | 0
.../sedona/viz/core/Serde/PixelSerializer.java | 0
.../viz/core/Serde/SedonaVizKryoRegistrator.java | 0
.../sedona/viz/core/VectorOverlayOperator.java | 0
.../sedona/viz/core/VisualizationOperator.java | 0
.../sedona/viz/core/VisualizationPartitioner.java | 0
.../coloringRule/ColoringRuleFactory.java | 0
.../coloringRule/GenericColoringRule.java | 0
.../viz/extension/coloringRule/LinearFunction.java | 0
.../extension/coloringRule/PiecewiseFunction.java | 0
.../imageGenerator/SedonaVizImageGenerator.java | 0
.../sedona/viz/extension/photoFilter/BoxBlur.java | 0
.../sedona/viz/extension/photoFilter/Embose.java | 0
.../viz/extension/photoFilter/GaussianBlur.java | 0
.../sedona/viz/extension/photoFilter/Outline.java | 0
.../sedona/viz/extension/photoFilter/Sharpen.java | 0
.../visualizationEffect/ChoroplethMap.java | 0
.../viz/extension/visualizationEffect/HeatMap.java | 0
.../extension/visualizationEffect/ScatterPlot.java | 0
.../org/apache/sedona/viz/showcase/Example.java | 0
.../org/apache/sedona/viz/showcase/nycheatmap.png | Bin
.../java/org/apache/sedona/viz/showcase/usrail.png | Bin
.../org/apache/sedona/viz/showcase/ustweet.png | Bin
.../apache/sedona/viz/utils/ColorizeOption.java | 0
.../org/apache/sedona/viz/utils/ImageType.java | 0
.../java/org/apache/sedona/viz/utils/Pixel.java | 0
.../sedona/viz/utils/RasterizationUtils.java | 0
.../org/apache/sedona/viz/utils/S3Operator.java | 0
.../apache/sedona/viz/showcase/ScalaExample.scala | 0
.../sedona/viz/sql/SedonaVizExtensions.scala | 0
.../org/apache/sedona/viz/sql/UDF/Catalog.scala | 0
.../apache/sedona/viz/sql/UDF/UdfRegistrator.scala | 0
.../apache/sedona/viz/sql/UDT/UdtRegistrator.scala | 0
.../sql/operator/AggregateWithinPartitons.scala | 0
.../sedona/viz/sql/operator/VizPartitioner.scala | 0
.../org/apache/sedona/viz/sql/utils/Conf.scala | 0
.../sedona/viz/sql/utils/LineageDecoder.scala | 0
.../viz/sql/utils/SedonaVizRegistrator.scala | 0
.../spark/sql/sedona_viz/UDT/ImageWrapperUDT.scala | 0
.../apache/spark/sql/sedona_viz/UDT/PixelUDT.scala | 0
.../sql/sedona_viz/UDT/UdtRegistratorWrapper.scala | 0
.../sql/sedona_viz/expressions/Colorize.scala | 0
.../sql/sedona_viz/expressions/ImageEncoder.scala | 0
.../sql/sedona_viz/expressions/Partitioner.scala | 0
.../sql/sedona_viz/expressions/Pixelize.scala | 5 +-
.../spark/sql/sedona_viz/expressions/Render.scala | 3 +-
.../org/apache/sedona/viz/ChoroplethmapTest.java | 0
.../java/org/apache/sedona/viz/HeatmapTest.java | 0
.../org/apache/sedona/viz/NYCTripPointMapper.java | 0
.../java/org/apache/sedona/viz/NYCTripTest.java | 0
.../sedona/viz/ParallelVisualizationTest.java | 0
.../org/apache/sedona/viz/ScatterplotTest.java | 0
.../java/org/apache/sedona/viz/VizTestBase.java | 2 +-
.../org/apache/sedona/sql/TestBaseScala.scala | 26 +--
.../org/apache/sedona/viz/rdd/scalaTest.scala | 2 +-
.../org/apache/sedona/viz/sql/TestBaseScala.scala | 2 +-
.../apache/sedona/viz/sql/optVizOperatorTest.scala | 2 +-
.../sedona/viz/sql/standardVizOperatorTest.scala | 0
viz/.gitignore | 10 --
viz/pom.xml | 177 ---------------------
viz/src/.gitignore | 1 -
viz/src/main/.gitignore | 1 -
viz/src/test/.gitignore | 1 -
viz/src/test/resources/log4j2.properties | 31 ----
74 files changed, 40 insertions(+), 242 deletions(-)
diff --git a/pom.xml b/pom.xml
index 1d83505a8..1bcc4ec37 100644
--- a/pom.xml
+++ b/pom.xml
@@ -656,7 +656,6 @@
<modules>
<module>common</module>
<module>spark</module>
- <module>viz</module>
<module>spark-shaded</module>
</modules>
</profile>
@@ -677,7 +676,6 @@
<modules>
<module>common</module>
<module>spark</module>
- <module>viz</module>
<module>spark-shaded</module>
<module>flink</module>
<module>flink-shaded</module>
diff --git a/spark/common/pom.xml b/spark/common/pom.xml
index 35899e47d..0a39fbbab 100644
--- a/spark/common/pom.xml
+++ b/spark/common/pom.xml
@@ -160,6 +160,23 @@
<groupId>org.mockito</groupId>
<artifactId>mockito-inline</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.beryx</groupId>
+ <artifactId>awt-color-factory</artifactId>
+ <version>1.0.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-s3</artifactId>
+ <version>1.12.362</version>
+ <scope>provided</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
diff --git a/viz/src/main/java/org/apache/sedona/viz/core/BigBufferedImage.java b/spark/common/src/main/java/org/apache/sedona/viz/core/BigBufferedImage.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/core/BigBufferedImage.java
rename to spark/common/src/main/java/org/apache/sedona/viz/core/BigBufferedImage.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/core/ColoringRule.java b/spark/common/src/main/java/org/apache/sedona/viz/core/ColoringRule.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/core/ColoringRule.java
rename to spark/common/src/main/java/org/apache/sedona/viz/core/ColoringRule.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/core/GlobalParameter.java b/spark/common/src/main/java/org/apache/sedona/viz/core/GlobalParameter.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/core/GlobalParameter.java
rename to spark/common/src/main/java/org/apache/sedona/viz/core/GlobalParameter.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/core/ImageGenerator.java b/spark/common/src/main/java/org/apache/sedona/viz/core/ImageGenerator.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/core/ImageGenerator.java
rename to spark/common/src/main/java/org/apache/sedona/viz/core/ImageGenerator.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/core/ImageSerializableWrapper.java b/spark/common/src/main/java/org/apache/sedona/viz/core/ImageSerializableWrapper.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/core/ImageSerializableWrapper.java
rename to spark/common/src/main/java/org/apache/sedona/viz/core/ImageSerializableWrapper.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/core/ImageStitcher.java b/spark/common/src/main/java/org/apache/sedona/viz/core/ImageStitcher.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/core/ImageStitcher.java
rename to spark/common/src/main/java/org/apache/sedona/viz/core/ImageStitcher.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/core/PhotoFilter.java b/spark/common/src/main/java/org/apache/sedona/viz/core/PhotoFilter.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/core/PhotoFilter.java
rename to spark/common/src/main/java/org/apache/sedona/viz/core/PhotoFilter.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/core/RasterOverlayOperator.java b/spark/common/src/main/java/org/apache/sedona/viz/core/RasterOverlayOperator.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/core/RasterOverlayOperator.java
rename to spark/common/src/main/java/org/apache/sedona/viz/core/RasterOverlayOperator.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/core/Serde/ImageWrapperSerializer.java b/spark/common/src/main/java/org/apache/sedona/viz/core/Serde/ImageWrapperSerializer.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/core/Serde/ImageWrapperSerializer.java
rename to spark/common/src/main/java/org/apache/sedona/viz/core/Serde/ImageWrapperSerializer.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/core/Serde/PixelSerializer.java b/spark/common/src/main/java/org/apache/sedona/viz/core/Serde/PixelSerializer.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/core/Serde/PixelSerializer.java
rename to spark/common/src/main/java/org/apache/sedona/viz/core/Serde/PixelSerializer.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/core/Serde/SedonaVizKryoRegistrator.java b/spark/common/src/main/java/org/apache/sedona/viz/core/Serde/SedonaVizKryoRegistrator.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/core/Serde/SedonaVizKryoRegistrator.java
rename to spark/common/src/main/java/org/apache/sedona/viz/core/Serde/SedonaVizKryoRegistrator.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/core/VectorOverlayOperator.java b/spark/common/src/main/java/org/apache/sedona/viz/core/VectorOverlayOperator.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/core/VectorOverlayOperator.java
rename to spark/common/src/main/java/org/apache/sedona/viz/core/VectorOverlayOperator.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/core/VisualizationOperator.java b/spark/common/src/main/java/org/apache/sedona/viz/core/VisualizationOperator.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/core/VisualizationOperator.java
rename to spark/common/src/main/java/org/apache/sedona/viz/core/VisualizationOperator.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/core/VisualizationPartitioner.java b/spark/common/src/main/java/org/apache/sedona/viz/core/VisualizationPartitioner.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/core/VisualizationPartitioner.java
rename to spark/common/src/main/java/org/apache/sedona/viz/core/VisualizationPartitioner.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/coloringRule/ColoringRuleFactory.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/coloringRule/ColoringRuleFactory.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/extension/coloringRule/ColoringRuleFactory.java
rename to spark/common/src/main/java/org/apache/sedona/viz/extension/coloringRule/ColoringRuleFactory.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/coloringRule/GenericColoringRule.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/coloringRule/GenericColoringRule.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/extension/coloringRule/GenericColoringRule.java
rename to spark/common/src/main/java/org/apache/sedona/viz/extension/coloringRule/GenericColoringRule.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/coloringRule/LinearFunction.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/coloringRule/LinearFunction.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/extension/coloringRule/LinearFunction.java
rename to spark/common/src/main/java/org/apache/sedona/viz/extension/coloringRule/LinearFunction.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/coloringRule/PiecewiseFunction.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/coloringRule/PiecewiseFunction.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/extension/coloringRule/PiecewiseFunction.java
rename to spark/common/src/main/java/org/apache/sedona/viz/extension/coloringRule/PiecewiseFunction.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/imageGenerator/SedonaVizImageGenerator.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/imageGenerator/SedonaVizImageGenerator.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/extension/imageGenerator/SedonaVizImageGenerator.java
rename to spark/common/src/main/java/org/apache/sedona/viz/extension/imageGenerator/SedonaVizImageGenerator.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/BoxBlur.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/BoxBlur.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/BoxBlur.java
rename to spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/BoxBlur.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/Embose.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/Embose.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/Embose.java
rename to spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/Embose.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/GaussianBlur.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/GaussianBlur.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/GaussianBlur.java
rename to spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/GaussianBlur.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/Outline.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/Outline.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/Outline.java
rename to spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/Outline.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/Sharpen.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/Sharpen.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/extension/photoFilter/Sharpen.java
rename to spark/common/src/main/java/org/apache/sedona/viz/extension/photoFilter/Sharpen.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/ChoroplethMap.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/ChoroplethMap.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/ChoroplethMap.java
rename to spark/common/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/ChoroplethMap.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/HeatMap.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/HeatMap.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/HeatMap.java
rename to spark/common/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/HeatMap.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/ScatterPlot.java b/spark/common/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/ScatterPlot.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/ScatterPlot.java
rename to spark/common/src/main/java/org/apache/sedona/viz/extension/visualizationEffect/ScatterPlot.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/showcase/Example.java b/spark/common/src/main/java/org/apache/sedona/viz/showcase/Example.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/showcase/Example.java
rename to spark/common/src/main/java/org/apache/sedona/viz/showcase/Example.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/showcase/nycheatmap.png b/spark/common/src/main/java/org/apache/sedona/viz/showcase/nycheatmap.png
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/showcase/nycheatmap.png
rename to spark/common/src/main/java/org/apache/sedona/viz/showcase/nycheatmap.png
diff --git a/viz/src/main/java/org/apache/sedona/viz/showcase/usrail.png b/spark/common/src/main/java/org/apache/sedona/viz/showcase/usrail.png
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/showcase/usrail.png
rename to spark/common/src/main/java/org/apache/sedona/viz/showcase/usrail.png
diff --git a/viz/src/main/java/org/apache/sedona/viz/showcase/ustweet.png b/spark/common/src/main/java/org/apache/sedona/viz/showcase/ustweet.png
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/showcase/ustweet.png
rename to spark/common/src/main/java/org/apache/sedona/viz/showcase/ustweet.png
diff --git a/viz/src/main/java/org/apache/sedona/viz/utils/ColorizeOption.java b/spark/common/src/main/java/org/apache/sedona/viz/utils/ColorizeOption.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/utils/ColorizeOption.java
rename to spark/common/src/main/java/org/apache/sedona/viz/utils/ColorizeOption.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/utils/ImageType.java b/spark/common/src/main/java/org/apache/sedona/viz/utils/ImageType.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/utils/ImageType.java
rename to spark/common/src/main/java/org/apache/sedona/viz/utils/ImageType.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/utils/Pixel.java b/spark/common/src/main/java/org/apache/sedona/viz/utils/Pixel.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/utils/Pixel.java
rename to spark/common/src/main/java/org/apache/sedona/viz/utils/Pixel.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/utils/RasterizationUtils.java b/spark/common/src/main/java/org/apache/sedona/viz/utils/RasterizationUtils.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/utils/RasterizationUtils.java
rename to spark/common/src/main/java/org/apache/sedona/viz/utils/RasterizationUtils.java
diff --git a/viz/src/main/java/org/apache/sedona/viz/utils/S3Operator.java b/spark/common/src/main/java/org/apache/sedona/viz/utils/S3Operator.java
similarity index 100%
rename from viz/src/main/java/org/apache/sedona/viz/utils/S3Operator.java
rename to spark/common/src/main/java/org/apache/sedona/viz/utils/S3Operator.java
diff --git a/viz/src/main/scala/org/apache/sedona/viz/showcase/ScalaExample.scala b/spark/common/src/main/scala/org/apache/sedona/viz/showcase/ScalaExample.scala
similarity index 100%
rename from viz/src/main/scala/org/apache/sedona/viz/showcase/ScalaExample.scala
rename to spark/common/src/main/scala/org/apache/sedona/viz/showcase/ScalaExample.scala
diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/SedonaVizExtensions.scala b/spark/common/src/main/scala/org/apache/sedona/viz/sql/SedonaVizExtensions.scala
similarity index 100%
rename from viz/src/main/scala/org/apache/sedona/viz/sql/SedonaVizExtensions.scala
rename to spark/common/src/main/scala/org/apache/sedona/viz/sql/SedonaVizExtensions.scala
diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/UDF/Catalog.scala b/spark/common/src/main/scala/org/apache/sedona/viz/sql/UDF/Catalog.scala
similarity index 100%
rename from viz/src/main/scala/org/apache/sedona/viz/sql/UDF/Catalog.scala
rename to spark/common/src/main/scala/org/apache/sedona/viz/sql/UDF/Catalog.scala
diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/UDF/UdfRegistrator.scala b/spark/common/src/main/scala/org/apache/sedona/viz/sql/UDF/UdfRegistrator.scala
similarity index 100%
rename from viz/src/main/scala/org/apache/sedona/viz/sql/UDF/UdfRegistrator.scala
rename to spark/common/src/main/scala/org/apache/sedona/viz/sql/UDF/UdfRegistrator.scala
diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/UDT/UdtRegistrator.scala b/spark/common/src/main/scala/org/apache/sedona/viz/sql/UDT/UdtRegistrator.scala
similarity index 100%
rename from viz/src/main/scala/org/apache/sedona/viz/sql/UDT/UdtRegistrator.scala
rename to spark/common/src/main/scala/org/apache/sedona/viz/sql/UDT/UdtRegistrator.scala
diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/operator/AggregateWithinPartitons.scala b/spark/common/src/main/scala/org/apache/sedona/viz/sql/operator/AggregateWithinPartitons.scala
similarity index 100%
rename from viz/src/main/scala/org/apache/sedona/viz/sql/operator/AggregateWithinPartitons.scala
rename to spark/common/src/main/scala/org/apache/sedona/viz/sql/operator/AggregateWithinPartitons.scala
diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/operator/VizPartitioner.scala b/spark/common/src/main/scala/org/apache/sedona/viz/sql/operator/VizPartitioner.scala
similarity index 100%
rename from viz/src/main/scala/org/apache/sedona/viz/sql/operator/VizPartitioner.scala
rename to spark/common/src/main/scala/org/apache/sedona/viz/sql/operator/VizPartitioner.scala
diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/utils/Conf.scala b/spark/common/src/main/scala/org/apache/sedona/viz/sql/utils/Conf.scala
similarity index 100%
rename from viz/src/main/scala/org/apache/sedona/viz/sql/utils/Conf.scala
rename to spark/common/src/main/scala/org/apache/sedona/viz/sql/utils/Conf.scala
diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/utils/LineageDecoder.scala b/spark/common/src/main/scala/org/apache/sedona/viz/sql/utils/LineageDecoder.scala
similarity index 100%
rename from viz/src/main/scala/org/apache/sedona/viz/sql/utils/LineageDecoder.scala
rename to spark/common/src/main/scala/org/apache/sedona/viz/sql/utils/LineageDecoder.scala
diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/utils/SedonaVizRegistrator.scala b/spark/common/src/main/scala/org/apache/sedona/viz/sql/utils/SedonaVizRegistrator.scala
similarity index 100%
rename from viz/src/main/scala/org/apache/sedona/viz/sql/utils/SedonaVizRegistrator.scala
rename to spark/common/src/main/scala/org/apache/sedona/viz/sql/utils/SedonaVizRegistrator.scala
diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/ImageWrapperUDT.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/ImageWrapperUDT.scala
similarity index 100%
rename from viz/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/ImageWrapperUDT.scala
rename to spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/ImageWrapperUDT.scala
diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/PixelUDT.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/PixelUDT.scala
similarity index 100%
rename from viz/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/PixelUDT.scala
rename to spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/PixelUDT.scala
diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/UdtRegistratorWrapper.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/UdtRegistratorWrapper.scala
similarity index 100%
rename from viz/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/UdtRegistratorWrapper.scala
rename to spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/UDT/UdtRegistratorWrapper.scala
diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Colorize.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Colorize.scala
similarity index 100%
rename from viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Colorize.scala
rename to spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Colorize.scala
diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/ImageEncoder.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/ImageEncoder.scala
similarity index 100%
rename from viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/ImageEncoder.scala
rename to spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/ImageEncoder.scala
diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Partitioner.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Partitioner.scala
similarity index 100%
rename from viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Partitioner.scala
rename to spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Partitioner.scala
diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala
similarity index 96%
rename from viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala
rename to spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala
index 587b6d4e0..9f7104b35 100644
--- a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala
+++ b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala
@@ -28,10 +28,11 @@ import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.Expression
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
-import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData}
+import org.apache.spark.sql.catalyst.util.GenericArrayData
import org.apache.spark.sql.sedona_viz.UDT.PixelUDT
import org.apache.spark.sql.types.{ArrayType, DataType}
-import org.locationtech.jts.geom.{Envelope, Geometry, LineString, MultiLineString, MultiPoint, MultiPolygon, Point, Polygon}
+import org.locationtech.jts.geom._
+
import scala.jdk.CollectionConverters._
case class ST_Pixelize(inputExpressions: Seq[Expression])
diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Render.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Render.scala
similarity index 98%
rename from viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Render.scala
rename to spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Render.scala
index d3cfcd6cb..f61215b46 100644
--- a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Render.scala
+++ b/spark/common/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Render.scala
@@ -18,8 +18,7 @@
*/
package org.apache.spark.sql.sedona_viz.expressions
-import java.awt.image.BufferedImage // scalastyle:ignore illegal.imports
-
+import java.awt.image.BufferedImage
import org.apache.sedona.viz.core.ImageSerializableWrapper
import org.apache.sedona.viz.utils.Pixel
import org.apache.spark.internal.Logging
diff --git a/viz/src/test/java/org/apache/sedona/viz/ChoroplethmapTest.java b/spark/common/src/test/java/org/apache/sedona/viz/ChoroplethmapTest.java
similarity index 100%
rename from viz/src/test/java/org/apache/sedona/viz/ChoroplethmapTest.java
rename to spark/common/src/test/java/org/apache/sedona/viz/ChoroplethmapTest.java
diff --git a/viz/src/test/java/org/apache/sedona/viz/HeatmapTest.java b/spark/common/src/test/java/org/apache/sedona/viz/HeatmapTest.java
similarity index 100%
rename from viz/src/test/java/org/apache/sedona/viz/HeatmapTest.java
rename to spark/common/src/test/java/org/apache/sedona/viz/HeatmapTest.java
diff --git a/viz/src/test/java/org/apache/sedona/viz/NYCTripPointMapper.java b/spark/common/src/test/java/org/apache/sedona/viz/NYCTripPointMapper.java
similarity index 100%
rename from viz/src/test/java/org/apache/sedona/viz/NYCTripPointMapper.java
rename to spark/common/src/test/java/org/apache/sedona/viz/NYCTripPointMapper.java
diff --git a/viz/src/test/java/org/apache/sedona/viz/NYCTripTest.java b/spark/common/src/test/java/org/apache/sedona/viz/NYCTripTest.java
similarity index 100%
rename from viz/src/test/java/org/apache/sedona/viz/NYCTripTest.java
rename to spark/common/src/test/java/org/apache/sedona/viz/NYCTripTest.java
diff --git a/viz/src/test/java/org/apache/sedona/viz/ParallelVisualizationTest.java b/spark/common/src/test/java/org/apache/sedona/viz/ParallelVisualizationTest.java
similarity index 100%
rename from viz/src/test/java/org/apache/sedona/viz/ParallelVisualizationTest.java
rename to spark/common/src/test/java/org/apache/sedona/viz/ParallelVisualizationTest.java
diff --git a/viz/src/test/java/org/apache/sedona/viz/ScatterplotTest.java b/spark/common/src/test/java/org/apache/sedona/viz/ScatterplotTest.java
similarity index 100%
rename from viz/src/test/java/org/apache/sedona/viz/ScatterplotTest.java
rename to spark/common/src/test/java/org/apache/sedona/viz/ScatterplotTest.java
diff --git a/viz/src/test/java/org/apache/sedona/viz/VizTestBase.java b/spark/common/src/test/java/org/apache/sedona/viz/VizTestBase.java
similarity index 99%
rename from viz/src/test/java/org/apache/sedona/viz/VizTestBase.java
rename to spark/common/src/test/java/org/apache/sedona/viz/VizTestBase.java
index 923b2a114..29c66cc4f 100644
--- a/viz/src/test/java/org/apache/sedona/viz/VizTestBase.java
+++ b/spark/common/src/test/java/org/apache/sedona/viz/VizTestBase.java
@@ -147,7 +147,7 @@ public class VizTestBase
Logger.getLogger("org").setLevel(Level.WARN);
Logger.getLogger("akka").setLevel(Level.WARN);
- String resourceFolder = System.getProperty("user.dir") + "/../spark/common/src/test/resources/";
+ String resourceFolder = System.getProperty("user.dir") + "/src/test/resources/";
prop = new Properties();
diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala b/spark/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala
index b54ac4008..662de6d9e 100644
--- a/spark/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala
+++ b/spark/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala
@@ -21,9 +21,10 @@ package org.apache.sedona.sql
import com.google.common.math.DoubleMath
import org.apache.log4j.{Level, Logger}
import org.apache.sedona.common.sphere.{Haversine, Spheroid}
-import org.apache.sedona.common.Functions.{hausdorffDistance, frechetDistance}
+import org.apache.sedona.common.Functions.{frechetDistance, hausdorffDistance}
import org.apache.sedona.spark.SedonaContext
-import org.apache.spark.sql.DataFrame
+import org.apache.spark.SparkContext
+import org.apache.spark.sql.{DataFrame, SparkSession}
import org.locationtech.jts.geom.{CoordinateSequence, CoordinateSequenceComparator}
import org.scalatest.{BeforeAndAfterAll, FunSpec}
@@ -35,15 +36,9 @@ trait TestBaseScala extends FunSpec with BeforeAndAfterAll {
Logger.getLogger("org.apache.sedona.core").setLevel(Level.WARN)
val warehouseLocation = System.getProperty("user.dir") + "/target/"
- val sparkSession = SedonaContext.builder().
- master("local[*]").appName("sedonasqlScalaTest")
- .config("spark.sql.warehouse.dir", warehouseLocation)
- // We need to be explicit about broadcasting in tests.
- .config("sedona.join.autoBroadcastJoinThreshold", "-1")
- .config("spark.kryoserializer.buffer.max", "64m")
- .getOrCreate()
+ var sparkSession:SparkSession = null
- val sc = sparkSession.sparkContext
+ var sc: SparkContext = null
val resourceFolder = System.getProperty("user.dir") + "/src/test/resources/"
val mixedWkbGeometryInputLocation = resourceFolder + "county_small_wkb.tsv"
@@ -74,12 +69,21 @@ trait TestBaseScala extends FunSpec with BeforeAndAfterAll {
val smallRasterDataLocation: String = resourceFolder + "raster/test1.tiff"
override def beforeAll(): Unit = {
+ sparkSession = SedonaContext.builder().
+ master("local[*]").appName("sedonasqlScalaTest")
+ .config("spark.sql.warehouse.dir", warehouseLocation)
+ // We need to be explicit about broadcasting in tests.
+ .config("sedona.join.autoBroadcastJoinThreshold", "-1")
+ .config("spark.kryoserializer.buffer.max", "64m")
+ .getOrCreate()
+
+ sc = sparkSession.sparkContext
SedonaContext.create(sparkSession)
}
override def afterAll(): Unit = {
//SedonaSQLRegistrator.dropAll(spark)
-// sparkSession.stop
+ sparkSession.stop
}
def loadCsv(path: String): DataFrame = {
diff --git a/viz/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala b/spark/common/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala
similarity index 99%
rename from viz/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala
rename to spark/common/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala
index bc0e65641..068e97ea3 100644
--- a/viz/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala
+++ b/spark/common/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala
@@ -45,7 +45,7 @@ class scalaTest extends FunSpec with BeforeAndAfterAll{
Logger.getLogger("org").setLevel(Level.WARN)
Logger.getLogger("akka").setLevel(Level.WARN)
val prop = new Properties()
- val resourceFolder = System.getProperty("user.dir") + "/../spark/common/src/test/resources/"
+ val resourceFolder = System.getProperty("user.dir") + "/src/test/resources/"
val demoOutputPath = "target/scala/demo"
var ConfFile = new FileInputStream(resourceFolder + "babylon.point.properties")
prop.load(ConfFile)
diff --git a/viz/src/test/scala/org/apache/sedona/viz/sql/TestBaseScala.scala b/spark/common/src/test/scala/org/apache/sedona/viz/sql/TestBaseScala.scala
similarity index 96%
rename from viz/src/test/scala/org/apache/sedona/viz/sql/TestBaseScala.scala
rename to spark/common/src/test/scala/org/apache/sedona/viz/sql/TestBaseScala.scala
index 2967413f3..67bb98346 100644
--- a/viz/src/test/scala/org/apache/sedona/viz/sql/TestBaseScala.scala
+++ b/spark/common/src/test/scala/org/apache/sedona/viz/sql/TestBaseScala.scala
@@ -31,7 +31,7 @@ trait TestBaseScala extends FunSpec with BeforeAndAfterAll{
Logger.getLogger("org.datasyslab").setLevel(Level.WARN)
var spark:SparkSession = _
- val resourceFolder = System.getProperty("user.dir") + "/../spark/common/src/test/resources/"
+ val resourceFolder = System.getProperty("user.dir") + "/src/test/resources/"
val polygonInputLocationWkt = resourceFolder + "county_small.tsv"
val polygonInputLocation = resourceFolder + "primaryroads-polygon.csv"
diff --git a/viz/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala b/spark/common/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala
similarity index 100%
rename from viz/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala
rename to spark/common/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala
index 9fd048557..8969a1afa 100644
--- a/viz/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala
+++ b/spark/common/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala
@@ -21,8 +21,8 @@ package org.apache.sedona.viz.sql
import org.apache.sedona.viz.sql.operator.{AggregateWithinPartitons, VizPartitioner}
import org.apache.sedona.viz.sql.utils.{Conf, LineageDecoder}
-import org.locationtech.jts.geom.Envelope
import org.apache.spark.sql.functions.lit
+import org.locationtech.jts.geom.Envelope
class optVizOperatorTest extends TestBaseScala {
diff --git a/viz/src/test/scala/org/apache/sedona/viz/sql/standardVizOperatorTest.scala b/spark/common/src/test/scala/org/apache/sedona/viz/sql/standardVizOperatorTest.scala
similarity index 100%
rename from viz/src/test/scala/org/apache/sedona/viz/sql/standardVizOperatorTest.scala
rename to spark/common/src/test/scala/org/apache/sedona/viz/sql/standardVizOperatorTest.scala
diff --git a/viz/.gitignore b/viz/.gitignore
deleted file mode 100644
index b2d2d80f2..000000000
--- a/viz/.gitignore
+++ /dev/null
@@ -1,10 +0,0 @@
-/target/
-/.settings/
-/.classpath
-/.project
-/dependency-reduced-pom.xml
-/doc/
-/.idea/
-*.iml
-/latest/
-/spark-warehouse/
diff --git a/viz/pom.xml b/viz/pom.xml
deleted file mode 100644
index 8d1ae9475..000000000
--- a/viz/pom.xml
+++ /dev/null
@@ -1,177 +0,0 @@
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing,
- ~ software distributed under the License is distributed on an
- ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ~ KIND, either express or implied. See the License for the
- ~ specific language governing permissions and limitations
- ~ under the License.
- -->
-
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
- <parent>
- <groupId>org.apache.sedona</groupId>
- <artifactId>sedona-parent</artifactId>
- <version>1.5.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
- </parent>
- <artifactId>sedona-viz-${spark.compat.version}_${scala.compat.version}</artifactId>
-
- <name>${project.groupId}:${project.artifactId}</name>
- <description>A cluster computing system for processing large-scale spatial data: RDD and SQL for Viz.</description>
- <url>http://sedona.apache.org/</url>
- <packaging>jar</packaging>
-
- <properties>
- <maven.deploy.skip>false</maven.deploy.skip>
- </properties>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.sedona</groupId>
- <artifactId>sedona-common</artifactId>
- <version>${project.version}</version>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.sedona</groupId>
- <artifactId>sedona-spark-${spark.compat.version}_${scala.compat.version}</artifactId>
- <version>${project.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.compat.version}</artifactId>
- <version>${spark.version}</version>
- <scope>provided</scope>
- <exclusions>
- <!-- Make sure Hadoop's guava version is used for older versions of Spark tests -->
- <exclusion>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- </exclusion>
- <!-- Exclude log4j 1 for older versions of Spark-->
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <!-- Exclude log4j-slf4j2-impl for Spark 3.4 -->
- <exclusion>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j2-impl</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.compat.version}</artifactId>
- <version>${spark.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-client</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-1.2-api</artifactId>
- </dependency>
- <dependency>
- <groupId>org.locationtech.jts</groupId>
- <artifactId>jts-core</artifactId>
- </dependency>
- <dependency>
- <groupId>org.geotools</groupId>
- <artifactId>gt-main</artifactId>
- </dependency>
- <dependency>
- <groupId>org.geotools</groupId>
- <artifactId>gt-referencing</artifactId>
- </dependency>
- <dependency>
- <groupId>org.geotools</groupId>
- <artifactId>gt-epsg-hsql</artifactId>
- </dependency>
- <dependency>
- <groupId>org.geotools</groupId>
- <artifactId>gt-coverage</artifactId>
- </dependency>
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <version>${scala.version}</version>
- </dependency>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
- <artifactId>scala-collection-compat_${scala.compat.version}</artifactId>
- <version>${scala-collection-compat.version}</version>
- </dependency>
- <dependency>
- <groupId>org.beryx</groupId>
- <artifactId>awt-color-factory</artifactId>
- <version>1.0.0</version>
- </dependency>
- <dependency>
- <groupId>com.amazonaws</groupId>
- <artifactId>aws-java-sdk-s3</artifactId>
- <version>1.12.362</version>
- <scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>com.fasterxml.jackson.core</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest_${scala.compat.version}</artifactId>
- <version>${scalatest.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <sourceDirectory>src/main/java</sourceDirectory>
- <plugins>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <executions>
- <execution>
- <id>attach-javadocs</id>
- <configuration>
- <skip>true</skip>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest-maven-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- </plugin>
- </plugins>
- </build>
-</project>
-
diff --git a/viz/src/.gitignore b/viz/src/.gitignore
deleted file mode 100644
index 5509140f2..000000000
--- a/viz/src/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.DS_Store
diff --git a/viz/src/main/.gitignore b/viz/src/main/.gitignore
deleted file mode 100644
index 5509140f2..000000000
--- a/viz/src/main/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.DS_Store
diff --git a/viz/src/test/.gitignore b/viz/src/test/.gitignore
deleted file mode 100644
index 5509140f2..000000000
--- a/viz/src/test/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-*.DS_Store
diff --git a/viz/src/test/resources/log4j2.properties b/viz/src/test/resources/log4j2.properties
deleted file mode 100644
index 5f8985946..000000000
--- a/viz/src/test/resources/log4j2.properties
+++ /dev/null
@@ -1,31 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Set everything to be logged to the file target/unit-tests.log
-rootLogger.level = info
-rootLogger.appenderRef.file.ref = File
-
-appender.file.type = File
-appender.file.name = File
-appender.file.fileName = target/unit-tests.log
-appender.file.append = true
-appender.file.layout.type = PatternLayout
-appender.file.layout.pattern = %d{yy/MM/dd HH:mm:ss.SSS} %t %p %c{1}: %m%n%ex
-
-# Ignore messages below warning level from Jetty, because it's a bit verbose
-logger.jetty.name = org.sparkproject.jetty
-logger.jetty.level = warn
[sedona] 02/02: Finalize
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch viz-to-spark
in repository https://gitbox.apache.org/repos/asf/sedona.git
commit df7ce237f2ea7e90a8bd176ecd8ba5e1a2507879
Author: Jia Yu <ji...@apache.org>
AuthorDate: Wed Sep 6 17:40:57 2023 -0700
Finalize
---
.../org/apache/sedona/sql/TestBaseScala.scala | 26 ++-
.../org/apache/sedona/viz/rdd/scalaTest.scala | 181 ---------------------
.../sql/{TestBaseScala.scala => VizTestBase.scala} | 24 +--
.../apache/sedona/viz/sql/optVizOperatorTest.scala | 10 +-
.../sedona/viz/sql/standardVizOperatorTest.scala | 40 ++---
5 files changed, 43 insertions(+), 238 deletions(-)
diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala b/spark/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala
index 662de6d9e..5113f6ab9 100644
--- a/spark/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala
+++ b/spark/common/src/test/scala/org/apache/sedona/sql/TestBaseScala.scala
@@ -20,11 +20,10 @@ package org.apache.sedona.sql
import com.google.common.math.DoubleMath
import org.apache.log4j.{Level, Logger}
-import org.apache.sedona.common.sphere.{Haversine, Spheroid}
import org.apache.sedona.common.Functions.{frechetDistance, hausdorffDistance}
+import org.apache.sedona.common.sphere.{Haversine, Spheroid}
import org.apache.sedona.spark.SedonaContext
-import org.apache.spark.SparkContext
-import org.apache.spark.sql.{DataFrame, SparkSession}
+import org.apache.spark.sql.DataFrame
import org.locationtech.jts.geom.{CoordinateSequence, CoordinateSequenceComparator}
import org.scalatest.{BeforeAndAfterAll, FunSpec}
@@ -36,9 +35,15 @@ trait TestBaseScala extends FunSpec with BeforeAndAfterAll {
Logger.getLogger("org.apache.sedona.core").setLevel(Level.WARN)
val warehouseLocation = System.getProperty("user.dir") + "/target/"
- var sparkSession:SparkSession = null
+ val sparkSession = SedonaContext.builder().
+ master("local[*]").appName("sedonasqlScalaTest")
+ .config("spark.sql.warehouse.dir", warehouseLocation)
+ // We need to be explicit about broadcasting in tests.
+ .config("sedona.join.autoBroadcastJoinThreshold", "-1")
+ .config("spark.kryoserializer.buffer.max", "64m")
+ .getOrCreate()
- var sc: SparkContext = null
+ val sc = sparkSession.sparkContext
val resourceFolder = System.getProperty("user.dir") + "/src/test/resources/"
val mixedWkbGeometryInputLocation = resourceFolder + "county_small_wkb.tsv"
@@ -69,21 +74,12 @@ trait TestBaseScala extends FunSpec with BeforeAndAfterAll {
val smallRasterDataLocation: String = resourceFolder + "raster/test1.tiff"
override def beforeAll(): Unit = {
- sparkSession = SedonaContext.builder().
- master("local[*]").appName("sedonasqlScalaTest")
- .config("spark.sql.warehouse.dir", warehouseLocation)
- // We need to be explicit about broadcasting in tests.
- .config("sedona.join.autoBroadcastJoinThreshold", "-1")
- .config("spark.kryoserializer.buffer.max", "64m")
- .getOrCreate()
-
- sc = sparkSession.sparkContext
SedonaContext.create(sparkSession)
}
override def afterAll(): Unit = {
//SedonaSQLRegistrator.dropAll(spark)
- sparkSession.stop
+// sparkSession.stop
}
def loadCsv(path: String): DataFrame = {
diff --git a/spark/common/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala b/spark/common/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala
deleted file mode 100644
index 068e97ea3..000000000
--- a/spark/common/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.sedona.viz.rdd
-
-import org.apache.log4j.{Level, Logger}
-import org.apache.sedona.common.enums.FileDataSplitter
-import org.apache.sedona.core.enums.{GridType, IndexType}
-import org.apache.sedona.core.formatMapper.EarthdataHDFPointMapper
-import org.apache.sedona.core.spatialOperator.JoinQuery
-import org.apache.sedona.core.spatialRDD.{PointRDD, PolygonRDD, RectangleRDD}
-import org.apache.sedona.viz.`extension`.visualizationEffect.{ChoroplethMap, HeatMap, ScatterPlot}
-import org.apache.sedona.viz.core.Serde.SedonaVizKryoRegistrator
-import org.apache.sedona.viz.core.{ImageGenerator, RasterOverlayOperator}
-import org.apache.sedona.viz.utils.{ColorizeOption, ImageType}
-import org.apache.spark.serializer.KryoSerializer
-import org.apache.spark.{SparkConf, SparkContext}
-import org.locationtech.jts.geom.Envelope
-import org.scalatest.{BeforeAndAfterAll, FunSpec}
-
-import java.awt.Color
-import java.io.FileInputStream
-import java.util.Properties
-
-class scalaTest extends FunSpec with BeforeAndAfterAll{
- val sparkConf = new SparkConf().setAppName("scalaTest").setMaster("local[*]")
- sparkConf.set("spark.serializer", classOf[KryoSerializer].getName)
- sparkConf.set("spark.kryo.registrator", classOf[SedonaVizKryoRegistrator].getName)
- var sparkContext:SparkContext = _
- Logger.getLogger("org").setLevel(Level.WARN)
- Logger.getLogger("akka").setLevel(Level.WARN)
- val prop = new Properties()
- val resourceFolder = System.getProperty("user.dir") + "/src/test/resources/"
- val demoOutputPath = "target/scala/demo"
- var ConfFile = new FileInputStream(resourceFolder + "babylon.point.properties")
- prop.load(ConfFile)
- val scatterPlotOutputPath = System.getProperty("user.dir") + "/" + demoOutputPath + "/scatterplot"
- val heatMapOutputPath = System.getProperty("user.dir") + "/" + demoOutputPath + "/heatmap"
- val choroplethMapOutputPath = System.getProperty("user.dir") + "/" + demoOutputPath + "/choroplethmap"
- val parallelFilterRenderStitchOutputPath = System.getProperty("user.dir") + "/" + demoOutputPath + "/parallelfilterrenderstitchheatmap"
- val earthdataScatterPlotOutputPath = System.getProperty("user.dir") + "/" + demoOutputPath + "/earthdatascatterplot"
- val PointInputLocation = resourceFolder + prop.getProperty("inputLocation")
- val PointOffset = prop.getProperty("offset").toInt
- val PointSplitter = FileDataSplitter.getFileDataSplitter(prop.getProperty("splitter"))
- val PointNumPartitions = prop.getProperty("numPartitions").toInt
- ConfFile = new FileInputStream(resourceFolder + "babylon.rectangle.properties")
- prop.load(ConfFile)
- val RectangleInputLocation = resourceFolder + prop.getProperty("inputLocation")
- val RectangleOffset = prop.getProperty("offset").toInt
- val RectangleSplitter = FileDataSplitter.getFileDataSplitter(prop.getProperty("splitter"))
- val RectangleNumPartitions = prop.getProperty("numPartitions").toInt
- ConfFile = new FileInputStream(resourceFolder + "babylon.polygon.properties")
- prop.load(ConfFile)
- val PolygonInputLocation = resourceFolder + prop.getProperty("inputLocation")
- val PolygonOffset = prop.getProperty("offset").toInt
- val PolygonSplitter = FileDataSplitter.getFileDataSplitter(prop.getProperty("splitter"))
- val PolygonNumPartitions = prop.getProperty("numPartitions").toInt
- ConfFile = new FileInputStream(resourceFolder + "babylon.linestring.properties")
- prop.load(ConfFile)
- val LineStringInputLocation = resourceFolder + prop.getProperty("inputLocation")
- val LineStringOffset = prop.getProperty("offset").toInt
- val LineStringSplitter = FileDataSplitter.getFileDataSplitter(prop.getProperty("splitter"))
- val LineStringNumPartitions = prop.getProperty("numPartitions").toInt
- val USMainLandBoundary = new Envelope(-126.790180, -64.630926, 24.863836, 50.000)
- val earthdataInputLocation = resourceFolder + "modis/modis.csv"
- val earthdataNumPartitions = 5
- val HDFIncrement = 5
- val HDFOffset = 2
- val HDFRootGroupName = "MOD_Swath_LST"
- val HDFDataVariableName = "LST"
- val HDFDataVariableList = Array("LST", "QC", "Error_LST", "Emis_31", "Emis_32")
- val HDFswitchXY = true
- val urlPrefix = resourceFolder + "modis/"
-
- override def beforeAll(): Unit = {
- sparkContext = new SparkContext(sparkConf)
- }
- override def afterAll(): Unit = {
- sparkContext.stop()
- }
- describe("SedonaViz in Scala") {
-
- it("should pass scatter plot") {
- val spatialRDD = new PolygonRDD(sparkContext, PolygonInputLocation, PolygonSplitter, false, PolygonNumPartitions)
- var visualizationOperator = new ScatterPlot(1000, 600, USMainLandBoundary, false)
- visualizationOperator.CustomizeColor(255, 255, 255, 255, Color.GREEN, true)
- visualizationOperator.Visualize(sparkContext, spatialRDD)
- var imageGenerator = new ImageGenerator
- imageGenerator.SaveRasterImageAsLocalFile(visualizationOperator.rasterImage, scatterPlotOutputPath, ImageType.PNG)
- /*
- visualizationOperator = new ScatterPlot(1000, 600, USMainLandBoundary, false, -1, -1, false, true)
- visualizationOperator.CustomizeColor(255, 255, 255, 255, Color.GREEN, true)
- visualizationOperator.Visualize(sparkContext, spatialRDD)
- imageGenerator = new ImageGenerator
- imageGenerator.SaveVectorImageAsLocalFile(visualizationOperator.vectorImage, scatterPlotOutputPath, ImageType.SVG)
- visualizationOperator = new ScatterPlot(1000, 600, USMainLandBoundary, false, -1, -1, true, true)
- visualizationOperator.CustomizeColor(255, 255, 255, 255, Color.GREEN, true)
- visualizationOperator.Visualize(sparkContext, spatialRDD)
- imageGenerator = new ImageGenerator
- imageGenerator.SaveVectorImageAsLocalFile(visualizationOperator.distributedVectorImage, scatterPlotOutputPath + "-distributed", ImageType.SVG)
- */
- true
- }
-
- it("should pass heat map") {
- val spatialRDD = new RectangleRDD(sparkContext, RectangleInputLocation, RectangleSplitter, false, RectangleNumPartitions)
- val visualizationOperator = new HeatMap(1000, 600, USMainLandBoundary, false, 2)
- visualizationOperator.Visualize(sparkContext, spatialRDD)
- val imageGenerator = new ImageGenerator
- imageGenerator.SaveRasterImageAsLocalFile(visualizationOperator.rasterImage, heatMapOutputPath, ImageType.PNG)
- true
- }
-
- it("should pass choropleth map") {
- val spatialRDD = new PointRDD(sparkContext, PointInputLocation, PointOffset, PointSplitter, false, PointNumPartitions)
- val queryRDD = new PolygonRDD(sparkContext, PolygonInputLocation, PolygonSplitter, false, PolygonNumPartitions)
- spatialRDD.analyze();
- spatialRDD.spatialPartitioning(GridType.KDBTREE)
- queryRDD.spatialPartitioning(spatialRDD.getPartitioner)
- spatialRDD.buildIndex(IndexType.RTREE, true)
- val joinResult = JoinQuery.SpatialJoinQueryCountByKey(spatialRDD, queryRDD, true, false)
- val visualizationOperator = new ChoroplethMap(1000, 600, USMainLandBoundary, false)
- visualizationOperator.CustomizeColor(255, 255, 255, 255, Color.RED, true)
- visualizationOperator.Visualize(sparkContext, joinResult)
- val frontImage = new ScatterPlot(1000, 600, USMainLandBoundary, false)
- frontImage.CustomizeColor(0, 0, 0, 255, Color.GREEN, true)
- frontImage.Visualize(sparkContext, queryRDD)
- val overlayOperator = new RasterOverlayOperator(visualizationOperator.rasterImage)
- overlayOperator.JoinImage(frontImage.rasterImage)
- val imageGenerator = new ImageGenerator
- imageGenerator.SaveRasterImageAsLocalFile(overlayOperator.backRasterImage, choroplethMapOutputPath, ImageType.PNG)
- true
- }
-
- it("should pass parallel filtering and rendering without stitching image tiles") {
- val spatialRDD = new RectangleRDD(sparkContext, RectangleInputLocation, RectangleSplitter, false, RectangleNumPartitions)
- val visualizationOperator = new HeatMap(1000, 600, USMainLandBoundary, false, 2, 4, 4, true, true)
- visualizationOperator.Visualize(sparkContext, spatialRDD)
- val imageGenerator = new ImageGenerator
- imageGenerator.SaveRasterImageAsLocalFile(visualizationOperator.distributedRasterImage, parallelFilterRenderStitchOutputPath, ImageType.PNG, 0, 4, 4)
- true
- }
-
- it("should pass parallel filtering and rendering with stitching image tiles") {
- val spatialRDD = new RectangleRDD(sparkContext, RectangleInputLocation, RectangleSplitter, false, RectangleNumPartitions)
- val visualizationOperator = new HeatMap(1000, 600, USMainLandBoundary, false, 2, 4, 4, true, true)
- visualizationOperator.Visualize(sparkContext, spatialRDD)
- val imageGenerator = new ImageGenerator
- imageGenerator.SaveRasterImageAsLocalFile(visualizationOperator.distributedRasterImage, parallelFilterRenderStitchOutputPath, ImageType.PNG)
- true
- }
-
- // Tests here have been ignored. A new feature that reads HDF will be added.
- ignore("should pass earth data hdf scatter plot") {
- val earthdataHDFPoint = new EarthdataHDFPointMapper(HDFIncrement, HDFOffset, HDFRootGroupName,
- HDFDataVariableList, HDFDataVariableName, HDFswitchXY, urlPrefix)
- val spatialRDD = new PointRDD(sparkContext, earthdataInputLocation, earthdataNumPartitions, earthdataHDFPoint)
- val visualizationOperator = new ScatterPlot(1000, 600, spatialRDD.boundaryEnvelope, ColorizeOption.EARTHOBSERVATION, false, false)
- visualizationOperator.CustomizeColor(255, 255, 255, 255, Color.BLUE, true)
- visualizationOperator.Visualize(sparkContext, spatialRDD)
- val imageGenerator = new ImageGenerator
- imageGenerator.SaveRasterImageAsLocalFile(visualizationOperator.rasterImage, earthdataScatterPlotOutputPath, ImageType.PNG)
- true
- }
- }
-}
diff --git a/spark/common/src/test/scala/org/apache/sedona/viz/sql/TestBaseScala.scala b/spark/common/src/test/scala/org/apache/sedona/viz/sql/VizTestBase.scala
similarity index 68%
rename from spark/common/src/test/scala/org/apache/sedona/viz/sql/TestBaseScala.scala
rename to spark/common/src/test/scala/org/apache/sedona/viz/sql/VizTestBase.scala
index 67bb98346..aeabace48 100644
--- a/spark/common/src/test/scala/org/apache/sedona/viz/sql/TestBaseScala.scala
+++ b/spark/common/src/test/scala/org/apache/sedona/viz/sql/VizTestBase.scala
@@ -19,46 +19,36 @@
package org.apache.sedona.viz.sql
import org.apache.log4j.{Level, Logger}
-import org.apache.sedona.spark.SedonaContext
+import org.apache.sedona.sql.TestBaseScala
import org.apache.sedona.viz.sql.utils.SedonaVizRegistrator
-import org.apache.spark.sql.{DataFrame, SparkSession}
-import org.scalatest.{BeforeAndAfterAll, FunSpec}
+import org.apache.spark.sql.DataFrame
-trait TestBaseScala extends FunSpec with BeforeAndAfterAll{
+trait VizTestBase extends TestBaseScala {
Logger.getLogger("org.apache").setLevel(Level.WARN)
Logger.getLogger("com").setLevel(Level.WARN)
Logger.getLogger("akka").setLevel(Level.WARN)
Logger.getLogger("org.datasyslab").setLevel(Level.WARN)
- var spark:SparkSession = _
- val resourceFolder = System.getProperty("user.dir") + "/src/test/resources/"
-
val polygonInputLocationWkt = resourceFolder + "county_small.tsv"
val polygonInputLocation = resourceFolder + "primaryroads-polygon.csv"
- val csvPointInputLocation = resourceFolder + "arealm.csv"
+ val pointInputLocation = resourceFolder + "arealm.csv"
override def beforeAll(): Unit = {
- spark = SedonaContext.create(SedonaContext.builder().
- master("local[*]").appName("SedonaVizSQL").getOrCreate())
- SedonaVizRegistrator.registerAll(spark)
+ SedonaVizRegistrator.registerAll(sparkSession)
getPoint().createOrReplaceTempView("pointtable")
getPolygon().createOrReplaceTempView("usdata")
}
def getPoint(): DataFrame = {
- val pointDf = spark.read.format("csv").option("delimiter", ",").option("header", "false").load(csvPointInputLocation).sample(false, 1)
+ val pointDf = sparkSession.read.format("csv").option("delimiter", ",").option("header", "false").load(pointInputLocation).sample(false, 1)
pointDf.selectExpr("ST_Point(cast(_c0 as Decimal(24,20)),cast(_c1 as Decimal(24,20))) as shape")
.filter("ST_Contains(ST_PolygonFromEnvelope(-126.790180,24.863836,-64.630926,50.000),shape)")
}
def getPolygon():DataFrame = {
- val polygonDf = spark.read.format("csv").option("delimiter", "\t").option("header", "false").load(polygonInputLocationWkt)
+ val polygonDf = sparkSession.read.format("csv").option("delimiter", "\t").option("header", "false").load(polygonInputLocationWkt)
polygonDf.selectExpr("ST_GeomFromWKT(_c0) as shape", "_c1 as rate", "_c2", "_c3")
.filter("ST_Contains(ST_PolygonFromEnvelope(-126.790180,24.863836,-64.630926,50.000),shape)")
}
- override def afterAll(): Unit = {
- spark.stop
- }
-
}
diff --git a/spark/common/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala b/spark/common/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala
index 8969a1afa..8c207c70b 100644
--- a/spark/common/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala
+++ b/spark/common/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala
@@ -24,12 +24,12 @@ import org.apache.sedona.viz.sql.utils.{Conf, LineageDecoder}
import org.apache.spark.sql.functions.lit
import org.locationtech.jts.geom.Envelope
-class optVizOperatorTest extends TestBaseScala {
+class optVizOperatorTest extends VizTestBase {
describe("SedonaViz SQL function Test") {
it("Passed full pipeline using optimized operator") {
- val table = spark.sql(
+ val table = sparkSession.sql(
"""
|SELECT pixel, shape FROM pointtable
|LATERAL VIEW EXPLODE(ST_Pixelize(shape, 1000, 1000, ST_PolygonFromEnvelope(-126.790180,24.863836,-64.630926,50.000))) AS pixel
@@ -49,7 +49,7 @@ class optVizOperatorTest extends TestBaseScala {
// Test the colorize operator
result.createOrReplaceTempView("pixelaggregates")
- val colorTable = spark.sql(
+ val colorTable = sparkSession.sql(
s"""
|SELECT pixel, ${Conf.PrimaryPID}, ${Conf.SecondaryPID}, ST_Colorize(weight, (SELECT max(weight) FROM pixelaggregates))
|FROM pixelaggregates
@@ -59,7 +59,7 @@ class optVizOperatorTest extends TestBaseScala {
}
it("Passed full pipeline - aggregate:avg - color:uniform") {
- var table = spark.sql(
+ var table = sparkSession.sql(
"""
|SELECT pixel, shape FROM pointtable
|LATERAL VIEW EXPLODE(ST_Pixelize(shape, 1000, 1000, ST_PolygonFromEnvelope(-126.790180,24.863836,-64.630926,50.000))) AS pixel
@@ -79,7 +79,7 @@ class optVizOperatorTest extends TestBaseScala {
// Test the colorize operator
result.createOrReplaceTempView("pixelaggregates")
- val colorTable = spark.sql(
+ val colorTable = sparkSession.sql(
s"""
|SELECT pixel, ${Conf.PrimaryPID}, ${Conf.SecondaryPID}, ST_Colorize(weight, 0, 'red')
|FROM pixelaggregates
diff --git a/spark/common/src/test/scala/org/apache/sedona/viz/sql/standardVizOperatorTest.scala b/spark/common/src/test/scala/org/apache/sedona/viz/sql/standardVizOperatorTest.scala
index 9d020c89b..06c77fb52 100644
--- a/spark/common/src/test/scala/org/apache/sedona/viz/sql/standardVizOperatorTest.scala
+++ b/spark/common/src/test/scala/org/apache/sedona/viz/sql/standardVizOperatorTest.scala
@@ -22,31 +22,31 @@ package org.apache.sedona.viz.sql
import org.apache.sedona.viz.core.{ImageGenerator, ImageSerializableWrapper}
import org.apache.sedona.viz.utils.ImageType
-class standardVizOperatorTest extends TestBaseScala {
+class standardVizOperatorTest extends VizTestBase {
describe("SedonaViz SQL function Test") {
it("Generate a single image") {
- spark.sql(
+ sparkSession.sql(
"""
|SELECT pixel, shape FROM pointtable
|LATERAL VIEW EXPLODE(ST_Pixelize(shape, 256, 256, ST_PolygonFromEnvelope(-126.790180,24.863836,-64.630926,50.000)) ) AS pixel
""".stripMargin).createOrReplaceTempView("pixels")
- spark.sql(
+ sparkSession.sql(
"""
|SELECT pixel, count(*) as weight
|FROM pixels
|GROUP BY pixel
""".stripMargin).createOrReplaceTempView("pixelaggregates")
- spark.sql(
+ sparkSession.sql(
"""
|SELECT ST_Render(pixel, ST_Colorize(weight, (SELECT max(weight) FROM pixelaggregates), 'red')) AS image
|FROM pixelaggregates
""".stripMargin).createOrReplaceTempView("images")
- var image = spark.table("images").take(1)(0)(0).asInstanceOf[ImageSerializableWrapper].getImage
+ var image = sparkSession.table("images").take(1)(0)(0).asInstanceOf[ImageSerializableWrapper].getImage
var imageGenerator = new ImageGenerator
imageGenerator.SaveRasterImageAsLocalFile(image, System.getProperty("user.dir") + "/target/points", ImageType.PNG)
- val imageString = spark.sql(
+ val imageString = sparkSession.sql(
"""
|SELECT ST_EncodeImage(image)
|FROM images
@@ -55,11 +55,11 @@ class standardVizOperatorTest extends TestBaseScala {
}
it("Generate a single image using a fat query") {
- spark.sql(
+ sparkSession.sql(
"""
|SELECT ST_Envelope_Aggr(shape) as bound FROM pointtable
""".stripMargin).createOrReplaceTempView("boundtable")
- spark.sql(
+ sparkSession.sql(
"""
|SELECT pixel, shape FROM pointtable
|LATERAL VIEW
@@ -71,14 +71,14 @@ class standardVizOperatorTest extends TestBaseScala {
| (SELECT ST_Transform(bound, 'epsg:4326','epsg:3857') FROM boundtable)
| )) AS pixel
""".stripMargin).createOrReplaceTempView("pixels")
- spark.sql(
+ sparkSession.sql(
"""
|CREATE OR REPLACE TEMP VIEW pixelaggregates AS
|SELECT pixel, count(*) as weight
|FROM pixels
|GROUP BY pixel
""".stripMargin)
- val images = spark.sql(
+ val images = sparkSession.sql(
"""
|SELECT
| ST_EncodeImage(ST_Render(pixel, ST_Colorize(weight, (SELECT max(weight) FROM pixelaggregates)))) AS image,
@@ -89,36 +89,36 @@ class standardVizOperatorTest extends TestBaseScala {
}
it("Passed the pipeline on points") {
- spark.sql(
+ sparkSession.sql(
"""
|SELECT pixel, shape FROM pointtable
|LATERAL VIEW EXPLODE(ST_Pixelize(shape, 1000, 800, ST_PolygonFromEnvelope(-126.790180,24.863836,-64.630926,50.000))) AS pixel
""".stripMargin).createOrReplaceTempView("pixels")
- spark.sql(
+ sparkSession.sql(
"""
|CREATE OR REPLACE TEMP VIEW pixelaggregates AS
|SELECT pixel, count(*) as weight
|FROM pixels
|GROUP BY pixel
""".stripMargin)
- val pixelaggregates = spark.table("pixelaggregates")
+ val pixelaggregates = sparkSession.table("pixelaggregates")
pixelaggregates.show(1)
}
it("Passed the pipeline on polygons") {
- spark.sql(
+ sparkSession.sql(
"""
|SELECT pixel, rate, shape FROM usdata
|LATERAL VIEW EXPLODE(ST_Pixelize(shape, 1000, 1000, ST_PolygonFromEnvelope(-126.790180,24.863836,-64.630926,50.000))) AS pixel
""".stripMargin).createOrReplaceTempView("pixels")
- spark.sql(
+ sparkSession.sql(
"""
|CREATE OR REPLACE TEMP VIEW pixelaggregates AS
|SELECT pixel, count(*) as weight
|FROM pixels
|GROUP BY pixel
""".stripMargin)
- val imageDf = spark.sql(
+ val imageDf = sparkSession.sql(
"""
|SELECT ST_Render(pixel, ST_Colorize(weight, (SELECT max(weight) FROM pixelaggregates))) AS image
|FROM pixelaggregates
@@ -130,24 +130,24 @@ class standardVizOperatorTest extends TestBaseScala {
it("Passed ST_TileName") {
var zoomLevel = 2
- spark.sql(
+ sparkSession.sql(
"""
|SELECT pixel, shape FROM pointtable
|LATERAL VIEW EXPLODE(ST_Pixelize(shape, 1000, 1000, ST_PolygonFromEnvelope(-126.790180,24.863836,-64.630926,50.000))) AS pixel
""".stripMargin).createOrReplaceTempView("pixels")
- spark.sql(
+ sparkSession.sql(
"""
|CREATE OR REPLACE TEMP VIEW pixelaggregates AS
|SELECT pixel, count(*) as weight
|FROM pixels
|GROUP BY pixel
""".stripMargin)
- spark.sql(
+ sparkSession.sql(
s"""
|SELECT pixel, weight, ST_TileName(pixel, $zoomLevel) AS pid
|FROM pixelaggregates
""".stripMargin).createOrReplaceTempView("pixel_weights")
- val images = spark.sql(
+ val images = sparkSession.sql(
s"""
|SELECT ST_Render(pixel, ST_Colorize(weight, (SELECT max(weight) FROM pixelaggregates)), $zoomLevel) AS image
|FROM pixel_weights