You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2023/03/08 04:29:58 UTC
[sedona] branch master updated: [SEDONA-229] Fix Linter Issues in Viz (#791)
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/sedona.git
The following commit(s) were added to refs/heads/master by this push:
new fb1881e3 [SEDONA-229] Fix Linter Issues in Viz (#791)
fb1881e3 is described below
commit fb1881e3907214e5a8c012f18bfba9f3663489a7
Author: Douglas Dennis <do...@gmail.com>
AuthorDate: Tue Mar 7 20:29:52 2023 -0800
[SEDONA-229] Fix Linter Issues in Viz (#791)
---
.../apache/sedona/viz/showcase/ScalaExample.scala | 4 +-
.../org/apache/sedona/viz/sql/UDF/Catalog.scala | 4 +-
.../apache/sedona/viz/sql/UDF/UdfRegistrator.scala | 2 +-
.../sql/operator/AggregateWithinPartitons.scala | 12 +++--
.../sedona/viz/sql/operator/VizPartitioner.scala | 10 ++--
.../sql/sedona_viz/expressions/Colorize.scala | 2 +-
.../sql/sedona_viz/expressions/ImageEncoder.scala | 2 +-
.../sql/sedona_viz/expressions/Partitioner.scala | 11 ++++-
.../sql/sedona_viz/expressions/Pixelize.scala | 55 ++++++++++++++++++----
.../spark/sql/sedona_viz/expressions/Render.scala | 6 +--
.../org/apache/sedona/viz/rdd/scalaTest.scala | 4 +-
.../apache/sedona/viz/sql/optVizOperatorTest.scala | 2 +-
.../sedona/viz/sql/standardVizOperatorTest.scala | 17 +++++--
13 files changed, 97 insertions(+), 34 deletions(-)
diff --git a/viz/src/main/scala/org/apache/sedona/viz/showcase/ScalaExample.scala b/viz/src/main/scala/org/apache/sedona/viz/showcase/ScalaExample.scala
index 3d3652f2..7365e47f 100644
--- a/viz/src/main/scala/org/apache/sedona/viz/showcase/ScalaExample.scala
+++ b/viz/src/main/scala/org/apache/sedona/viz/showcase/ScalaExample.scala
@@ -18,7 +18,7 @@
*/
package org.apache.sedona.viz.showcase
-import java.awt.Color
+import java.awt.Color // scalastyle:ignore illegal.imports
import java.io.FileInputStream
import java.util.Properties
@@ -202,4 +202,4 @@ object ScalaExample extends App {
}
-}
\ No newline at end of file
+}
diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/UDF/Catalog.scala b/viz/src/main/scala/org/apache/sedona/viz/sql/UDF/Catalog.scala
index 1ef376bb..2e87c0b2 100644
--- a/viz/src/main/scala/org/apache/sedona/viz/sql/UDF/Catalog.scala
+++ b/viz/src/main/scala/org/apache/sedona/viz/sql/UDF/Catalog.scala
@@ -20,7 +20,7 @@ package org.apache.sedona.viz.sql.UDF
import org.apache.spark.sql.catalyst.analysis.FunctionRegistry.FunctionBuilder
import org.apache.spark.sql.expressions.UserDefinedAggregateFunction
-import org.apache.spark.sql.sedona_viz.expressions._
+import org.apache.spark.sql.sedona_viz.expressions.{ST_Colorize, ST_EncodeImage, ST_Pixelize, ST_Render, ST_TileName}
object Catalog {
val expressions: Seq[FunctionBuilder] = Seq(
@@ -33,4 +33,4 @@ object Catalog {
val aggregateExpressions: Seq[UserDefinedAggregateFunction] = Seq(
new ST_Render
)
-}
\ No newline at end of file
+}
diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/UDF/UdfRegistrator.scala b/viz/src/main/scala/org/apache/sedona/viz/sql/UDF/UdfRegistrator.scala
index 785f02a0..37c6bee8 100644
--- a/viz/src/main/scala/org/apache/sedona/viz/sql/UDF/UdfRegistrator.scala
+++ b/viz/src/main/scala/org/apache/sedona/viz/sql/UDF/UdfRegistrator.scala
@@ -48,4 +48,4 @@ object UdfRegistrator {
Catalog.expressions.foreach(f => sparkSession.sessionState.functionRegistry.dropFunction(FunctionIdentifier(f.getClass.getSimpleName.dropRight(1))))
Catalog.aggregateExpressions.foreach(f => sparkSession.sessionState.functionRegistry.dropFunction(FunctionIdentifier(f.getClass.getSimpleName)))
}
-}
\ No newline at end of file
+}
diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/operator/AggregateWithinPartitons.scala b/viz/src/main/scala/org/apache/sedona/viz/sql/operator/AggregateWithinPartitons.scala
index e28d249f..e9892c9b 100644
--- a/viz/src/main/scala/org/apache/sedona/viz/sql/operator/AggregateWithinPartitons.scala
+++ b/viz/src/main/scala/org/apache/sedona/viz/sql/operator/AggregateWithinPartitons.scala
@@ -20,7 +20,7 @@ package org.apache.sedona.viz.sql.operator
import org.apache.sedona.viz.sql.utils.Conf
import org.apache.sedona.viz.utils.Pixel
-import org.apache.spark.sql.functions._
+import org.apache.spark.sql.functions.lit
import org.apache.spark.sql.{DataFrame, Row}
import scala.collection.mutable
@@ -55,7 +55,7 @@ object AggregateWithinPartitons {
if (aggFunc.equalsIgnoreCase("count")) formattedDf = dataFrame.select(keyCol, Conf.PrimaryPID, Conf.SecondaryPID).withColumn(valueCol, lit(0.0))
else formattedDf = dataFrame.select(keyCol, Conf.PrimaryPID, Conf.SecondaryPID, valueCol)
- //formattedDf.show()
+ // formattedDf.show()
val aggRdd = formattedDf.rdd.mapPartitions(iterator => {
var aggregator = new mutable.HashMap[Pixel, Tuple4[Double, Double, String, String]]()
@@ -65,7 +65,13 @@ object AggregateWithinPartitons {
val cursorValue = cursorRow.getAs[Double](valueCol)
val currentAggregate = aggregator.getOrElse(cursorKey, Tuple4(0.0, 0.0, "", ""))
// Update the aggregator values, partition ids are appended directly
- aggregator.update(cursorKey, Tuple4(currentAggregate._1 + cursorValue, currentAggregate._2 + 1, cursorRow.getAs[String](Conf.PrimaryPID), cursorRow.getAs[String](Conf.SecondaryPID)))
+ aggregator.update(
+ cursorKey,
+ Tuple4(
+ currentAggregate._1 + cursorValue,
+ currentAggregate._2 + 1,
+ cursorRow.getAs[String](Conf.PrimaryPID),
+ cursorRow.getAs[String](Conf.SecondaryPID)))
}
var result = new ArrayBuffer[Row]()
aggFunc match {
diff --git a/viz/src/main/scala/org/apache/sedona/viz/sql/operator/VizPartitioner.scala b/viz/src/main/scala/org/apache/sedona/viz/sql/operator/VizPartitioner.scala
index 8f2a8049..edc2689a 100644
--- a/viz/src/main/scala/org/apache/sedona/viz/sql/operator/VizPartitioner.scala
+++ b/viz/src/main/scala/org/apache/sedona/viz/sql/operator/VizPartitioner.scala
@@ -22,7 +22,7 @@ import org.apache.sedona.core.spatialPartitioning.QuadtreePartitioning
import org.apache.sedona.core.spatialPartitioning.quadtree.QuadRectangle
import org.apache.sedona.viz.sql.utils.{Conf, LineageDecoder}
import org.apache.sedona.viz.utils.Pixel
-import org.apache.spark.sql.functions._
+import org.apache.spark.sql.functions.{expr, lit}
import org.apache.spark.sql.{DataFrame, Row}
import org.locationtech.jts.geom.{Envelope, Geometry}
@@ -32,8 +32,10 @@ import scala.collection.mutable.ArrayBuffer
object VizPartitioner {
/**
- * Partition the data frame to many pieces. Each partition has two partition ids. Primary id is the uniform map tile id and secondary id is the non-uniformed partition id.
- * The generated DataFrame guarantees that rows with the same secondary ids are in the partition. But each partition may contain rows with different secondary ids.
+ * Partition the data frame to many pieces. Each partition has two partition ids.
+ * Primary id is the uniform map tile id and secondary id is the non-uniformed partition id.
+ * The generated DataFrame guarantees that rows with the same secondary ids are in the partition.
+ * But each partition may contain rows with different secondary ids.
*
* @param dataFrame
* @param zoomLevel
@@ -84,4 +86,4 @@ object VizPartitioner {
dfWithPID.repartition(dfWithPID.select(Conf.SecondaryPID).distinct().count().toInt, expr(Conf.SecondaryPID))
// .sortWithinPartitions(Conf.SecondaryPID)
}
-}
\ No newline at end of file
+}
diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Colorize.scala b/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Colorize.scala
index c059b7ce..cd1a182a 100644
--- a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Colorize.scala
+++ b/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Colorize.scala
@@ -66,4 +66,4 @@ case class ST_Colorize(inputExpressions: Seq[Expression])
protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = {
copy(inputExpressions = newChildren)
}
-}
\ No newline at end of file
+}
diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/ImageEncoder.scala b/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/ImageEncoder.scala
index b138921d..598d43ef 100644
--- a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/ImageEncoder.scala
+++ b/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/ImageEncoder.scala
@@ -51,4 +51,4 @@ case class ST_EncodeImage(inputExpressions: Seq[Expression])
protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = {
copy(inputExpressions = newChildren)
}
-}
\ No newline at end of file
+}
diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Partitioner.scala b/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Partitioner.scala
index 9497b7d4..f1c86487 100644
--- a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Partitioner.scala
+++ b/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Partitioner.scala
@@ -38,7 +38,14 @@ case class ST_TileName(inputExpressions: Seq[Expression])
val partPerAxis = Math.pow(2, zoomLevel).intValue()
val serializer = new PixelSerializer
val pixel = serializer.readPixel(inputArray.toByteArray())
- val id = VisualizationPartitioner.Calculate2DPartitionId(pixel.getResolutionX, pixel.getResolutionY, partPerAxis, partPerAxis, pixel.getX.intValue(), pixel.getY.intValue())
+ val id =
+ VisualizationPartitioner.Calculate2DPartitionId(
+ pixel.getResolutionX,
+ pixel.getResolutionY,
+ partPerAxis,
+ partPerAxis,
+ pixel.getX.intValue(),
+ pixel.getY.intValue())
UTF8String.fromString(zoomLevel + "-" + id._1 + "-" + id._2)
}
@@ -49,4 +56,4 @@ case class ST_TileName(inputExpressions: Seq[Expression])
protected def withNewChildrenInternal(newChildren: IndexedSeq[Expression]) = {
copy(inputExpressions = newChildren)
}
-}
\ No newline at end of file
+}
diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala b/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala
index a0c63043..587b6d4e 100644
--- a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala
+++ b/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Pixelize.scala
@@ -31,7 +31,8 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.catalyst.util.{ArrayData, GenericArrayData}
import org.apache.spark.sql.sedona_viz.UDT.PixelUDT
import org.apache.spark.sql.types.{ArrayType, DataType}
-import org.locationtech.jts.geom._
+import org.locationtech.jts.geom.{Envelope, Geometry, LineString, MultiLineString, MultiPoint, MultiPolygon, Point, Polygon}
+import scala.jdk.CollectionConverters._
case class ST_Pixelize(inputExpressions: Seq[Expression])
extends Expression with CodegenFallback with Logging {
@@ -55,29 +56,67 @@ case class ST_Pixelize(inputExpressions: Seq[Expression])
RasterizationUtils.FindPixelCoordinates(resolutionX, resolutionY, boundary, inputGeometry.asInstanceOf[Point], ColorizeOption.NORMAL, reverseCoordinate)
}
case geometry: MultiLineString => {
- var manyPixels = RasterizationUtils.FindPixelCoordinates(resolutionX, resolutionY, boundary, geometry.getGeometryN(0).asInstanceOf[LineString], reverseCoordinate)
+ var manyPixels =
+ RasterizationUtils.FindPixelCoordinates(
+ resolutionX,
+ resolutionY,
+ boundary,
+ geometry.getGeometryN(0).asInstanceOf[LineString],
+ reverseCoordinate)
for (i <- 1 to geometry.getNumGeometries - 1) {
- manyPixels.addAll(RasterizationUtils.FindPixelCoordinates(resolutionX, resolutionY, boundary, geometry.getGeometryN(i).asInstanceOf[LineString], reverseCoordinate))
+ manyPixels.addAll(
+ RasterizationUtils.FindPixelCoordinates(
+ resolutionX,
+ resolutionY,
+ boundary,
+ geometry.getGeometryN(i).asInstanceOf[LineString],
+ reverseCoordinate))
}
manyPixels
}
case geometry: MultiPolygon => {
- var manyPixels = RasterizationUtils.FindPixelCoordinates(resolutionX, resolutionY, boundary, geometry.getGeometryN(0).asInstanceOf[Polygon], reverseCoordinate)
+ var manyPixels =
+ RasterizationUtils.FindPixelCoordinates(
+ resolutionX,
+ resolutionY,
+ boundary,
+ geometry.getGeometryN(0).asInstanceOf[Polygon],
+ reverseCoordinate)
for (i <- 1 to geometry.getNumGeometries - 1) {
- manyPixels.addAll(RasterizationUtils.FindPixelCoordinates(resolutionX, resolutionY, boundary, geometry.getGeometryN(i).asInstanceOf[Polygon], reverseCoordinate))
+ manyPixels.addAll(
+ RasterizationUtils.FindPixelCoordinates(
+ resolutionX,
+ resolutionY,
+ boundary,
+ geometry.getGeometryN(i).asInstanceOf[Polygon],
+ reverseCoordinate))
}
manyPixels
}
case geometry: MultiPoint => {
- var manyPixels = RasterizationUtils.FindPixelCoordinates(resolutionX, resolutionY, boundary, geometry.getGeometryN(0).asInstanceOf[Point], ColorizeOption.NORMAL, reverseCoordinate)
+ var manyPixels =
+ RasterizationUtils.FindPixelCoordinates(
+ resolutionX,
+ resolutionY,
+ boundary,
+ geometry.getGeometryN(0).asInstanceOf[Point],
+ ColorizeOption.NORMAL,
+ reverseCoordinate)
for (i <- 1 to geometry.getNumGeometries - 1) {
- manyPixels.addAll(RasterizationUtils.FindPixelCoordinates(resolutionX, resolutionY, boundary, geometry.getGeometryN(i).asInstanceOf[Point], ColorizeOption.NORMAL, reverseCoordinate))
+ manyPixels.addAll(
+ RasterizationUtils.FindPixelCoordinates(
+ resolutionX,
+ resolutionY,
+ boundary,
+ geometry.getGeometryN(i).asInstanceOf[Point],
+ ColorizeOption.NORMAL,
+ reverseCoordinate))
}
manyPixels
}
}
assert(pixels.size() > 0)
- import scala.jdk.CollectionConverters._
+
return new GenericArrayData(pixels.asScala.map(f=> {
val out = new ByteArrayOutputStream()
val kryo = new Kryo()
diff --git a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Render.scala b/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Render.scala
index adaba3b8..d3cfcd6c 100644
--- a/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Render.scala
+++ b/viz/src/main/scala/org/apache/spark/sql/sedona_viz/expressions/Render.scala
@@ -18,7 +18,7 @@
*/
package org.apache.spark.sql.sedona_viz.expressions
-import java.awt.image.BufferedImage
+import java.awt.image.BufferedImage // scalastyle:ignore illegal.imports
import org.apache.sedona.viz.core.ImageSerializableWrapper
import org.apache.sedona.viz.utils.Pixel
@@ -26,7 +26,7 @@ import org.apache.spark.internal.Logging
import org.apache.spark.sql.Row
import org.apache.spark.sql.expressions.{MutableAggregationBuffer, UserDefinedAggregateFunction}
import org.apache.spark.sql.sedona_viz.UDT.{ImageWrapperUDT, PixelUDT}
-import org.apache.spark.sql.types._
+import org.apache.spark.sql.types.{ArrayType, DataType, IntegerType, StructType}
case class ST_Render() extends UserDefinedAggregateFunction with Logging {
// This is the input fields for your aggregate function.
@@ -115,4 +115,4 @@ case class ST_Render() extends UserDefinedAggregateFunction with Logging {
}
new ImageSerializableWrapper(bufferedImage)
}
-}
\ No newline at end of file
+}
diff --git a/viz/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala b/viz/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala
index 592917d4..3abdf94c 100644
--- a/viz/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala
+++ b/viz/src/test/scala/org/apache/sedona/viz/rdd/scalaTest.scala
@@ -18,7 +18,7 @@
*/
package org.apache.sedona.viz.rdd
-import java.awt.Color
+import java.awt.Color // scalastyle:ignore illegal.imports
import java.io.FileInputStream
import java.util.Properties
@@ -177,4 +177,4 @@ class scalaTest extends FunSpec with BeforeAndAfterAll{
true
}
}
-}
\ No newline at end of file
+}
diff --git a/viz/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala b/viz/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala
index cf22baef..9fd04855 100644
--- a/viz/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala
+++ b/viz/src/test/scala/org/apache/sedona/viz/sql/optVizOperatorTest.scala
@@ -22,7 +22,7 @@ package org.apache.sedona.viz.sql
import org.apache.sedona.viz.sql.operator.{AggregateWithinPartitons, VizPartitioner}
import org.apache.sedona.viz.sql.utils.{Conf, LineageDecoder}
import org.locationtech.jts.geom.Envelope
-import org.apache.spark.sql.functions._
+import org.apache.spark.sql.functions.lit
class optVizOperatorTest extends TestBaseScala {
diff --git a/viz/src/test/scala/org/apache/sedona/viz/sql/standardVizOperatorTest.scala b/viz/src/test/scala/org/apache/sedona/viz/sql/standardVizOperatorTest.scala
index 33416822..a5346739 100644
--- a/viz/src/test/scala/org/apache/sedona/viz/sql/standardVizOperatorTest.scala
+++ b/viz/src/test/scala/org/apache/sedona/viz/sql/standardVizOperatorTest.scala
@@ -45,7 +45,7 @@ class standardVizOperatorTest extends TestBaseScala {
""".stripMargin).createOrReplaceTempView("images")
var image = spark.table("images").take(1)(0)(0).asInstanceOf[ImageSerializableWrapper].getImage
var imageGenerator = new ImageGenerator
- imageGenerator.SaveRasterImageAsLocalFile(image, System.getProperty("user.dir")+"/target/points", ImageType.PNG)
+ imageGenerator.SaveRasterImageAsLocalFile(image, System.getProperty("user.dir") + "/target/points", ImageType.PNG)
val imageString = spark.sql(
"""
|SELECT ST_EncodeImage(image)
@@ -62,7 +62,14 @@ class standardVizOperatorTest extends TestBaseScala {
spark.sql(
"""
|SELECT pixel, shape FROM pointtable
- |LATERAL VIEW EXPLODE(ST_Pixelize(ST_Transform(ST_FlipCoordinates(shape), 'epsg:4326','epsg:3857'), 256, 256, (SELECT ST_Transform(ST_FlipCoordinates(bound), 'epsg:4326','epsg:3857') FROM boundtable))) AS pixel
+ |LATERAL VIEW
+ | EXPLODE(
+ | ST_Pixelize(
+ | ST_Transform(ST_FlipCoordinates(shape), 'epsg:4326','epsg:3857'),
+ | 256,
+ | 256,
+ | (SELECT ST_Transform(ST_FlipCoordinates(bound), 'epsg:4326','epsg:3857') FROM boundtable)
+ | )) AS pixel
""".stripMargin).createOrReplaceTempView("pixels")
spark.sql(
"""
@@ -73,7 +80,9 @@ class standardVizOperatorTest extends TestBaseScala {
""".stripMargin)
val images = spark.sql(
"""
- |SELECT ST_EncodeImage(ST_Render(pixel, ST_Colorize(weight, (SELECT max(weight) FROM pixelaggregates)))) AS image, (SELECT ST_AsText(bound) FROM boundtable) AS boundary
+ |SELECT
+ | ST_EncodeImage(ST_Render(pixel, ST_Colorize(weight, (SELECT max(weight) FROM pixelaggregates)))) AS image,
+ | (SELECT ST_AsText(bound) FROM boundtable) AS boundary
|FROM pixelaggregates
""".stripMargin)
images.show(1)
@@ -116,7 +125,7 @@ class standardVizOperatorTest extends TestBaseScala {
""".stripMargin)
var image = imageDf.take(1)(0)(0).asInstanceOf[ImageSerializableWrapper].getImage
var imageGenerator = new ImageGenerator
- imageGenerator.SaveRasterImageAsLocalFile(image, System.getProperty("user.dir")+"/target/polygons", ImageType.PNG)
+ imageGenerator.SaveRasterImageAsLocalFile(image, System.getProperty("user.dir") + "/target/polygons", ImageType.PNG)
}
it("Passed ST_TileName") {