You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2024/10/12 00:53:31 UTC

(sedona) branch master updated: [SEDONA-662] remove dead code from dbscan code (#1622)

This is an automated email from the ASF dual-hosted git repository.

jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/sedona.git


The following commit(s) were added to refs/heads/master by this push:
     new 1fbac22a6 [SEDONA-662] remove dead code from dbscan code (#1622)
1fbac22a6 is described below

commit 1fbac22a6f1c919cc8b278dcf17c667fa3f15a31
Author: James Willis <ja...@users.noreply.github.com>
AuthorDate: Fri Oct 11 17:53:26 2024 -0700

    [SEDONA-662] remove dead code from dbscan code (#1622)
    
    Co-authored-by: jameswillis <ja...@wherobots.com>
---
 python/sedona/stats/clustering/dbscan.py                             | 1 -
 python/tests/stats/test_dbscan.py                                    | 1 -
 .../src/main/scala/org/apache/sedona/stats/clustering/DBSCAN.scala   | 5 +----
 3 files changed, 1 insertion(+), 6 deletions(-)

diff --git a/python/sedona/stats/clustering/dbscan.py b/python/sedona/stats/clustering/dbscan.py
index bb816e61a..f1501963d 100644
--- a/python/sedona/stats/clustering/dbscan.py
+++ b/python/sedona/stats/clustering/dbscan.py
@@ -25,7 +25,6 @@ from typing import Optional
 from pyspark.sql import DataFrame, SparkSession
 
 ID_COLUMN_NAME = "__id"
-DEFAULT_MAX_SAMPLE_SIZE = 1000000  # 1 million
 
 
 def dbscan(
diff --git a/python/tests/stats/test_dbscan.py b/python/tests/stats/test_dbscan.py
index 60cc8a991..6c6a0d3b3 100644
--- a/python/tests/stats/test_dbscan.py
+++ b/python/tests/stats/test_dbscan.py
@@ -18,7 +18,6 @@
 import pyspark.sql.functions as f
 import pytest
 
-from itertools import product
 from sedona.sql.st_constructors import ST_MakePoint
 from sedona.sql.st_functions import ST_Buffer
 from sklearn.cluster import DBSCAN as sklearnDBSCAN
diff --git a/spark/common/src/main/scala/org/apache/sedona/stats/clustering/DBSCAN.scala b/spark/common/src/main/scala/org/apache/sedona/stats/clustering/DBSCAN.scala
index 5bc691c2d..02a6145cb 100644
--- a/spark/common/src/main/scala/org/apache/sedona/stats/clustering/DBSCAN.scala
+++ b/spark/common/src/main/scala/org/apache/sedona/stats/clustering/DBSCAN.scala
@@ -22,7 +22,7 @@ import org.apache.sedona.stats.Util.getGeometryColumnName
 import org.apache.spark.sql.functions._
 import org.apache.spark.sql.sedona_sql.UDT.GeometryUDT
 import org.apache.spark.sql.sedona_sql.expressions.st_functions.{ST_Distance, ST_DistanceSpheroid}
-import org.apache.spark.sql.{Column, DataFrame, SparkSession}
+import org.apache.spark.sql.{Column, DataFrame}
 import org.graphframes.GraphFrame
 
 object DBSCAN {
@@ -60,9 +60,6 @@ object DBSCAN {
       includeOutliers: Boolean = true,
       useSpheroid: Boolean = false): DataFrame = {
 
-    // We want to disable broadcast joins because the broadcast reference were using too much driver memory
-    val spark = SparkSession.getActiveSession.get
-
     val geometryCol = geometry match {
       case null => getGeometryColumnName(dataframe)
       case _ => geometry