You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2026/09/09 22:56:07 UTC

(spark) branch master updated: Revert "[SPARK-59307][ML] Detect cyclic node ids when loading decision tree and bisecting k-means models"

This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 5f8de75af009 Revert "[SPARK-59307][ML] Detect cyclic node ids when loading decision tree and bisecting k-means models"
5f8de75af009 is described below

commit 5f8de75af009e332fe8a57da19d60f16efea731d
Author: Hyukjin Kwon <hy...@databricks.com>
AuthorDate: Thu Sep 10 07:50:27 2026 +0900

    Revert "[SPARK-59307][ML] Detect cyclic node ids when loading decision tree and bisecting k-means models"
    
    This reverts commit 846e33ef641de122d1e83de6fc16152280fdbbd2.
---
 .../mllib/clustering/BisectingKMeansModel.scala    | 25 +-----------
 .../spark/mllib/tree/model/DecisionTreeModel.scala | 20 ++--------
 .../mllib/clustering/BisectingKMeansSuite.scala    | 39 +------------------
 .../spark/mllib/tree/DecisionTreeSuite.scala       | 45 +---------------------
 4 files changed, 8 insertions(+), 121 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.scala
index af97f0bb1ba8..113445269ce7 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.scala
@@ -17,8 +17,6 @@
 
 package org.apache.spark.mllib.clustering
 
-import scala.collection.mutable
-
 import org.json4s._
 import org.json4s.DefaultFormats
 import org.json4s.JsonDSL._
@@ -158,32 +156,13 @@ object BisectingKMeansModel extends Loader[BisectingKMeansModel] {
     }
   }
 
-  private def buildTree(rootId: Int, nodes: Map[Int, Data]): ClusteringTreeNode =
-    buildTree(rootId, nodes, mutable.Set.empty)
-
-  /**
-   * `visiting` accumulates every node id reached while building the tree. A valid tree reaches each
-   * id exactly once, so encountering an id that is already present means the saved data is not a
-   * tree: either its child ids form a cycle (which would otherwise recurse until the driver hits a
-   * StackOverflowError) or a child id is shared by more than one parent. Either way we fail with a
-   * clear message. Valid trees build unchanged.
-   */
-  private def buildTree(
-      rootId: Int,
-      nodes: Map[Int, Data],
-      visiting: mutable.Set[Int]): ClusteringTreeNode = {
-    if (!visiting.add(rootId)) {
-      throw new IllegalArgumentException(
-        s"Node id $rootId is reached more than once while loading the bisecting k-means " +
-          s"model; the saved node data is not a tree (its child ids form a cycle, or a child " +
-          s"id is shared by more than one parent).")
-    }
+  private def buildTree(rootId: Int, nodes: Map[Int, Data]): ClusteringTreeNode = {
     val root = nodes(rootId)
     if (root.children.isEmpty) {
       new ClusteringTreeNode(root.index, root.size, new VectorWithNorm(root.center, root.norm),
         root.cost, root.height, new Array[ClusteringTreeNode](0))
     } else {
-      val children = root.children.map(c => buildTree(c, nodes, visiting))
+      val children = root.children.map(c => buildTree(c, nodes))
       new ClusteringTreeNode(root.index, root.size, new VectorWithNorm(root.center, root.norm),
         root.cost, root.height, children.toArray)
     }
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.scala
index dc6cdf6d2aa4..299fd1073f8e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.scala
@@ -279,38 +279,26 @@ object DecisionTreeModel extends Loader[DecisionTreeModel] with Logging {
       val dataMap: Map[Int, NodeData] = data.map(n => n.nodeId -> n).toMap
       assert(dataMap.contains(1),
         s"DecisionTree missing root node (id = 1).")
-      constructNode(1, dataMap, mutable.Map.empty, mutable.Set.empty)
+      constructNode(1, dataMap, mutable.Map.empty)
     }
 
     /**
      * Builds a node from the node data map and adds new nodes to the input nodes map.
-     *
-     * `visiting` records the node ids currently being constructed (a fully constructed node is in
-     * `nodes` and every later visit short-circuits on it before `visiting` is consulted). A saved
-     * model whose left/right node ids form a cycle (which never happens for a valid tree) would
-     * otherwise recurse until the driver hits a StackOverflowError; detecting a node reached while
-     * it is still being constructed turns that into a clear, contained failure. Valid models
-     * (including shared leaves, which are served from `nodes`) are unaffected.
      */
     private def constructNode(
       id: Int,
       dataMap: Map[Int, NodeData],
-      nodes: mutable.Map[Int, Node],
-      visiting: mutable.Set[Int]): Node = {
+      nodes: mutable.Map[Int, Node]): Node = {
       if (nodes.contains(id)) {
         return nodes(id)
       }
-      if (!visiting.add(id)) {
-        throw new IllegalArgumentException(
-          s"Cycle detected among node ids while loading the decision tree model (node id = $id).")
-      }
       val data = dataMap(id)
       val node =
         if (data.isLeaf) {
           Node(data.nodeId, data.predict.toPredict, data.impurity, data.isLeaf)
         } else {
-          val leftNode = constructNode(data.leftNodeId.get, dataMap, nodes, visiting)
-          val rightNode = constructNode(data.rightNodeId.get, dataMap, nodes, visiting)
+          val leftNode = constructNode(data.leftNodeId.get, dataMap, nodes)
+          val rightNode = constructNode(data.rightNodeId.get, dataMap, nodes)
           val stats = new InformationGainStats(data.infoGain.get, data.impurity, leftNode.impurity,
             rightNode.impurity, leftNode.predict, rightNode.predict)
           new Node(data.nodeId, data.predict.toPredict, data.impurity, data.isLeaf,
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/clustering/BisectingKMeansSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/clustering/BisectingKMeansSuite.scala
index 9cfe0b4355b2..10d5f325d68e 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/clustering/BisectingKMeansSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/clustering/BisectingKMeansSuite.scala
@@ -17,13 +17,10 @@
 
 package org.apache.spark.mllib.clustering
 
-import org.apache.hadoop.fs.{FileUtil, Path => HadoopPath}
-
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.mllib.linalg.Vectors
-import org.apache.spark.mllib.util.{Loader, MLlibTestSparkContext}
+import org.apache.spark.mllib.util.MLlibTestSparkContext
 import org.apache.spark.mllib.util.TestingUtils._
-import org.apache.spark.sql.functions.{array, col, lit, when}
 import org.apache.spark.util.Utils
 
 class BisectingKMeansSuite extends SparkFunSuite with MLlibTestSparkContext {
@@ -202,38 +199,4 @@ class BisectingKMeansSuite extends SparkFunSuite with MLlibTestSparkContext {
       Utils.deleteRecursively(tempDir)
     }
   }
-
-  test("load rejects a model whose node ids form a cycle") {
-    val srcDir = Utils.createTempDir()
-    val dstDir = Utils.createTempDir()
-    try {
-      val src = srcDir.toURI.toString
-      val dst = dstDir.toURI.toString
-      val data = sc.parallelize((1 until 8).map(i => Vectors.dense(i.toDouble)), 2)
-      val model = new BisectingKMeans().run(data)
-      model.save(sc, src)
-      val rootId = model.root.index
-
-      // Copy the metadata verbatim, and write a modified data set where the root node lists
-      // itself as its own child (a cycle). Reading src and writing dst avoids a self-overwrite.
-      val hadoopConf = sc.hadoopConfiguration
-      val srcMeta = new HadoopPath(Loader.metadataPath(src))
-      val dstMeta = new HadoopPath(Loader.metadataPath(dst))
-      FileUtil.copy(
-        srcMeta.getFileSystem(hadoopConf), srcMeta,
-        dstMeta.getFileSystem(hadoopConf), dstMeta, false, hadoopConf)
-      spark.read.parquet(Loader.dataPath(src))
-        .withColumn("children",
-          when(col("index") === rootId, array(lit(rootId))).otherwise(col("children")))
-        .write.parquet(Loader.dataPath(dst))
-
-      val e = intercept[IllegalArgumentException] {
-        BisectingKMeansModel.load(sc, dst)
-      }
-      assert(e.getMessage.contains("is reached more than once"))
-    } finally {
-      Utils.deleteRecursively(srcDir)
-      Utils.deleteRecursively(dstDir)
-    }
-  }
 }
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/tree/DecisionTreeSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/tree/DecisionTreeSuite.scala
index d31db54b8898..2f5579acea46 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/tree/DecisionTreeSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/tree/DecisionTreeSuite.scala
@@ -19,8 +19,6 @@ package org.apache.spark.mllib.tree
 
 import scala.jdk.CollectionConverters._
 
-import org.apache.hadoop.fs.{FileUtil, Path => HadoopPath}
-
 import org.apache.spark.SparkFunSuite
 import org.apache.spark.ml.tree.impl.DecisionTreeMetadata
 import org.apache.spark.mllib.linalg.Vectors
@@ -30,8 +28,7 @@ import org.apache.spark.mllib.tree.configuration.FeatureType._
 import org.apache.spark.mllib.tree.configuration.Strategy
 import org.apache.spark.mllib.tree.impurity.{Entropy, Gini, Variance}
 import org.apache.spark.mllib.tree.model._
-import org.apache.spark.mllib.util.{Loader, MLlibTestSparkContext}
-import org.apache.spark.sql.functions.{col, lit, when}
+import org.apache.spark.mllib.util.MLlibTestSparkContext
 import org.apache.spark.util.ArrayImplicits._
 import org.apache.spark.util.Utils
 
@@ -434,46 +431,6 @@ class DecisionTreeSuite extends SparkFunSuite with MLlibTestSparkContext {
       }
     }
   }
-
-  test("load rejects a model whose node ids form a cycle") {
-    val srcDir = Utils.createTempDir()
-    val dstDir = Utils.createTempDir()
-    try {
-      val src = srcDir.toURI.toString
-      val dst = dstDir.toURI.toString
-      val arr = Array(
-        LabeledPoint(0.0, Vectors.dense(0.0)),
-        LabeledPoint(1.0, Vectors.dense(1.0)),
-        LabeledPoint(0.0, Vectors.dense(0.0)),
-        LabeledPoint(1.0, Vectors.dense(1.0)))
-      val strategy =
-        new Strategy(algo = Classification, impurity = Gini, maxDepth = 4, numClasses = 2)
-      val model = DecisionTree.train(sc.parallelize(arr.toImmutableArraySeq), strategy)
-      model.save(sc, src)
-
-      // Copy the metadata verbatim, and write a modified data set where the root (nodeId 1)
-      // points its left child at itself (a cycle). Reading src and writing dst avoids a
-      // self-overwrite.
-      val hadoopConf = sc.hadoopConfiguration
-      val srcMeta = new HadoopPath(Loader.metadataPath(src))
-      val dstMeta = new HadoopPath(Loader.metadataPath(dst))
-      FileUtil.copy(
-        srcMeta.getFileSystem(hadoopConf), srcMeta,
-        dstMeta.getFileSystem(hadoopConf), dstMeta, false, hadoopConf)
-      spark.read.parquet(Loader.dataPath(src))
-        .withColumn("leftNodeId",
-          when(col("nodeId") === 1, lit(1)).otherwise(col("leftNodeId")))
-        .write.parquet(Loader.dataPath(dst))
-
-      val e = intercept[IllegalArgumentException] {
-        DecisionTreeModel.load(sc, dst)
-      }
-      assert(e.getMessage.contains("Cycle detected"))
-    } finally {
-      Utils.deleteRecursively(srcDir)
-      Utils.deleteRecursively(dstDir)
-    }
-  }
 }
 
 object DecisionTreeSuite extends SparkFunSuite {


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org