You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2026/09/09 22:56:07 UTC
(spark) branch master updated: Revert "[SPARK-59307][ML] Detect cyclic node ids when loading decision tree and bisecting k-means models"
This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 5f8de75af009 Revert "[SPARK-59307][ML] Detect cyclic node ids when loading decision tree and bisecting k-means models"
5f8de75af009 is described below
commit 5f8de75af009e332fe8a57da19d60f16efea731d
Author: Hyukjin Kwon <hy...@databricks.com>
AuthorDate: Thu Sep 10 07:50:27 2026 +0900
Revert "[SPARK-59307][ML] Detect cyclic node ids when loading decision tree and bisecting k-means models"
This reverts commit 846e33ef641de122d1e83de6fc16152280fdbbd2.
---
.../mllib/clustering/BisectingKMeansModel.scala | 25 +-----------
.../spark/mllib/tree/model/DecisionTreeModel.scala | 20 ++--------
.../mllib/clustering/BisectingKMeansSuite.scala | 39 +------------------
.../spark/mllib/tree/DecisionTreeSuite.scala | 45 +---------------------
4 files changed, 8 insertions(+), 121 deletions(-)
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.scala
index af97f0bb1ba8..113445269ce7 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.scala
@@ -17,8 +17,6 @@
package org.apache.spark.mllib.clustering
-import scala.collection.mutable
-
import org.json4s._
import org.json4s.DefaultFormats
import org.json4s.JsonDSL._
@@ -158,32 +156,13 @@ object BisectingKMeansModel extends Loader[BisectingKMeansModel] {
}
}
- private def buildTree(rootId: Int, nodes: Map[Int, Data]): ClusteringTreeNode =
- buildTree(rootId, nodes, mutable.Set.empty)
-
- /**
- * `visiting` accumulates every node id reached while building the tree. A valid tree reaches each
- * id exactly once, so encountering an id that is already present means the saved data is not a
- * tree: either its child ids form a cycle (which would otherwise recurse until the driver hits a
- * StackOverflowError) or a child id is shared by more than one parent. Either way we fail with a
- * clear message. Valid trees build unchanged.
- */
- private def buildTree(
- rootId: Int,
- nodes: Map[Int, Data],
- visiting: mutable.Set[Int]): ClusteringTreeNode = {
- if (!visiting.add(rootId)) {
- throw new IllegalArgumentException(
- s"Node id $rootId is reached more than once while loading the bisecting k-means " +
- s"model; the saved node data is not a tree (its child ids form a cycle, or a child " +
- s"id is shared by more than one parent).")
- }
+ private def buildTree(rootId: Int, nodes: Map[Int, Data]): ClusteringTreeNode = {
val root = nodes(rootId)
if (root.children.isEmpty) {
new ClusteringTreeNode(root.index, root.size, new VectorWithNorm(root.center, root.norm),
root.cost, root.height, new Array[ClusteringTreeNode](0))
} else {
- val children = root.children.map(c => buildTree(c, nodes, visiting))
+ val children = root.children.map(c => buildTree(c, nodes))
new ClusteringTreeNode(root.index, root.size, new VectorWithNorm(root.center, root.norm),
root.cost, root.height, children.toArray)
}
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.scala
index dc6cdf6d2aa4..299fd1073f8e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/model/DecisionTreeModel.scala
@@ -279,38 +279,26 @@ object DecisionTreeModel extends Loader[DecisionTreeModel] with Logging {
val dataMap: Map[Int, NodeData] = data.map(n => n.nodeId -> n).toMap
assert(dataMap.contains(1),
s"DecisionTree missing root node (id = 1).")
- constructNode(1, dataMap, mutable.Map.empty, mutable.Set.empty)
+ constructNode(1, dataMap, mutable.Map.empty)
}
/**
* Builds a node from the node data map and adds new nodes to the input nodes map.
- *
- * `visiting` records the node ids currently being constructed (a fully constructed node is in
- * `nodes` and every later visit short-circuits on it before `visiting` is consulted). A saved
- * model whose left/right node ids form a cycle (which never happens for a valid tree) would
- * otherwise recurse until the driver hits a StackOverflowError; detecting a node reached while
- * it is still being constructed turns that into a clear, contained failure. Valid models
- * (including shared leaves, which are served from `nodes`) are unaffected.
*/
private def constructNode(
id: Int,
dataMap: Map[Int, NodeData],
- nodes: mutable.Map[Int, Node],
- visiting: mutable.Set[Int]): Node = {
+ nodes: mutable.Map[Int, Node]): Node = {
if (nodes.contains(id)) {
return nodes(id)
}
- if (!visiting.add(id)) {
- throw new IllegalArgumentException(
- s"Cycle detected among node ids while loading the decision tree model (node id = $id).")
- }
val data = dataMap(id)
val node =
if (data.isLeaf) {
Node(data.nodeId, data.predict.toPredict, data.impurity, data.isLeaf)
} else {
- val leftNode = constructNode(data.leftNodeId.get, dataMap, nodes, visiting)
- val rightNode = constructNode(data.rightNodeId.get, dataMap, nodes, visiting)
+ val leftNode = constructNode(data.leftNodeId.get, dataMap, nodes)
+ val rightNode = constructNode(data.rightNodeId.get, dataMap, nodes)
val stats = new InformationGainStats(data.infoGain.get, data.impurity, leftNode.impurity,
rightNode.impurity, leftNode.predict, rightNode.predict)
new Node(data.nodeId, data.predict.toPredict, data.impurity, data.isLeaf,
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/clustering/BisectingKMeansSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/clustering/BisectingKMeansSuite.scala
index 9cfe0b4355b2..10d5f325d68e 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/clustering/BisectingKMeansSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/clustering/BisectingKMeansSuite.scala
@@ -17,13 +17,10 @@
package org.apache.spark.mllib.clustering
-import org.apache.hadoop.fs.{FileUtil, Path => HadoopPath}
-
import org.apache.spark.SparkFunSuite
import org.apache.spark.mllib.linalg.Vectors
-import org.apache.spark.mllib.util.{Loader, MLlibTestSparkContext}
+import org.apache.spark.mllib.util.MLlibTestSparkContext
import org.apache.spark.mllib.util.TestingUtils._
-import org.apache.spark.sql.functions.{array, col, lit, when}
import org.apache.spark.util.Utils
class BisectingKMeansSuite extends SparkFunSuite with MLlibTestSparkContext {
@@ -202,38 +199,4 @@ class BisectingKMeansSuite extends SparkFunSuite with MLlibTestSparkContext {
Utils.deleteRecursively(tempDir)
}
}
-
- test("load rejects a model whose node ids form a cycle") {
- val srcDir = Utils.createTempDir()
- val dstDir = Utils.createTempDir()
- try {
- val src = srcDir.toURI.toString
- val dst = dstDir.toURI.toString
- val data = sc.parallelize((1 until 8).map(i => Vectors.dense(i.toDouble)), 2)
- val model = new BisectingKMeans().run(data)
- model.save(sc, src)
- val rootId = model.root.index
-
- // Copy the metadata verbatim, and write a modified data set where the root node lists
- // itself as its own child (a cycle). Reading src and writing dst avoids a self-overwrite.
- val hadoopConf = sc.hadoopConfiguration
- val srcMeta = new HadoopPath(Loader.metadataPath(src))
- val dstMeta = new HadoopPath(Loader.metadataPath(dst))
- FileUtil.copy(
- srcMeta.getFileSystem(hadoopConf), srcMeta,
- dstMeta.getFileSystem(hadoopConf), dstMeta, false, hadoopConf)
- spark.read.parquet(Loader.dataPath(src))
- .withColumn("children",
- when(col("index") === rootId, array(lit(rootId))).otherwise(col("children")))
- .write.parquet(Loader.dataPath(dst))
-
- val e = intercept[IllegalArgumentException] {
- BisectingKMeansModel.load(sc, dst)
- }
- assert(e.getMessage.contains("is reached more than once"))
- } finally {
- Utils.deleteRecursively(srcDir)
- Utils.deleteRecursively(dstDir)
- }
- }
}
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/tree/DecisionTreeSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/tree/DecisionTreeSuite.scala
index d31db54b8898..2f5579acea46 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/tree/DecisionTreeSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/tree/DecisionTreeSuite.scala
@@ -19,8 +19,6 @@ package org.apache.spark.mllib.tree
import scala.jdk.CollectionConverters._
-import org.apache.hadoop.fs.{FileUtil, Path => HadoopPath}
-
import org.apache.spark.SparkFunSuite
import org.apache.spark.ml.tree.impl.DecisionTreeMetadata
import org.apache.spark.mllib.linalg.Vectors
@@ -30,8 +28,7 @@ import org.apache.spark.mllib.tree.configuration.FeatureType._
import org.apache.spark.mllib.tree.configuration.Strategy
import org.apache.spark.mllib.tree.impurity.{Entropy, Gini, Variance}
import org.apache.spark.mllib.tree.model._
-import org.apache.spark.mllib.util.{Loader, MLlibTestSparkContext}
-import org.apache.spark.sql.functions.{col, lit, when}
+import org.apache.spark.mllib.util.MLlibTestSparkContext
import org.apache.spark.util.ArrayImplicits._
import org.apache.spark.util.Utils
@@ -434,46 +431,6 @@ class DecisionTreeSuite extends SparkFunSuite with MLlibTestSparkContext {
}
}
}
-
- test("load rejects a model whose node ids form a cycle") {
- val srcDir = Utils.createTempDir()
- val dstDir = Utils.createTempDir()
- try {
- val src = srcDir.toURI.toString
- val dst = dstDir.toURI.toString
- val arr = Array(
- LabeledPoint(0.0, Vectors.dense(0.0)),
- LabeledPoint(1.0, Vectors.dense(1.0)),
- LabeledPoint(0.0, Vectors.dense(0.0)),
- LabeledPoint(1.0, Vectors.dense(1.0)))
- val strategy =
- new Strategy(algo = Classification, impurity = Gini, maxDepth = 4, numClasses = 2)
- val model = DecisionTree.train(sc.parallelize(arr.toImmutableArraySeq), strategy)
- model.save(sc, src)
-
- // Copy the metadata verbatim, and write a modified data set where the root (nodeId 1)
- // points its left child at itself (a cycle). Reading src and writing dst avoids a
- // self-overwrite.
- val hadoopConf = sc.hadoopConfiguration
- val srcMeta = new HadoopPath(Loader.metadataPath(src))
- val dstMeta = new HadoopPath(Loader.metadataPath(dst))
- FileUtil.copy(
- srcMeta.getFileSystem(hadoopConf), srcMeta,
- dstMeta.getFileSystem(hadoopConf), dstMeta, false, hadoopConf)
- spark.read.parquet(Loader.dataPath(src))
- .withColumn("leftNodeId",
- when(col("nodeId") === 1, lit(1)).otherwise(col("leftNodeId")))
- .write.parquet(Loader.dataPath(dst))
-
- val e = intercept[IllegalArgumentException] {
- DecisionTreeModel.load(sc, dst)
- }
- assert(e.getMessage.contains("Cycle detected"))
- } finally {
- Utils.deleteRecursively(srcDir)
- Utils.deleteRecursively(dstDir)
- }
- }
}
object DecisionTreeSuite extends SparkFunSuite {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org