(spark) branch branch-4.x updated: [SPARK-58396][ML][CONNECT] Include FPGrowth metadata in size estimates

[ru...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new 1f9f20412f23 [SPARK-58396][ML][CONNECT] Include FPGrowth metadata in size estimates
1f9f20412f23 is described below

commit 1f9f20412f23da9e7a3f70988478c88f0835d6cb
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Wed Jul 29 08:40:19 2026 +0800

    [SPARK-58396][ML][CONNECT] Include FPGrowth metadata in size estimates
    
    ### What changes were proposed in this pull request?
    
    Adds parameter metadata to `FPGrowthModel.estimatedSize` while retaining its existing frequent-itemset and item-support estimates. Adds regression coverage using the suite's small FP-growth dataset.
    
    ### Why are the changes needed?
    
    The specialized estimate counted learned data but omitted the model's parameter metadata.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Added `FPGrowthModel estimated size` coverage in `FPGrowthSuite`. The suite was not run locally.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex (GPT-5)
    
    Closes #57590 from zhengruifeng/fpgrowth-model-size-metadata-dev3.
    
    Authored-by: Ruifeng Zheng <ru...@apache.org>
    Signed-off-by: Ruifeng Zheng <ru...@apache.org>
    (cherry picked from commit 233f1a414c61b8494388106e4f4d7bea5d35b9ca)
    Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
 mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala      | 8 ++++++--
 mllib/src/test/scala/org/apache/spark/ml/fpm/FPGrowthSuite.scala | 8 ++++++++
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala b/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
index f4150cae13a2..0a5213228a8b 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/fpm/FPGrowth.scala
@@ -324,13 +324,17 @@ class FPGrowthModel private[ml] (
   }
 
   private[spark] override def estimatedSize: Long = {
+    var size = estimateMatadataSize
+    // freqItemsets: DataFrame("items": Array, "freq": Long)
     freqItemsets match {
       case df: org.apache.spark.sql.classic.DataFrame =>
-        df.toArrowBatchRdd.map(_.length.toLong).reduce(_ + _) +
-          SizeEstimator.estimate(itemSupport)
+        size += df.toArrowBatchRdd.map(_.length.toLong).reduce(_ + _)
       case o => throw new UnsupportedOperationException(
         s"Unsupported dataframe type: ${o.getClass.getName}")
     }
+    // itemSupport: scala.collection.Map[Any, Double]
+    size += SizeEstimator.estimate(itemSupport)
+    size
   }
 }
 
diff --git a/mllib/src/test/scala/org/apache/spark/ml/fpm/FPGrowthSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/fpm/FPGrowthSuite.scala
index 3d994366b891..1641e31d1462 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/fpm/FPGrowthSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/fpm/FPGrowthSuite.scala
@@ -154,6 +154,14 @@ class FPGrowthSuite extends SparkFunSuite with MLlibTestSparkContext with Defaul
     ParamsSuite.checkParams(model)
   }
 
+  test("FPGrowthModel estimated size") {
+    val model = new FPGrowth().setMinSupport(0.5).fit(dataset)
+    val maxSize = 1024 * 16
+    assert(
+      model.estimatedSize < maxSize,
+      s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+  }
+
   test("read/write") {
     def checkModelData(model: FPGrowthModel, model2: FPGrowthModel): Unit = {
       assert(model.freqItemsets.collect().toSet.equals(


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org