(spark) branch branch-4.x updated: [SPARK-58911][ML][FOLLOWUP] Validate effective CountVectorizer DF range

[ru...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new c999c332a76b [SPARK-58911][ML][FOLLOWUP] Validate effective CountVectorizer DF range
c999c332a76b is described below

commit c999c332a76b3c2ec2d246650998a2705aba1103
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Fri Aug 28 09:59:12 2026 +0800

    [SPARK-58911][ML][FOLLOWUP] Validate effective CountVectorizer DF range
    
    ### What changes were proposed in this pull request?
    
    This follow-up to #57755 validates the effective `minDF` and `maxDF` values after fractional
    thresholds are converted using the input document count. If the effective maximum is below the
    minimum, the DataFrame expression raises an error. Regression coverage includes both combinations
    of fractional and absolute thresholds.
    
    ### Why are the changes needed?
    
    The DataFrame rewrite retained validation when both thresholds use the same units, but dropped the
    post-conversion validation needed when one threshold is fractional and the other is absolute. An
    invalid effective range consequently produced an empty vocabulary instead of reporting the invalid
    parameters.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. On the unreleased `master` branch, invalid mixed-unit DF ranges now raise an error instead of
    silently producing an empty vocabulary. This restores the validation behavior from before #57755.
    
    ### How was this patch tested?
    
    Added regression coverage to `CountVectorizerSuite` for both mixed-unit invalid-range directions.
    The test suite was not run locally. `git diff --check` passes.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex (GPT-5)
    
    Closes #58327 from zhengruifeng/ml-countvectorizer-df-validation-dev-6.
    
    Authored-by: Ruifeng Zheng <ru...@apache.org>
    Signed-off-by: Ruifeng Zheng <ru...@foxmail.com>
    (cherry picked from commit 289160d37f3e24e16a1e5adf90482e0bf5f92517)
    Signed-off-by: Ruifeng Zheng <ru...@foxmail.com>
---
 .../apache/spark/ml/feature/CountVectorizer.scala    |  7 +++++--
 .../spark/ml/feature/CountVectorizerSuite.scala      | 20 ++++++++++++++++++++
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizer.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizer.scala
index 2ef1bf1f43a3..bd1917bb9e34 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizer.scala
@@ -194,8 +194,11 @@ class CountVectorizer @Since("1.5.0") (@Since("1.5.0") override val uid: String)
     val vocSize = $(vocabSize)
     val input = dataset.select($(inputCol))
     val inputSizeCol = input.select(count(lit(0))).scalar()
-    val minDfCol = if ($(minDF) >= 1.0) lit($(minDF)) else inputSizeCol * lit($(minDF))
-    val maxDfCol = if ($(maxDF) >= 1.0) lit($(maxDF)) else inputSizeCol * lit($(maxDF))
+    val rawMinDfCol = if ($(minDF) >= 1.0) lit($(minDF)) else inputSizeCol * lit($(minDF))
+    val rawMaxDfCol = if ($(maxDF) >= 1.0) lit($(maxDF)) else inputSizeCol * lit($(maxDF))
+    val invalidDfRangeError = raise_error(lit("maxDF must be >= minDF."))
+    val minDfCol = when(rawMaxDfCol >= rawMinDfCol, rawMinDfCol).otherwise(invalidDfRangeError)
+    val maxDfCol = when(rawMaxDfCol >= rawMinDfCol, rawMaxDfCol).otherwise(invalidDfRangeError)
     val filteringRequired = isSet(minDF) || isSet(maxDF)
     val wordCounts = if (filteringRequired) {
       input
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/CountVectorizerSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/CountVectorizerSuite.scala
index c00733998050..40474ea816b2 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/CountVectorizerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/CountVectorizerSuite.scala
@@ -16,6 +16,7 @@
  */
 package org.apache.spark.ml.feature
 
+import org.apache.spark.SparkRuntimeException
 import org.apache.spark.ml.linalg.{Vector, Vectors}
 import org.apache.spark.ml.param.ParamsSuite
 import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
@@ -214,6 +215,25 @@ class CountVectorizerSuite extends MLTest with DefaultReadWriteTest {
     }
   }
 
+  test("CountVectorizer validates mixed minDF and maxDF") {
+    val df = Seq(Seq("a"), Seq("b"), Seq("c"), Seq("d")).toDF("words")
+
+    Seq((0.75, 2.0), (3.0, 0.5)).foreach { case (minDf, maxDf) =>
+      val error = intercept[SparkRuntimeException] {
+        new CountVectorizer()
+          .setInputCol("words")
+          .setOutputCol("features")
+          .setMinDF(minDf)
+          .setMaxDF(maxDf)
+          .fit(df)
+      }
+      checkError(
+        exception = error,
+        condition = "USER_RAISED_EXCEPTION",
+        parameters = Map("errorMessage" -> "maxDF must be >= minDF."))
+    }
+  }
+
   test("CountVectorizerModel with minTF count") {
     val df = Seq(
       (0, split("a a a b b c c c d "), Vectors.sparse(4, Seq((0, 3.0), (2, 3.0)))),


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org