You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2026/09/01 01:04:35 UTC
(spark) branch branch-4.x updated: [SPARK-59052][ML][FOLLOWUP] Refine ClassificationModel column expressions and documentation
This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new f36c22be53cf [SPARK-59052][ML][FOLLOWUP] Refine ClassificationModel column expressions and documentation
f36c22be53cf is described below
commit f36c22be53cf5d1245bdd29396d3fa9741f05bbd
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Tue Sep 1 09:04:02 2026 +0800
[SPARK-59052][ML][FOLLOWUP] Refine ClassificationModel column expressions and documentation
### What changes were proposed in this pull request?
This is a follow-up to #58341. It:
- reads the second value of `LinearSVCModel`'s dense raw-prediction vector with the SQL `fget`
expression instead of the generic sparse-aware `vector_get` expression;
- evaluates `rawScore > threshold` before the NaN check, allowing the generated expression to skip
the NaN check when the threshold comparison is false; and
- documents which `ClassificationModel` column-producing methods are used for each configured
output-column combination.
### Why are the changes needed?
`LinearSVCModel.predictRawColumn` always produces a dense two-element vector, so the generic
vector accessor's sparse-vector handling is unnecessary. The reordered predicate avoids work on
the common false branch, and the Scaladoc makes the transform routing between column-producing
methods explicit for model implementers.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
```bash
JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 ./build/sbt \
mllib/scalastyle \
mllib/doc \
'mllib/testOnly org.apache.spark.ml.classification.ClassifierSuite org.apache.spark.ml.classification.LinearSVCSuite'
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-5)
Closes #58384 from zhengruifeng/ml-spark-59052-classifier-followup-dev-6.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Ruifeng Zheng <ru...@apache.org>
(cherry picked from commit aaaca946ea4140b0331d6cd501ed0dc3baf640d5)
Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
.../spark/ml/classification/Classifier.scala | 39 ++++++++++++++++++++++
.../apache/spark/ml/classification/LinearSVC.scala | 7 ++--
2 files changed, 42 insertions(+), 4 deletions(-)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala
index 46fdb93ad9bd..5e29d58a905c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/Classifier.scala
@@ -86,6 +86,37 @@ abstract class Classifier[
* Model produced by a [[Classifier]].
* Classes are indexed {0, 1, ..., numClasses - 1}.
*
+ * `transform` selects column-producing methods based on which output columns are set:
+ *
+ * <table>
+ * <caption>Column-producing methods by requested output columns</caption>
+ * <tr>
+ * <th>Raw prediction</th>
+ * <th>Prediction</th>
+ * <th>Column-producing methods</th>
+ * </tr>
+ * <tr>
+ * <td>Not set</td>
+ * <td>Not set</td>
+ * <td>None</td>
+ * </tr>
+ * <tr>
+ * <td>Set</td>
+ * <td>Not set</td>
+ * <td><code>predictRawColumn</code></td>
+ * </tr>
+ * <tr>
+ * <td>Not set</td>
+ * <td>Set</td>
+ * <td><code>predictionColumn</code></td>
+ * </tr>
+ * <tr>
+ * <td>Set</td>
+ * <td>Set</td>
+ * <td><code>predictRawColumn</code> => <code>raw2predictionColumn</code></td>
+ * </tr>
+ * </table>
+ *
* @tparam FeaturesType Type of input features. E.g., `Vector`
* @tparam M Concrete Model type
*/
@@ -97,6 +128,9 @@ abstract class ClassificationModel[FeaturesType, M <: ClassificationModel[Featur
* The default wraps [[predictRaw]] in a UDF. Models may override this with a native expression or
* a UDF that snapshots prediction state.
*
+ * `transform` uses this whenever [[rawPredictionCol]] is set. When [[predictionCol]] is also set,
+ * it is used together with [[raw2predictionColumn]].
+ *
* @param features input features column
* @return raw-prediction column of type `Vector`
*/
@@ -107,6 +141,9 @@ abstract class ClassificationModel[FeaturesType, M <: ClassificationModel[Featur
/**
* Returns an expression that produces a predicted label from a raw-prediction vector column.
*
+ * `transform` uses this when both [[rawPredictionCol]] and [[predictionCol]] are set. It consumes
+ * the output of [[predictRawColumn]].
+ *
* @param rawPrediction input raw-prediction column
* @return prediction column of type `Double`
*/
@@ -117,6 +154,8 @@ abstract class ClassificationModel[FeaturesType, M <: ClassificationModel[Featur
/**
* Returns an expression that produces a predicted label directly from a features column.
*
+ * `transform` uses this when [[predictionCol]] is set and [[rawPredictionCol]] is not set.
+ *
* @param features input features column
* @return prediction column of type `Double`
*/
diff --git a/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala b/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
index d9b335325643..e994a463bf66 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
@@ -29,7 +29,6 @@ import org.apache.spark.SparkException
import org.apache.spark.annotation.Since
import org.apache.spark.internal.Logging
import org.apache.spark.internal.LogKeys.{COUNT, RANGE}
-import org.apache.spark.ml.{functions => MLFunctions}
import org.apache.spark.ml.feature._
import org.apache.spark.ml.linalg._
import org.apache.spark.ml.optim.aggregator._
@@ -43,7 +42,7 @@ import org.apache.spark.ml.util.Instrumentation.instrumented
import org.apache.spark.mllib.util.MLUtils
import org.apache.spark.rdd.RDD
import org.apache.spark.sql._
-import org.apache.spark.sql.functions._
+import org.apache.spark.sql.functions.{get => fget, _}
import org.apache.spark.storage.StorageLevel
/** Params for linear SVM Classifier. */
@@ -390,8 +389,8 @@ class LinearSVCModel private[classification] (
override protected def raw2predictionColumn(rawPrediction: Column): Column = {
val localThreshold = getThreshold
- val rawScore = MLFunctions.vector_get(rawPrediction, lit(1))
- when(!isnan(rawScore) && rawScore > localThreshold, 1.0).otherwise(0.0)
+ val rawScore = fget(unwrap_udt(rawPrediction).getField("values"), lit(1))
+ when(rawScore > localThreshold && !isnan(rawScore), 1.0).otherwise(0.0)
}
override protected def predictionColumn(features: Column): Column = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org