You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/09/03 05:11:24 UTC
(spark) branch master updated: [SPARK-59173][ML] Use isEmpty/nonEmpty instead of size comparisons in MLlib
This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 08757655cc2a [SPARK-59173][ML] Use isEmpty/nonEmpty instead of size comparisons in MLlib
08757655cc2a is described below
commit 08757655cc2a6f613db306dc2480920f2f5b1bb1
Author: Uros <22...@users.noreply.github.com>
AuthorDate: Wed Sep 2 22:11:14 2026 -0700
[SPARK-59173][ML] Use isEmpty/nonEmpty instead of size comparisons in MLlib
### What changes were proposed in this pull request?
This replaces size-vs-zero comparisons with the direct emptiness / non-emptiness predicates in the `mllib` module:
- **Emptiness** (`x.size == 0` -> `x.isEmpty`): `TreeEnsembleModel.totalImportances` (an `OpenHashMap`, which `extends Iterable`) and two `Seq[ShuffleDependency]` checks in `ALSSuite`.
- **Non-emptiness** (`x.size > 0` -> `x.nonEmpty`): `DecisionTreeMetadata.featureArity` and `Strategy.categoricalFeaturesInfo`, both Scala `Map[Int, Int]`.
There are no Java-collection cases in this module.
Deliberately left unchanged, because they are **not** collection-emptiness checks -- they are `mllib.linalg.Vector` dimensions or numeric sizes/thresholds, and `Vector` has no `isEmpty`/`nonEmpty`:
- `ChiSqTest` (`expected` / `observed` are `Vector` dimensions, compared with `!= 0` and `== 0.0`);
- `BinaryClassificationPMMLModelExport` and `GeneralizedLinearPMMLModelExport` (`model.weights`, a `Vector`);
- `MultivariateOnlineSummarizer` (`instance`, a `Vector` -- the message reads "Vector should have dimension larger than zero");
- `BlockMatrix` (a matrix row `Vector`);
- `ALS` (`RatingBlockBuilder.size`, a custom builder with no `nonEmpty`);
- `SpearmanCorrelation` (`cachedUids.size >= 10000000`, a size threshold, not an emptiness check).
### Why are the changes needed?
`isEmpty` / `nonEmpty` state the intent directly. Behavior is unchanged: every converted receiver (`OpenHashMap`, `Seq`, Scala `Map`) defines the predicate as exactly equivalent to the original comparison.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing tests. This is a behavior-preserving refactor; the `mllib` module compiles cleanly. Counterpart of SPARK-59149 (SQL) and SPARK-59172 (core).
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Opus 4.8)
Closes #58471 from uros-b/mllib-isempty.
Authored-by: Uros <22...@users.noreply.github.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../scala/org/apache/spark/ml/tree/impl/DecisionTreeMetadata.scala | 2 +-
mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala | 2 +-
.../scala/org/apache/spark/mllib/tree/configuration/Strategy.scala | 2 +-
.../src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala | 4 ++--
4 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DecisionTreeMetadata.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DecisionTreeMetadata.scala
index 181168385011..ead7b4d27f46 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DecisionTreeMetadata.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/DecisionTreeMetadata.scala
@@ -66,7 +66,7 @@ private[spark] class DecisionTreeMetadata(
def isMulticlass: Boolean = numClasses > 2
- def isMulticlassWithCategoricalFeatures: Boolean = isMulticlass && (featureArity.size > 0)
+ def isMulticlassWithCategoricalFeatures: Boolean = isMulticlass && (featureArity.nonEmpty)
def isCategorical(featureIndex: Int): Boolean = featureArity.contains(featureIndex)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
index 5f45cc331562..b230a18ee064 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
@@ -229,7 +229,7 @@ private[ml] object TreeEnsembleModel {
maxFeatureIndex + 1
}
if (d == 0) {
- assert(totalImportances.size == 0, s"Unknown error in computing feature" +
+ assert(totalImportances.isEmpty, s"Unknown error in computing feature" +
s" importance: No splits found, but some non-zero importances.")
}
val (indices, values) = totalImportances.iterator.toSeq.sortBy(_._1).unzip
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/tree/configuration/Strategy.scala b/mllib/src/main/scala/org/apache/spark/mllib/tree/configuration/Strategy.scala
index c1f82157e82f..ab6c6f746756 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/tree/configuration/Strategy.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/tree/configuration/Strategy.scala
@@ -98,7 +98,7 @@ class Strategy @Since("1.3.0") (
*/
@Since("1.2.0")
def isMulticlassWithCategoricalFeatures: Boolean = {
- isMulticlassClassification && (categoricalFeaturesInfo.size > 0)
+ isMulticlassClassification && (categoricalFeaturesInfo.nonEmpty)
}
// scalastyle:off argcount
diff --git a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
index a5bff057bfec..58cc9900d90d 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
@@ -995,8 +995,8 @@ class ALSSuite extends MLTest with DefaultReadWriteTest with Logging {
val shuffledItemFactors = getShuffledDependencies(itemFactors.rdd).filter { dep =>
dep.rdd.name != null && dep.rdd.name.contains("itemFactors")
}
- assert(shuffledUserFactors.size == 0)
- assert(shuffledItemFactors.size == 0)
+ assert(shuffledUserFactors.isEmpty)
+ assert(shuffledItemFactors.isEmpty)
}
private def checkRecommendations(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org