You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/08/25 08:45:22 UTC
(spark) branch master updated: [SPARK-53362][ML][CONNECT] Fix IDFModel local loader bug
This is an automated email from the ASF dual-hosted git repository.
weichenxu123 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new c13c10fa194a [SPARK-53362][ML][CONNECT] Fix IDFModel local loader bug
c13c10fa194a is described below
commit c13c10fa194a1870f08cc610a0fc81e9d100a4c0
Author: Weichen Xu <we...@databricks.com>
AuthorDate: Mon Aug 25 16:45:05 2025 +0800
[SPARK-53362][ML][CONNECT] Fix IDFModel local loader bug
### What changes were proposed in this pull request?
Fix IDFModel local loader bug:
https://github.com/apache/spark/blob/50a2ebe87f5ca4dbcc732cf3a543d6ebaea856ad/mllib/src/main/scala/org/apache/spark/ml/feature/IDF.scala#L234
This line code shouldn't be executed when it is loading from local disk. Otherwise file formatting error might be raised.
### Why are the changes needed?
Bugfix.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
updated test in mllib/src/test/scala/org/apache/spark/ml/feature/IDFSuite.scala
### Was this patch authored or co-authored using generative AI tooling?
no.
Closes #52111 from WeichenXu123/SPARK-53362.
Authored-by: Weichen Xu <we...@databricks.com>
Signed-off-by: Weichen Xu <we...@databricks.com>
---
mllib/src/main/scala/org/apache/spark/ml/feature/IDF.scala | 2 +-
.../src/test/scala/org/apache/spark/ml/feature/IDFSuite.scala | 11 +++++++----
python/pyspark/ml/util.py | 3 +++
3 files changed, 11 insertions(+), 5 deletions(-)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/IDF.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/IDF.scala
index 12d957ea360f..289217e1b817 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/IDF.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/IDF.scala
@@ -231,7 +231,6 @@ object IDFModel extends MLReadable[IDFModel] {
override def load(path: String): IDFModel = {
val metadata = DefaultParamsReader.loadMetadata(path, sparkSession, className)
val dataPath = new Path(path, "data").toString
- val data = sparkSession.read.parquet(dataPath)
val model = if (majorVersion(metadata.sparkVersion) >= 3) {
val data = ReadWriteUtils.loadObject[Data](dataPath, sparkSession, deserializeData)
@@ -240,6 +239,7 @@ object IDFModel extends MLReadable[IDFModel] {
new feature.IDFModel(OldVectors.fromML(data.idf), data.docFreq, data.numDocs)
)
} else {
+ val data = sparkSession.read.parquet(dataPath)
val Row(idf: Vector) = MLUtils.convertVectorColumnsToML(data, "idf")
.select("idf")
.head()
diff --git a/mllib/src/test/scala/org/apache/spark/ml/feature/IDFSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/feature/IDFSuite.scala
index 582a11df793e..4f5c8b25de77 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/IDFSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/IDFSuite.scala
@@ -120,9 +120,12 @@ class IDFSuite extends MLTest with DefaultReadWriteTest {
new OldIDFModel(Vectors.dense(1.0, 2.0), Array(1, 2), 2))
.setInputCol("myInputCol")
.setOutputCol("myOutputCol")
- val newInstance = testDefaultReadWrite(instance)
- assert(newInstance.idf === instance.idf)
- assert(newInstance.docFreq === instance.docFreq)
- assert(newInstance.numDocs === instance.numDocs)
+
+ for (testSaveToLocal <- Seq(false, true)) {
+ val newInstance = testDefaultReadWrite(instance, testSaveToLocal = testSaveToLocal)
+ assert(newInstance.idf === instance.idf)
+ assert(newInstance.docFreq === instance.docFreq)
+ assert(newInstance.numDocs === instance.numDocs)
+ }
}
}
diff --git a/python/pyspark/ml/util.py b/python/pyspark/ml/util.py
index 5f23826c73cc..9072e88ca29f 100644
--- a/python/pyspark/ml/util.py
+++ b/python/pyspark/ml/util.py
@@ -339,6 +339,9 @@ def try_remote_call(f: FuncT) -> FuncT:
session.client.execute_command(create_summary_command) # type: ignore
return remote_call()
+
+ # for other unexpected error, re-raise it.
+ raise
else:
return f(self, name, *args)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org