You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2026/09/09 07:44:49 UTC
(spark) branch master updated: [SPARK-59339][ML][SQL] Add a SQL expression for ML vector dot products
This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 70d1a0dfcbe1 [SPARK-59339][ML][SQL] Add a SQL expression for ML vector dot products
70d1a0dfcbe1 is described below
commit 70d1a0dfcbe13a20b3d1fcb848e7cdacdc1fcb7a
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Wed Sep 9 15:44:36 2026 +0800
[SPARK-59339][ML][SQL] Add a SQL expression for ML vector dot products
### What changes were proposed in this pull request?
This PR adds an internal `VectorDotProduct` Catalyst expression, registered as
`ml_vector_dot_product`, for MLlib dense and sparse vectors. The expression supports interpreted
evaluation and whole-stage code generation for all dense/sparse input combinations.
It also adds private ML helpers for vector-column and constant-coefficient inputs. This PR does not
integrate the expression into any ML algorithm yet.
### Why are the changes needed?
The internal expression operates directly on the vector's SQL struct representation, preserves
sparse inputs without densifying them, and allows Catalyst code generation. It provides a focused
building block for future ML transform optimizations without changing an algorithm in this PR.
### Does this PR introduce _any_ user-facing change?
No. This changes the internal execution of ML model transforms.
### How was this patch tested?
New tests cover interpreted and code-generated evaluation for dense/dense, dense/sparse,
sparse/dense, and sparse/sparse inputs, as well as nulls, empty vectors, infinite and NaN values,
and mismatched dimensions.
The following focused suites and cases were run:
```text
build/sbt 'catalyst/testOnly *VectorDotProductSuite'
build/sbt 'mllib/testOnly *FunctionsSuite -- -z "vector_dot_product"'
```
`dev/scalastyle` also passed.
#### Benchmark
The expression was compared with a Scala UDF implementing `(v1, v2) => v1.dot(v2)`. Inputs were
cached and consumed through a noop sink on Java 17 with `local[1]`. Each case had a two-second
warm-up followed by five timed iterations; the table reports the best iteration. Sparse vectors
had 10% density. Row counts were scaled inversely with dimensionality to keep approximately 10
million dense feature operations per iteration.
| Features | Rows | Inputs | `ml_vector_dot_product` | Scala UDF | Speedup |
|---------:|-----:|--------|------------------------:|----------:|--------:|
| 10 | 1,000,000 | dense/dense | 105 ms | 189 ms | 1.81x |
| 10 | 1,000,000 | sparse/dense | 92 ms | 205 ms | 2.22x |
| 10 | 1,000,000 | sparse/sparse | 97 ms | 238 ms | 2.46x |
| 100 | 100,000 | dense/dense | 45 ms | 58 ms | 1.30x |
| 100 | 100,000 | sparse/dense | 29 ms | 55 ms | 1.90x |
| 100 | 100,000 | sparse/sparse | 23 ms | 52 ms | 2.23x |
| 1,000 | 10,000 | dense/dense | 39 ms | 51 ms | 1.31x |
| 1,000 | 10,000 | sparse/dense | 21 ms | 44 ms | 2.10x |
| 1,000 | 10,000 | sparse/sparse | 14 ms | 35 ms | 2.50x |
| 10,000 | 1,000 | dense/dense | 36 ms | 52 ms | 1.44x |
| 10,000 | 1,000 | sparse/dense | 22 ms | 41 ms | 1.82x |
| 10,000 | 1,000 | sparse/sparse | 14 ms | 32 ms | 2.26x |
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-5)
Closes #58626 from zhengruifeng/ml-vector-dot-dev-5.
Authored-by: Ruifeng Zheng <ru...@apache.org>
Signed-off-by: Ruifeng Zheng <ru...@foxmail.com>
---
.../main/scala/org/apache/spark/ml/functions.scala | 26 +++
.../scala/org/apache/spark/ml/FunctionsSuite.scala | 36 ++++
.../sql/catalyst/analysis/FunctionRegistry.scala | 1 +
.../catalyst/expressions/ml/VectorDotProduct.scala | 239 +++++++++++++++++++++
.../expressions/ml/VectorDotProductSuite.scala | 96 +++++++++
5 files changed, 398 insertions(+)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/functions.scala b/mllib/src/main/scala/org/apache/spark/ml/functions.scala
index ed22ea868e47..87a8a7d98ea0 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/functions.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/functions.scala
@@ -18,8 +18,10 @@
package org.apache.spark.ml
import org.apache.spark.annotation.Since
+import org.apache.spark.ml.linalg.{DenseVector, SparseVector, Vector}
import org.apache.spark.sql.{functions => sf}
import org.apache.spark.sql.Column
+import org.apache.spark.sql.types.{ArrayType, IntegerType}
// scalastyle:off
@Since("3.0.0")
@@ -59,6 +61,30 @@ object functions {
Column.internalFn("ml_vector_posexplode", sf.unwrap_udt(v), sf.lit(mode))
}
+ private[ml] def vector_dot_product(left: Column, right: Column): Column =
+ Column.internalFn("ml_vector_dot_product", sf.unwrap_udt(left), sf.unwrap_udt(right))
+
+ private[ml] def vector_dot_product(left: Column, right: Vector): Column = {
+ val rightStruct = right match {
+ case sparse: SparseVector =>
+ sf.struct(
+ sf.lit(0.toByte).alias("type"),
+ sf.lit(sparse.size).alias("size"),
+ sf.lit(sparse.indices).alias("indices"),
+ sf.lit(sparse.values).alias("values"))
+ case dense: DenseVector =>
+ sf.struct(
+ sf.lit(1.toByte).alias("type"),
+ sf.lit(null).cast(IntegerType).alias("size"),
+ sf.lit(null).cast(ArrayType(IntegerType)).alias("indices"),
+ sf.lit(dense.values).alias("values"))
+ }
+ Column.internalFn(
+ "ml_vector_dot_product",
+ sf.unwrap_udt(left),
+ rightStruct)
+ }
+
private[ml] def array_binary_search(a: Column, v: Column): Column =
Column.internalFn("array_binary_search", a, v)
diff --git a/mllib/src/test/scala/org/apache/spark/ml/FunctionsSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/FunctionsSuite.scala
index c6c478fe31ba..ddc70e532b82 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/FunctionsSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/FunctionsSuite.scala
@@ -220,6 +220,42 @@ class FunctionsSuite extends MLTest {
assert(generators.map(_.prettyName).distinct === Seq("ml_vector_posexplode"))
}
+ test("test vector_dot_product") {
+ val df = Seq(
+ (Vectors.dense(1.0, 2.0, 3.0), Vectors.dense(4.0, 5.0, 6.0)),
+ (Vectors.dense(1.0, 2.0, 3.0), Vectors.sparse(3, Seq((0, 4.0), (2, 6.0)))),
+ (Vectors.sparse(3, Seq((0, 1.0), (2, 3.0))), Vectors.dense(4.0, 5.0, 6.0)),
+ (Vectors.sparse(3, Seq((0, 1.0), (2, 3.0))),
+ Vectors.sparse(3, Seq((0, 4.0), (2, 6.0)))),
+ (Vectors.dense(Array.emptyDoubleArray), Vectors.sparse(0, Seq.empty)),
+ (null.asInstanceOf[Vector], Vectors.dense(1.0)))
+ .toDF("left", "right")
+
+ val result = df.select(vector_dot_product($"left", $"right")).collect().map(_.get(0)).toSeq
+ assert(result === Seq(32.0, 22.0, 22.0, 22.0, 0.0, null))
+ assert(df.select(vector_dot_product($"left", $"right")).schema.head.dataType.typeName ===
+ "double")
+
+ val constantResult = df.limit(4)
+ .select(vector_dot_product($"left", Vectors.dense(4.0, 5.0, 6.0)))
+ .as[Double]
+ .collect()
+ assert(constantResult === Array(32.0, 32.0, 22.0, 22.0))
+ val sparseConstantResult = df.limit(4)
+ .select(vector_dot_product($"left", Vectors.sparse(3, Seq((0, 4.0), (2, 6.0)))))
+ .as[Double]
+ .collect()
+ assert(sparseConstantResult === Array.fill(4)(22.0))
+
+ val error = intercept[IllegalArgumentException] {
+ Seq((Vectors.dense(1.0), Vectors.dense(1.0, 2.0)))
+ .toDF("left", "right")
+ .select(vector_dot_product($"left", $"right"))
+ .collect()
+ }
+ assert(error.getMessage.contains("vectors with non-matching sizes"))
+ }
+
test("test get_vector") {
val df = Seq(
(Vectors.dense(1.0, 2.0, 3.0), 0),
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index 3fbf0570f2e6..a55f4f089a78 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -1222,6 +1222,7 @@ object FunctionRegistry {
registerInternalExpression[CastTimestampNTZToLong]("timestamp_ntz_to_long")
registerInternalExpression[ArrayBinarySearch]("array_binary_search")
registerInternalExpression[VectorPosExplode]("ml_vector_posexplode")
+ registerInternalExpression[VectorDotProduct]("ml_vector_dot_product")
}
registerInternalExpressions()
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ml/VectorDotProduct.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ml/VectorDotProduct.scala
new file mode 100644
index 000000000000..77ca899b8469
--- /dev/null
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/ml/VectorDotProduct.scala
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions.ml
+
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.expressions.{BinaryExpression, ExpectsInputTypes, Expression}
+import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCode}
+import org.apache.spark.sql.catalyst.util.ArrayData
+import org.apache.spark.sql.types._
+
+/**
+ * Computes the dot product of two SQL struct representations of MLlib vectors. This expression is
+ * dedicated only for Spark ML and should be used together with `unwrap_udt`.
+ */
+case class VectorDotProduct(left: Expression, right: Expression)
+ extends BinaryExpression with ExpectsInputTypes {
+
+ override def nullIntolerant: Boolean = true
+
+ override def prettyName: String = "ml_vector_dot_product"
+
+ override def inputTypes: Seq[AbstractDataType] =
+ Seq(VectorDotProduct.vectorSqlType, VectorDotProduct.vectorSqlType)
+
+ override def dataType: DataType = DoubleType
+
+ override def nullable: Boolean = left.nullable || right.nullable
+
+ override protected def nullSafeEval(leftInput: Any, rightInput: Any): Any = {
+ VectorDotProduct.dot(
+ leftInput.asInstanceOf[InternalRow],
+ rightInput.asInstanceOf[InternalRow])
+ }
+
+ override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
+ nullSafeCodeGen(ctx, ev, (leftInput, rightInput) => {
+ val leftType = ctx.freshName("leftType")
+ val leftValues = ctx.freshName("leftValues")
+ val leftSize = ctx.freshName("leftSize")
+ val rightType = ctx.freshName("rightType")
+ val rightValues = ctx.freshName("rightValues")
+ val rightSize = ctx.freshName("rightSize")
+ val index = ctx.freshName("index")
+ val size = ctx.freshName("size")
+ val sparseIndices = ctx.freshName("sparseIndices")
+ val leftIndices = ctx.freshName("leftIndices")
+ val rightIndices = ctx.freshName("rightIndices")
+ val leftIndex = ctx.freshName("leftIndex")
+ val rightIndex = ctx.freshName("rightIndex")
+ val leftNumActives = ctx.freshName("leftNumActives")
+ val rightNumActives = ctx.freshName("rightNumActives")
+ val activeIndex = ctx.freshName("activeIndex")
+
+ s"""
+ |final byte $leftType = $leftInput.getByte(0);
+ |final ArrayData $leftValues = $leftInput.getArray(3);
+ |int $leftSize = -1;
+ |if ($leftType == ${VectorDotProduct.SparseVectorType}) {
+ | $leftSize = $leftInput.getInt(1);
+ |} else if ($leftType == ${VectorDotProduct.DenseVectorType}) {
+ | $leftSize = $leftValues.numElements();
+ |} else {
+ | throw new IllegalArgumentException("Unknown vector type " + $leftType + ".");
+ |}
+ |
+ |final byte $rightType = $rightInput.getByte(0);
+ |final ArrayData $rightValues = $rightInput.getArray(3);
+ |int $rightSize = -1;
+ |if ($rightType == ${VectorDotProduct.SparseVectorType}) {
+ | $rightSize = $rightInput.getInt(1);
+ |} else if ($rightType == ${VectorDotProduct.DenseVectorType}) {
+ | $rightSize = $rightValues.numElements();
+ |} else {
+ | throw new IllegalArgumentException("Unknown vector type " + $rightType + ".");
+ |}
+ |
+ |if ($leftSize != $rightSize) {
+ | throw new IllegalArgumentException(
+ | "requirement failed: VectorDotProduct was given vectors with non-matching sizes:" +
+ | " left.size = " + $leftSize + ", right.size = " + $rightSize);
+ |}
+ |
+ |${ev.value} = 0.0D;
+ |if ($leftType == ${VectorDotProduct.DenseVectorType} &&
+ | $rightType == ${VectorDotProduct.DenseVectorType}) {
+ | for (int $index = 0; $index < $leftSize; $index++) {
+ | ${ev.value} += $leftValues.getDouble($index) * $rightValues.getDouble($index);
+ | }
+ |} else if ($leftType == ${VectorDotProduct.SparseVectorType} &&
+ | $rightType == ${VectorDotProduct.DenseVectorType}) {
+ | final ArrayData $sparseIndices = $leftInput.getArray(2);
+ | final int $size = $sparseIndices.numElements();
+ | for (int $index = 0; $index < $size; $index++) {
+ | ${ev.value} += $leftValues.getDouble($index) *
+ | $rightValues.getDouble($sparseIndices.getInt($index));
+ | }
+ |} else if ($leftType == ${VectorDotProduct.DenseVectorType} &&
+ | $rightType == ${VectorDotProduct.SparseVectorType}) {
+ | final ArrayData $sparseIndices = $rightInput.getArray(2);
+ | final int $size = $sparseIndices.numElements();
+ | for (int $index = 0; $index < $size; $index++) {
+ | ${ev.value} += $rightValues.getDouble($index) *
+ | $leftValues.getDouble($sparseIndices.getInt($index));
+ | }
+ |} else {
+ | final ArrayData $leftIndices = $leftInput.getArray(2);
+ | final ArrayData $rightIndices = $rightInput.getArray(2);
+ | final int $leftNumActives = $leftIndices.numElements();
+ | final int $rightNumActives = $rightIndices.numElements();
+ | int $leftIndex = 0;
+ | int $rightIndex = 0;
+ | while ($leftIndex < $leftNumActives && $rightIndex < $rightNumActives) {
+ | final int $activeIndex = $leftIndices.getInt($leftIndex);
+ | while ($rightIndex < $rightNumActives &&
+ | $rightIndices.getInt($rightIndex) < $activeIndex) {
+ | $rightIndex++;
+ | }
+ | if ($rightIndex < $rightNumActives &&
+ | $rightIndices.getInt($rightIndex) == $activeIndex) {
+ | ${ev.value} += $leftValues.getDouble($leftIndex) *
+ | $rightValues.getDouble($rightIndex);
+ | $rightIndex++;
+ | }
+ | $leftIndex++;
+ | }
+ |}
+ |""".stripMargin
+ })
+ }
+
+ override protected def withNewChildrenInternal(
+ newLeft: Expression,
+ newRight: Expression): VectorDotProduct = copy(left = newLeft, right = newRight)
+}
+
+object VectorDotProduct {
+ private val SparseVectorType: Byte = 0
+ private val DenseVectorType: Byte = 1
+
+ private[ml] val vectorSqlType = StructType(Array(
+ StructField("type", ByteType, nullable = false),
+ StructField("size", IntegerType, nullable = true),
+ StructField("indices", ArrayType(IntegerType, containsNull = false), nullable = true),
+ StructField("values", ArrayType(DoubleType, containsNull = false), nullable = true)))
+
+ private def vectorSize(vector: InternalRow, vectorType: Byte, values: ArrayData): Int = {
+ vectorType match {
+ case SparseVectorType => vector.getInt(1)
+ case DenseVectorType => values.numElements()
+ case _ => throw new IllegalArgumentException(s"Unknown vector type $vectorType.")
+ }
+ }
+
+ private def dot(left: InternalRow, right: InternalRow): Double = {
+ val leftType = left.getByte(0)
+ val leftValues = left.getArray(3)
+ val leftSize = vectorSize(left, leftType, leftValues)
+ val rightType = right.getByte(0)
+ val rightValues = right.getArray(3)
+ val rightSize = vectorSize(right, rightType, rightValues)
+ require(leftSize == rightSize,
+ "VectorDotProduct was given vectors with non-matching sizes:" +
+ s" left.size = $leftSize, right.size = $rightSize")
+ (leftType, rightType) match {
+ case (DenseVectorType, DenseVectorType) =>
+ dotDenseDense(leftValues, rightValues)
+ case (SparseVectorType, DenseVectorType) =>
+ dotSparseDense(left.getArray(2), leftValues, rightValues)
+ case (DenseVectorType, SparseVectorType) =>
+ dotSparseDense(right.getArray(2), rightValues, leftValues)
+ case (SparseVectorType, SparseVectorType) =>
+ dotSparseSparse(left.getArray(2), leftValues, right.getArray(2), rightValues)
+ }
+ }
+
+ private def dotDenseDense(left: ArrayData, right: ArrayData): Double = {
+ var sum = 0.0
+ var index = 0
+ val size = left.numElements()
+ while (index < size) {
+ sum += left.getDouble(index) * right.getDouble(index)
+ index += 1
+ }
+ sum
+ }
+
+ private def dotSparseDense(
+ sparseIndices: ArrayData,
+ sparseValues: ArrayData,
+ denseValues: ArrayData): Double = {
+ var sum = 0.0
+ var index = 0
+ val size = sparseIndices.numElements()
+ while (index < size) {
+ sum += sparseValues.getDouble(index) * denseValues.getDouble(sparseIndices.getInt(index))
+ index += 1
+ }
+ sum
+ }
+
+ private def dotSparseSparse(
+ leftIndices: ArrayData,
+ leftValues: ArrayData,
+ rightIndices: ArrayData,
+ rightValues: ArrayData): Double = {
+ var sum = 0.0
+ var leftIndex = 0
+ var rightIndex = 0
+ val leftSize = leftIndices.numElements()
+ val rightSize = rightIndices.numElements()
+ while (leftIndex < leftSize && rightIndex < rightSize) {
+ val index = leftIndices.getInt(leftIndex)
+ while (rightIndex < rightSize && rightIndices.getInt(rightIndex) < index) {
+ rightIndex += 1
+ }
+ if (rightIndex < rightSize && rightIndices.getInt(rightIndex) == index) {
+ sum += leftValues.getDouble(leftIndex) * rightValues.getDouble(rightIndex)
+ rightIndex += 1
+ }
+ leftIndex += 1
+ }
+ sum
+ }
+}
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ml/VectorDotProductSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ml/VectorDotProductSuite.scala
new file mode 100644
index 000000000000..484c0e32af1a
--- /dev/null
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ml/VectorDotProductSuite.scala
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.catalyst.expressions.ml
+
+import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.expressions.{ExpressionEvalHelper, GenericInternalRow, Literal, UnsafeArrayData}
+
+class VectorDotProductSuite extends SparkFunSuite with ExpressionEvalHelper {
+ private val vectorSqlType = VectorDotProduct.vectorSqlType
+
+ private def dense(values: Double*): Literal = {
+ val row = new GenericInternalRow(Array[Any](
+ 1.toByte,
+ null,
+ null,
+ UnsafeArrayData.fromPrimitiveArray(values.toArray)))
+ Literal(row, vectorSqlType)
+ }
+
+ private def sparse(size: Int, indices: Array[Int], values: Array[Double]): Literal = {
+ val row = new GenericInternalRow(Array[Any](
+ 0.toByte,
+ size,
+ UnsafeArrayData.fromPrimitiveArray(indices),
+ UnsafeArrayData.fromPrimitiveArray(values)))
+ Literal(row, vectorSqlType)
+ }
+
+ test("vector dot product interpreted and code-generated evaluation") {
+ val denseVector = dense(1.0, 2.0, 3.0)
+ val sparseVector = sparse(3, Array(0, 2), Array(1.0, 3.0))
+ val denseWeights = dense(4.0, 5.0, 6.0)
+ val sparseWeights = sparse(3, Array(0, 2), Array(4.0, 6.0))
+
+ val expression = VectorDotProduct(denseVector, denseWeights)
+ assert(expression.prettyName === "ml_vector_dot_product")
+ checkEvaluation(expression, 32.0)
+ checkEvaluation(VectorDotProduct(denseVector, sparseWeights), 22.0)
+ checkEvaluation(VectorDotProduct(sparseVector, denseWeights), 22.0)
+ checkEvaluation(VectorDotProduct(sparseVector, sparseWeights), 22.0)
+ }
+
+ test("vector dot product with null vectors") {
+ val nullVector = Literal(null, vectorSqlType)
+ val vector = dense(1.0)
+
+ checkEvaluation(VectorDotProduct(nullVector, vector), null)
+ checkEvaluation(VectorDotProduct(vector, nullVector), null)
+ }
+
+ test("vector dot product with empty vectors") {
+ val emptyDense = dense()
+ val emptySparse = sparse(0, Array.emptyIntArray, Array.emptyDoubleArray)
+
+ checkEvaluation(VectorDotProduct(emptyDense, emptyDense), 0.0)
+ checkEvaluation(VectorDotProduct(emptyDense, emptySparse), 0.0)
+ checkEvaluation(VectorDotProduct(emptySparse, emptyDense), 0.0)
+ checkEvaluation(VectorDotProduct(emptySparse, emptySparse), 0.0)
+ }
+
+ test("vector dot product with infinite and NaN values") {
+ val denseOne = dense(1.0)
+ val sparseOne = sparse(1, Array(0), Array(1.0))
+
+ Seq(Double.PositiveInfinity, Double.NegativeInfinity, Double.NaN).foreach { value =>
+ val denseValue = dense(value)
+ val sparseValue = sparse(1, Array(0), Array(value))
+
+ checkEvaluation(VectorDotProduct(denseValue, denseOne), value)
+ checkEvaluation(VectorDotProduct(denseValue, sparseOne), value)
+ checkEvaluation(VectorDotProduct(sparseValue, denseOne), value)
+ checkEvaluation(VectorDotProduct(sparseValue, sparseOne), value)
+ }
+ }
+
+ test("vector dot product rejects vectors with different sizes") {
+ checkExceptionInExpression[IllegalArgumentException](
+ VectorDotProduct(dense(1.0), dense(1.0, 2.0)),
+ "vectors with non-matching sizes")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org