You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ru...@apache.org on 2025/01/17 09:54:44 UTC

(spark) branch branch-4.0 updated: [SPARK-50851][ML][CONNECT][PYTHON] Express ML params with `proto.Expression.Literal`

This is an automated email from the ASF dual-hosted git repository.

ruifengz pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.0 by this push:
     new 93e198d2fb80 [SPARK-50851][ML][CONNECT][PYTHON] Express ML params with `proto.Expression.Literal`
93e198d2fb80 is described below

commit 93e198d2fb8058d4553232d14f991085e0f4e46b
Author: Ruifeng Zheng <ru...@apache.org>
AuthorDate: Fri Jan 17 17:54:12 2025 +0800

    [SPARK-50851][ML][CONNECT][PYTHON] Express ML params with `proto.Expression.Literal`
    
    ### What changes were proposed in this pull request?
    Express ML params with `proto.Expression.Literal`:
    1, introduce `Literal.SpecializedArray` for large primitive literal arrays (e.g. the initial model coefficients which can be large)
    ```
        message SpecializedArray {
          oneof value_type {
            Bools bools = 1;
            Ints ints = 2;
            Longs longs = 3;
            Floats floats = 4;
            Doubles doubles = 5;
            Strings strings = 6;
          }
    
          message Bools {
            repeated bool values = 1;
          }
    
          message Ints {
            repeated int32 values = 1;
          }
    
          message Longs {
            repeated int64 values = 1;
          }
    
          message Floats {
            repeated float values = 1;
          }
    
          message Doubles {
            repeated double values = 1;
          }
    
          message Strings {
            repeated string values = 1;
          }
        }
    ```
    2, Replace `proto.Param ` with `proto.Expression` to be consistent with SQL side
    For `Param[Vector]` and `Param[Matrix]`, apply `proto.Expression.Literal.Struct` with the underlying schema of `VectorUDT` and `MatrixUDT`.
    
    E.g. for `Param[Vector]` with value `Vectors.sparse(4, [(1, 1.0), (3, 5.5)])`, the message is like:
    ```
    literal {
    struct {
      struct_type {
        struct {
          ... <- schema of VectorUDT
        }
      }
      elements {
        byte: 0
      }
      elements {
        integer: 4
      }
      elements {
        specialized_array {
          ints {
            values: 1
            values: 3
          }
        }
      }
      elements {
        specialized_array {
          doubles {
            values: 1
            values: 5.5
          }
        }
      }
    }
    
    ```
    
    ### Why are the changes needed?
    1, to optimize large literal arrays, for both ML and SQL (we can apply it in SQL side later)
    2, be consistent with SQL side, e.g. the parameterized SQL
    ```
      // (Optional) A map of parameter names to expressions.
      // It cannot coexist with `pos_arguments`.
      map<string, Expression.Literal> named_arguments = 4;
    
      // (Optional) A sequence of expressions for positional parameters in the SQL query text.
      // It cannot coexist with `named_arguments`.
      repeated Expression pos_arguments = 5;
    ```
    3, to minimize the protobuf change
    
    ### Does this PR introduce _any_ user-facing change?
    no, refactor-only
    
    ### How was this patch tested?
    existing tests
    
    ### Was this patch authored or co-authored using generative AI tooling?
    no
    
    Closes #49529 from zhengruifeng/ml_proto_expr.
    
    Authored-by: Ruifeng Zheng <ru...@apache.org>
    Signed-off-by: Ruifeng Zheng <ru...@apache.org>
    (cherry picked from commit 2721a50234b05db3e59b5a37225f9358978a4590)
    Signed-off-by: Ruifeng Zheng <ru...@apache.org>
---
 .../org/apache/spark/ml/linalg/MatrixUDT.scala     |  39 +--
 .../org/apache/spark/ml/linalg/VectorUDT.scala     |   7 +-
 python/pyspark/ml/connect/serialize.py             | 200 ++++++++-----
 python/pyspark/sql/connect/proto/common_pb2.py     |  14 +-
 python/pyspark/sql/connect/proto/common_pb2.pyi    | 102 +++++++
 .../pyspark/sql/connect/proto/expressions_pb2.py   | 124 ++++----
 .../pyspark/sql/connect/proto/expressions_pb2.pyi  |  85 ++++++
 python/pyspark/sql/connect/proto/ml_common_pb2.py  |  32 +--
 python/pyspark/sql/connect/proto/ml_common_pb2.pyi | 278 +-----------------
 python/pyspark/sql/connect/proto/ml_pb2.py         |  35 +--
 python/pyspark/sql/connect/proto/ml_pb2.pyi        |   5 +-
 python/pyspark/sql/connect/proto/relations_pb2.py  | 316 ++++++++++-----------
 python/pyspark/sql/connect/proto/relations_pb2.pyi |   4 +-
 .../src/main/protobuf/spark/connect/common.proto   |  24 ++
 .../main/protobuf/spark/connect/expressions.proto  |  13 +
 .../src/main/protobuf/spark/connect/ml.proto       |   3 +-
 .../main/protobuf/spark/connect/ml_common.proto    |  54 +---
 .../main/protobuf/spark/connect/relations.proto    |   2 +-
 .../apache/spark/sql/connect/ml/MLHandler.scala    |   6 +-
 .../org/apache/spark/sql/connect/ml/MLUtils.scala  | 107 ++++---
 .../apache/spark/sql/connect/ml/Serializer.scala   | 200 ++++++++-----
 .../spark/sql/connect/ml/MLBackendSuite.scala      |  15 +-
 .../org/apache/spark/sql/connect/ml/MLSuite.scala  | 112 +++-----
 23 files changed, 906 insertions(+), 871 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/linalg/MatrixUDT.scala b/mllib/src/main/scala/org/apache/spark/ml/linalg/MatrixUDT.scala
index 27a67d561e35..89ddac889f94 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/linalg/MatrixUDT.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/linalg/MatrixUDT.scala
@@ -27,23 +27,7 @@ import org.apache.spark.sql.types._
  */
 private[spark] class MatrixUDT extends UserDefinedType[Matrix] {
 
-  override def sqlType: StructType = {
-    // type: 0 = sparse, 1 = dense
-    // the dense matrix is built by numRows, numCols, values and isTransposed, all of which are
-    // set as not nullable, except values since in the future, support for binary matrices might
-    // be added for which values are not needed.
-    // the sparse matrix needs colPtrs and rowIndices, which are set as
-    // null, while building the dense matrix.
-    StructType(Array(
-      StructField("type", ByteType, nullable = false),
-      StructField("numRows", IntegerType, nullable = false),
-      StructField("numCols", IntegerType, nullable = false),
-      StructField("colPtrs", ArrayType(IntegerType, containsNull = false), nullable = true),
-      StructField("rowIndices", ArrayType(IntegerType, containsNull = false), nullable = true),
-      StructField("values", ArrayType(DoubleType, containsNull = false), nullable = true),
-      StructField("isTransposed", BooleanType, nullable = false)
-      ))
-  }
+  override def sqlType: StructType = MatrixUDT.sqlType
 
   override def serialize(obj: Matrix): InternalRow = {
     val row = new GenericInternalRow(7)
@@ -108,3 +92,24 @@ private[spark] class MatrixUDT extends UserDefinedType[Matrix] {
 
   private[spark] override def asNullable: MatrixUDT = this
 }
+
+private[spark] object MatrixUDT {
+
+  val sqlType: StructType = {
+    // type: 0 = sparse, 1 = dense
+    // the dense matrix is built by numRows, numCols, values and isTransposed, all of which are
+    // set as not nullable, except values since in the future, support for binary matrices might
+    // be added for which values are not needed.
+    // the sparse matrix needs colPtrs and rowIndices, which are set as
+    // null, while building the dense matrix.
+    StructType(Array(
+      StructField("type", ByteType, nullable = false),
+      StructField("numRows", IntegerType, nullable = false),
+      StructField("numCols", IntegerType, nullable = false),
+      StructField("colPtrs", ArrayType(IntegerType, containsNull = false), nullable = true),
+      StructField("rowIndices", ArrayType(IntegerType, containsNull = false), nullable = true),
+      StructField("values", ArrayType(DoubleType, containsNull = false), nullable = true),
+      StructField("isTransposed", BooleanType, nullable = false)
+    ))
+  }
+}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/linalg/VectorUDT.scala b/mllib/src/main/scala/org/apache/spark/ml/linalg/VectorUDT.scala
index 302a94c87811..d4648c1f0a1b 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/linalg/VectorUDT.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/linalg/VectorUDT.scala
@@ -27,7 +27,7 @@ import org.apache.spark.sql.types._
  */
 private[spark] class VectorUDT extends UserDefinedType[Vector] {
 
-  override final def sqlType: StructType = _sqlType
+  override final def sqlType: StructType = VectorUDT.sqlType
 
   override def serialize(obj: Vector): InternalRow = {
     obj match {
@@ -86,8 +86,11 @@ private[spark] class VectorUDT extends UserDefinedType[Vector] {
   override def typeName: String = "vector"
 
   private[spark] override def asNullable: VectorUDT = this
+}
+
+private[spark] object VectorUDT {
 
-  private[this] val _sqlType = {
+  val sqlType = {
     // type: 0 = sparse, 1 = dense
     // We only use "values" for dense vectors, and "size", "indices", and "values" for sparse
     // vectors. The "values" field is nullable because we might want to add binary vectors later,
diff --git a/python/pyspark/ml/connect/serialize.py b/python/pyspark/ml/connect/serialize.py
index 69e3af1f4c78..62b21460feb7 100644
--- a/python/pyspark/ml/connect/serialize.py
+++ b/python/pyspark/ml/connect/serialize.py
@@ -18,54 +18,107 @@ from typing import Any, List, TYPE_CHECKING, Mapping, Dict
 
 import pyspark.sql.connect.proto as pb2
 from pyspark.ml.linalg import (
-    Vectors,
-    Matrices,
+    VectorUDT,
+    MatrixUDT,
     DenseVector,
     SparseVector,
     DenseMatrix,
     SparseMatrix,
 )
-from pyspark.sql.connect.expressions import LiteralExpression
 
 if TYPE_CHECKING:
     from pyspark.sql.connect.client import SparkConnectClient
     from pyspark.ml.param import Params
 
 
-def serialize_param(value: Any, client: "SparkConnectClient") -> pb2.Param:
-    if isinstance(value, DenseVector):
-        return pb2.Param(vector=pb2.Vector(dense=pb2.Vector.Dense(value=value.values.tolist())))
-    elif isinstance(value, SparseVector):
-        return pb2.Param(
-            vector=pb2.Vector(
-                sparse=pb2.Vector.Sparse(
-                    size=value.size, index=value.indices.tolist(), value=value.values.tolist()
-                )
-            )
-        )
-    elif isinstance(value, DenseMatrix):
-        return pb2.Param(
-            matrix=pb2.Matrix(
-                dense=pb2.Matrix.Dense(
-                    num_rows=value.numRows, num_cols=value.numCols, value=value.values.tolist()
-                )
-            )
-        )
+def literal_null() -> pb2.Expression.Literal:
+    dt = pb2.DataType()
+    dt.null.CopyFrom(pb2.DataType.NULL())
+    return pb2.Expression.Literal(null=dt)
+
+
+def build_int_list(value: List[int]) -> pb2.Expression.Literal:
+    p = pb2.Expression.Literal()
+    p.specialized_array.ints.values.extend(value)
+    return p
+
+
+def build_float_list(value: List[float]) -> pb2.Expression.Literal:
+    p = pb2.Expression.Literal()
+    p.specialized_array.doubles.values.extend(value)
+    return p
+
+
+def serialize_param(value: Any, client: "SparkConnectClient") -> pb2.Expression.Literal:
+    from pyspark.sql.connect.types import pyspark_types_to_proto_types
+    from pyspark.sql.connect.expressions import LiteralExpression
+
+    if isinstance(value, SparseVector):
+        p = pb2.Expression.Literal()
+        p.struct.struct_type.CopyFrom(pyspark_types_to_proto_types(VectorUDT.sqlType()))
+        # type = 0
+        p.struct.elements.append(pb2.Expression.Literal(byte=0))
+        # size
+        p.struct.elements.append(pb2.Expression.Literal(integer=value.size))
+        # indices
+        p.struct.elements.append(build_int_list(value.indices.tolist()))
+        # values
+        p.struct.elements.append(build_float_list(value.values.tolist()))
+        return p
+
+    elif isinstance(value, DenseVector):
+        p = pb2.Expression.Literal()
+        p.struct.struct_type.CopyFrom(pyspark_types_to_proto_types(VectorUDT.sqlType()))
+        # type = 1
+        p.struct.elements.append(pb2.Expression.Literal(byte=1))
+        # size = null
+        p.struct.elements.append(literal_null())
+        # indices = null
+        p.struct.elements.append(literal_null())
+        # values
+        p.struct.elements.append(build_float_list(value.values.tolist()))
+        return p
+
     elif isinstance(value, SparseMatrix):
-        return pb2.Param(
-            matrix=pb2.Matrix(
-                sparse=pb2.Matrix.Sparse(
-                    num_rows=value.numRows,
-                    num_cols=value.numCols,
-                    colptr=value.colPtrs.tolist(),
-                    row_index=value.rowIndices.tolist(),
-                    value=value.values.tolist(),
-                )
-            )
-        )
+        p = pb2.Expression.Literal()
+        p.struct.struct_type.CopyFrom(pyspark_types_to_proto_types(MatrixUDT.sqlType()))
+        # type = 0
+        p.struct.elements.append(pb2.Expression.Literal(byte=0))
+        # numRows
+        p.struct.elements.append(pb2.Expression.Literal(integer=value.numRows))
+        # numCols
+        p.struct.elements.append(pb2.Expression.Literal(integer=value.numCols))
+        # colPtrs
+        p.struct.elements.append(build_int_list(value.colPtrs.tolist()))
+        # rowIndices
+        p.struct.elements.append(build_int_list(value.rowIndices.tolist()))
+        # values
+        p.struct.elements.append(build_float_list(value.values.tolist()))
+        # isTransposed
+        p.struct.elements.append(pb2.Expression.Literal(boolean=value.isTransposed))
+        return p
+
+    elif isinstance(value, DenseMatrix):
+        p = pb2.Expression.Literal()
+        p.struct.struct_type.CopyFrom(pyspark_types_to_proto_types(MatrixUDT.sqlType()))
+        # type = 1
+        p.struct.elements.append(pb2.Expression.Literal(byte=1))
+        # numRows
+        p.struct.elements.append(pb2.Expression.Literal(integer=value.numRows))
+        # numCols
+        p.struct.elements.append(pb2.Expression.Literal(integer=value.numCols))
+        # colPtrs = null
+        p.struct.elements.append(literal_null())
+        # rowIndices = null
+        p.struct.elements.append(literal_null())
+        # values
+        p.struct.elements.append(build_float_list(value.values.tolist()))
+        # isTransposed
+        p.struct.elements.append(pb2.Expression.Literal(boolean=value.isTransposed))
+        return p
+
     else:
-        literal = LiteralExpression._from_value(value).to_plan(client).literal
-        return pb2.Param(literal=literal)
+        return LiteralExpression._from_value(value).to_plan(client).literal
 
 
 def serialize(client: "SparkConnectClient", *args: Any) -> List[Any]:
@@ -80,38 +133,51 @@ def serialize(client: "SparkConnectClient", *args: Any) -> List[Any]:
     return result
 
 
-def deserialize_param(param: pb2.Param) -> Any:
-    if param.HasField("literal"):
-        return LiteralExpression._to_value(param.literal)
-    if param.HasField("vector"):
-        vector = param.vector
-        if vector.HasField("dense"):
-            return Vectors.dense(vector.dense.value)
-        elif vector.HasField("sparse"):
-            return Vectors.sparse(vector.sparse.size, vector.sparse.index, vector.sparse.value)
-        else:
-            raise ValueError("Unsupported vector type")
-    if param.HasField("matrix"):
-        matrix = param.matrix
-        if matrix.HasField("dense"):
-            return DenseMatrix(
-                matrix.dense.num_rows,
-                matrix.dense.num_cols,
-                matrix.dense.value,
-                matrix.dense.is_transposed,
-            )
-        elif matrix.HasField("sparse"):
-            return Matrices.sparse(
-                matrix.sparse.num_rows,
-                matrix.sparse.num_cols,
-                matrix.sparse.colptr,
-                matrix.sparse.row_index,
-                matrix.sparse.value,
-            )
+def deserialize_param(literal: pb2.Expression.Literal) -> Any:
+    from pyspark.sql.connect.types import proto_schema_to_pyspark_data_type
+    from pyspark.sql.connect.expressions import LiteralExpression
+
+    if literal.HasField("struct"):
+        s = literal.struct
+        schema = proto_schema_to_pyspark_data_type(s.struct_type)
+
+        if schema == VectorUDT.sqlType():
+            assert len(s.elements) == 4
+            tpe = s.elements[0].byte
+            if tpe == 0:
+                size = s.elements[1].integer
+                indices = s.elements[2].specialized_array.ints.values
+                values = s.elements[3].specialized_array.doubles.values
+                return SparseVector(size, indices, values)
+            elif tpe == 1:
+                values = s.elements[3].specialized_array.doubles.values
+                return DenseVector(values)
+            else:
+                raise ValueError(f"Unknown Vector type {tpe}")
+
+        elif schema == MatrixUDT.sqlType():
+            assert len(s.elements) == 7
+            tpe = s.elements[0].byte
+            if tpe == 0:
+                numRows = s.elements[1].integer
+                numCols = s.elements[2].integer
+                colPtrs = s.elements[3].specialized_array.ints.values
+                rowIndices = s.elements[4].specialized_array.ints.values
+                values = s.elements[5].specialized_array.doubles.values
+                isTransposed = s.elements[6].boolean
+                return SparseMatrix(numRows, numCols, colPtrs, rowIndices, values, isTransposed)
+            elif tpe == 1:
+                numRows = s.elements[1].integer
+                numCols = s.elements[2].integer
+                values = s.elements[5].specialized_array.doubles.values
+                isTransposed = s.elements[6].boolean
+                return DenseMatrix(numRows, numCols, values, isTransposed)
+            else:
+                raise ValueError(f"Unknown Matrix type {tpe}")
         else:
-            raise ValueError("Unsupported matrix type")
-
-    raise ValueError("Unsupported param type")
+            raise ValueError(f"Unsupported parameter struct {schema}")
+    else:
+        return LiteralExpression._to_value(literal)
 
 
 def deserialize(ml_command_result_properties: Dict[str, Any]) -> Any:
@@ -126,7 +192,7 @@ def deserialize(ml_command_result_properties: Dict[str, Any]) -> Any:
 
 
 def serialize_ml_params(instance: "Params", client: "SparkConnectClient") -> pb2.MlParams:
-    params: Mapping[str, pb2.Param] = {
+    params: Mapping[str, pb2.Expression.Literal] = {
         k.name: serialize_param(v, client) for k, v in instance._paramMap.items()
     }
     return pb2.MlParams(params=params)
diff --git a/python/pyspark/sql/connect/proto/common_pb2.py b/python/pyspark/sql/connect/proto/common_pb2.py
index 85f475b31a9e..2f57ce2f73b4 100644
--- a/python/pyspark/sql/connect/proto/common_pb2.py
+++ b/python/pyspark/sql/connect/proto/common_pb2.py
@@ -35,7 +35,7 @@ _sym_db = _symbol_database.Default()
 
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\n\x1aspark/connect/common.proto\x12\rspark.connect"\xb0\x01\n\x0cStorageLevel\x12\x19\n\x08use_disk\x18\x01 \x01(\x08R\x07useDisk\x12\x1d\n\nuse_memory\x18\x02 \x01(\x08R\tuseMemory\x12 \n\x0cuse_off_heap\x18\x03 \x01(\x08R\nuseOffHeap\x12"\n\x0c\x64\x65serialized\x18\x04 \x01(\x08R\x0c\x64\x65serialized\x12 \n\x0breplication\x18\x05 \x01(\x05R\x0breplication"G\n\x13ResourceInformation\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1c\n\taddresses\x18\x02 \x03(\tR\taddresses"\xc3 [...]
+    b'\n\x1aspark/connect/common.proto\x12\rspark.connect"\xb0\x01\n\x0cStorageLevel\x12\x19\n\x08use_disk\x18\x01 \x01(\x08R\x07useDisk\x12\x1d\n\nuse_memory\x18\x02 \x01(\x08R\tuseMemory\x12 \n\x0cuse_off_heap\x18\x03 \x01(\x08R\nuseOffHeap\x12"\n\x0c\x64\x65serialized\x18\x04 \x01(\x08R\x0c\x64\x65serialized\x12 \n\x0breplication\x18\x05 \x01(\x05R\x0breplication"G\n\x13ResourceInformation\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1c\n\taddresses\x18\x02 \x03(\tR\taddresses"\xc3 [...]
 )
 
 _globals = globals()
@@ -70,4 +70,16 @@ if not _descriptor._USE_C_DESCRIPTORS:
     _globals["_ORIGIN"]._serialized_end = 1091
     _globals["_PYTHONORIGIN"]._serialized_start = 1093
     _globals["_PYTHONORIGIN"]._serialized_end = 1164
+    _globals["_BOOLS"]._serialized_start = 1166
+    _globals["_BOOLS"]._serialized_end = 1197
+    _globals["_INTS"]._serialized_start = 1199
+    _globals["_INTS"]._serialized_end = 1229
+    _globals["_LONGS"]._serialized_start = 1231
+    _globals["_LONGS"]._serialized_end = 1262
+    _globals["_FLOATS"]._serialized_start = 1264
+    _globals["_FLOATS"]._serialized_end = 1296
+    _globals["_DOUBLES"]._serialized_start = 1298
+    _globals["_DOUBLES"]._serialized_end = 1331
+    _globals["_STRINGS"]._serialized_start = 1333
+    _globals["_STRINGS"]._serialized_end = 1366
 # @@protoc_insertion_point(module_scope)
diff --git a/python/pyspark/sql/connect/proto/common_pb2.pyi b/python/pyspark/sql/connect/proto/common_pb2.pyi
index eda172e26cf4..fc5e5adef432 100644
--- a/python/pyspark/sql/connect/proto/common_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/common_pb2.pyi
@@ -347,3 +347,105 @@ class PythonOrigin(google.protobuf.message.Message):
     ) -> None: ...
 
 global___PythonOrigin = PythonOrigin
+
+class Bools(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    VALUES_FIELD_NUMBER: builtins.int
+    @property
+    def values(
+        self,
+    ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.bool]: ...
+    def __init__(
+        self,
+        *,
+        values: collections.abc.Iterable[builtins.bool] | None = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["values", b"values"]) -> None: ...
+
+global___Bools = Bools
+
+class Ints(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    VALUES_FIELD_NUMBER: builtins.int
+    @property
+    def values(
+        self,
+    ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: ...
+    def __init__(
+        self,
+        *,
+        values: collections.abc.Iterable[builtins.int] | None = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["values", b"values"]) -> None: ...
+
+global___Ints = Ints
+
+class Longs(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    VALUES_FIELD_NUMBER: builtins.int
+    @property
+    def values(
+        self,
+    ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: ...
+    def __init__(
+        self,
+        *,
+        values: collections.abc.Iterable[builtins.int] | None = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["values", b"values"]) -> None: ...
+
+global___Longs = Longs
+
+class Floats(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    VALUES_FIELD_NUMBER: builtins.int
+    @property
+    def values(
+        self,
+    ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.float]: ...
+    def __init__(
+        self,
+        *,
+        values: collections.abc.Iterable[builtins.float] | None = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["values", b"values"]) -> None: ...
+
+global___Floats = Floats
+
+class Doubles(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    VALUES_FIELD_NUMBER: builtins.int
+    @property
+    def values(
+        self,
+    ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.float]: ...
+    def __init__(
+        self,
+        *,
+        values: collections.abc.Iterable[builtins.float] | None = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["values", b"values"]) -> None: ...
+
+global___Doubles = Doubles
+
+class Strings(google.protobuf.message.Message):
+    DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+    VALUES_FIELD_NUMBER: builtins.int
+    @property
+    def values(
+        self,
+    ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: ...
+    def __init__(
+        self,
+        *,
+        values: collections.abc.Iterable[builtins.str] | None = ...,
+    ) -> None: ...
+    def ClearField(self, field_name: typing_extensions.Literal["values", b"values"]) -> None: ...
+
+global___Strings = Strings
diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.py b/python/pyspark/sql/connect/proto/expressions_pb2.py
index 7edcbcac15c7..021bf4db2a12 100644
--- a/python/pyspark/sql/connect/proto/expressions_pb2.py
+++ b/python/pyspark/sql/connect/proto/expressions_pb2.py
@@ -40,7 +40,7 @@ from pyspark.sql.connect.proto import common_pb2 as spark_dot_connect_dot_common
 
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\n\x1fspark/connect/expressions.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x19spark/connect/types.proto\x1a\x1aspark/connect/common.proto"\x97\x32\n\nExpression\x12\x37\n\x06\x63ommon\x18\x12 \x01(\x0b\x32\x1f.spark.connect.ExpressionCommonR\x06\x63ommon\x12=\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralH\x00R\x07literal\x12\x62\n\x14unresolved_attribute\x18\x02 \x01(\x0b\x32-.spark.connect.Expression.UnresolvedAttributeH\x00R\x13unresolved [...]
+    b'\n\x1fspark/connect/expressions.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x19spark/connect/types.proto\x1a\x1aspark/connect/common.proto"\xbd\x35\n\nExpression\x12\x37\n\x06\x63ommon\x18\x12 \x01(\x0b\x32\x1f.spark.connect.ExpressionCommonR\x06\x63ommon\x12=\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralH\x00R\x07literal\x12\x62\n\x14unresolved_attribute\x18\x02 \x01(\x0b\x32-.spark.connect.Expression.UnresolvedAttributeH\x00R\x13unresolved [...]
 )
 
 _globals = globals()
@@ -54,7 +54,7 @@ if not _descriptor._USE_C_DESCRIPTORS:
         "DESCRIPTOR"
     ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
     _globals["_EXPRESSION"]._serialized_start = 133
-    _globals["_EXPRESSION"]._serialized_end = 6556
+    _globals["_EXPRESSION"]._serialized_end = 6978
     _globals["_EXPRESSION_WINDOW"]._serialized_start = 2060
     _globals["_EXPRESSION_WINDOW"]._serialized_end = 2843
     _globals["_EXPRESSION_WINDOW_WINDOWFRAME"]._serialized_start = 2350
@@ -74,63 +74,65 @@ if not _descriptor._USE_C_DESCRIPTORS:
     _globals["_EXPRESSION_CAST_EVALMODE"]._serialized_start = 3475
     _globals["_EXPRESSION_CAST_EVALMODE"]._serialized_end = 3573
     _globals["_EXPRESSION_LITERAL"]._serialized_start = 3592
-    _globals["_EXPRESSION_LITERAL"]._serialized_end = 5155
-    _globals["_EXPRESSION_LITERAL_DECIMAL"]._serialized_start = 4427
-    _globals["_EXPRESSION_LITERAL_DECIMAL"]._serialized_end = 4544
-    _globals["_EXPRESSION_LITERAL_CALENDARINTERVAL"]._serialized_start = 4546
-    _globals["_EXPRESSION_LITERAL_CALENDARINTERVAL"]._serialized_end = 4644
-    _globals["_EXPRESSION_LITERAL_ARRAY"]._serialized_start = 4647
-    _globals["_EXPRESSION_LITERAL_ARRAY"]._serialized_end = 4777
-    _globals["_EXPRESSION_LITERAL_MAP"]._serialized_start = 4780
-    _globals["_EXPRESSION_LITERAL_MAP"]._serialized_end = 5007
-    _globals["_EXPRESSION_LITERAL_STRUCT"]._serialized_start = 5010
-    _globals["_EXPRESSION_LITERAL_STRUCT"]._serialized_end = 5139
-    _globals["_EXPRESSION_UNRESOLVEDATTRIBUTE"]._serialized_start = 5158
-    _globals["_EXPRESSION_UNRESOLVEDATTRIBUTE"]._serialized_end = 5344
-    _globals["_EXPRESSION_UNRESOLVEDFUNCTION"]._serialized_start = 5347
-    _globals["_EXPRESSION_UNRESOLVEDFUNCTION"]._serialized_end = 5605
-    _globals["_EXPRESSION_EXPRESSIONSTRING"]._serialized_start = 5607
-    _globals["_EXPRESSION_EXPRESSIONSTRING"]._serialized_end = 5657
-    _globals["_EXPRESSION_UNRESOLVEDSTAR"]._serialized_start = 5659
-    _globals["_EXPRESSION_UNRESOLVEDSTAR"]._serialized_end = 5783
-    _globals["_EXPRESSION_UNRESOLVEDREGEX"]._serialized_start = 5785
-    _globals["_EXPRESSION_UNRESOLVEDREGEX"]._serialized_end = 5871
-    _globals["_EXPRESSION_UNRESOLVEDEXTRACTVALUE"]._serialized_start = 5874
-    _globals["_EXPRESSION_UNRESOLVEDEXTRACTVALUE"]._serialized_end = 6006
-    _globals["_EXPRESSION_UPDATEFIELDS"]._serialized_start = 6009
-    _globals["_EXPRESSION_UPDATEFIELDS"]._serialized_end = 6196
-    _globals["_EXPRESSION_ALIAS"]._serialized_start = 6198
-    _globals["_EXPRESSION_ALIAS"]._serialized_end = 6318
-    _globals["_EXPRESSION_LAMBDAFUNCTION"]._serialized_start = 6321
-    _globals["_EXPRESSION_LAMBDAFUNCTION"]._serialized_end = 6479
-    _globals["_EXPRESSION_UNRESOLVEDNAMEDLAMBDAVARIABLE"]._serialized_start = 6481
-    _globals["_EXPRESSION_UNRESOLVEDNAMEDLAMBDAVARIABLE"]._serialized_end = 6543
-    _globals["_EXPRESSIONCOMMON"]._serialized_start = 6558
-    _globals["_EXPRESSIONCOMMON"]._serialized_end = 6623
-    _globals["_COMMONINLINEUSERDEFINEDFUNCTION"]._serialized_start = 6626
-    _globals["_COMMONINLINEUSERDEFINEDFUNCTION"]._serialized_end = 6990
-    _globals["_PYTHONUDF"]._serialized_start = 6993
-    _globals["_PYTHONUDF"]._serialized_end = 7197
-    _globals["_SCALARSCALAUDF"]._serialized_start = 7200
-    _globals["_SCALARSCALAUDF"]._serialized_end = 7414
-    _globals["_JAVAUDF"]._serialized_start = 7417
-    _globals["_JAVAUDF"]._serialized_end = 7566
-    _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_start = 7568
-    _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_end = 7667
-    _globals["_CALLFUNCTION"]._serialized_start = 7669
-    _globals["_CALLFUNCTION"]._serialized_end = 7777
-    _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_start = 7779
-    _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_end = 7871
-    _globals["_MERGEACTION"]._serialized_start = 7874
-    _globals["_MERGEACTION"]._serialized_end = 8386
-    _globals["_MERGEACTION_ASSIGNMENT"]._serialized_start = 8096
-    _globals["_MERGEACTION_ASSIGNMENT"]._serialized_end = 8202
-    _globals["_MERGEACTION_ACTIONTYPE"]._serialized_start = 8205
-    _globals["_MERGEACTION_ACTIONTYPE"]._serialized_end = 8372
-    _globals["_LAZYEXPRESSION"]._serialized_start = 8388
-    _globals["_LAZYEXPRESSION"]._serialized_end = 8453
-    _globals["_SUBQUERYEXPRESSION"]._serialized_start = 8456
-    _globals["_SUBQUERYEXPRESSION"]._serialized_end = 8681
-    _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_start = 8588
-    _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_end = 8681
+    _globals["_EXPRESSION_LITERAL"]._serialized_end = 5577
+    _globals["_EXPRESSION_LITERAL_DECIMAL"]._serialized_start = 4526
+    _globals["_EXPRESSION_LITERAL_DECIMAL"]._serialized_end = 4643
+    _globals["_EXPRESSION_LITERAL_CALENDARINTERVAL"]._serialized_start = 4645
+    _globals["_EXPRESSION_LITERAL_CALENDARINTERVAL"]._serialized_end = 4743
+    _globals["_EXPRESSION_LITERAL_ARRAY"]._serialized_start = 4746
+    _globals["_EXPRESSION_LITERAL_ARRAY"]._serialized_end = 4876
+    _globals["_EXPRESSION_LITERAL_MAP"]._serialized_start = 4879
+    _globals["_EXPRESSION_LITERAL_MAP"]._serialized_end = 5106
+    _globals["_EXPRESSION_LITERAL_STRUCT"]._serialized_start = 5109
+    _globals["_EXPRESSION_LITERAL_STRUCT"]._serialized_end = 5238
+    _globals["_EXPRESSION_LITERAL_SPECIALIZEDARRAY"]._serialized_start = 5241
+    _globals["_EXPRESSION_LITERAL_SPECIALIZEDARRAY"]._serialized_end = 5561
+    _globals["_EXPRESSION_UNRESOLVEDATTRIBUTE"]._serialized_start = 5580
+    _globals["_EXPRESSION_UNRESOLVEDATTRIBUTE"]._serialized_end = 5766
+    _globals["_EXPRESSION_UNRESOLVEDFUNCTION"]._serialized_start = 5769
+    _globals["_EXPRESSION_UNRESOLVEDFUNCTION"]._serialized_end = 6027
+    _globals["_EXPRESSION_EXPRESSIONSTRING"]._serialized_start = 6029
+    _globals["_EXPRESSION_EXPRESSIONSTRING"]._serialized_end = 6079
+    _globals["_EXPRESSION_UNRESOLVEDSTAR"]._serialized_start = 6081
+    _globals["_EXPRESSION_UNRESOLVEDSTAR"]._serialized_end = 6205
+    _globals["_EXPRESSION_UNRESOLVEDREGEX"]._serialized_start = 6207
+    _globals["_EXPRESSION_UNRESOLVEDREGEX"]._serialized_end = 6293
+    _globals["_EXPRESSION_UNRESOLVEDEXTRACTVALUE"]._serialized_start = 6296
+    _globals["_EXPRESSION_UNRESOLVEDEXTRACTVALUE"]._serialized_end = 6428
+    _globals["_EXPRESSION_UPDATEFIELDS"]._serialized_start = 6431
+    _globals["_EXPRESSION_UPDATEFIELDS"]._serialized_end = 6618
+    _globals["_EXPRESSION_ALIAS"]._serialized_start = 6620
+    _globals["_EXPRESSION_ALIAS"]._serialized_end = 6740
+    _globals["_EXPRESSION_LAMBDAFUNCTION"]._serialized_start = 6743
+    _globals["_EXPRESSION_LAMBDAFUNCTION"]._serialized_end = 6901
+    _globals["_EXPRESSION_UNRESOLVEDNAMEDLAMBDAVARIABLE"]._serialized_start = 6903
+    _globals["_EXPRESSION_UNRESOLVEDNAMEDLAMBDAVARIABLE"]._serialized_end = 6965
+    _globals["_EXPRESSIONCOMMON"]._serialized_start = 6980
+    _globals["_EXPRESSIONCOMMON"]._serialized_end = 7045
+    _globals["_COMMONINLINEUSERDEFINEDFUNCTION"]._serialized_start = 7048
+    _globals["_COMMONINLINEUSERDEFINEDFUNCTION"]._serialized_end = 7412
+    _globals["_PYTHONUDF"]._serialized_start = 7415
+    _globals["_PYTHONUDF"]._serialized_end = 7619
+    _globals["_SCALARSCALAUDF"]._serialized_start = 7622
+    _globals["_SCALARSCALAUDF"]._serialized_end = 7836
+    _globals["_JAVAUDF"]._serialized_start = 7839
+    _globals["_JAVAUDF"]._serialized_end = 7988
+    _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_start = 7990
+    _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_end = 8089
+    _globals["_CALLFUNCTION"]._serialized_start = 8091
+    _globals["_CALLFUNCTION"]._serialized_end = 8199
+    _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_start = 8201
+    _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_end = 8293
+    _globals["_MERGEACTION"]._serialized_start = 8296
+    _globals["_MERGEACTION"]._serialized_end = 8808
+    _globals["_MERGEACTION_ASSIGNMENT"]._serialized_start = 8518
+    _globals["_MERGEACTION_ASSIGNMENT"]._serialized_end = 8624
+    _globals["_MERGEACTION_ACTIONTYPE"]._serialized_start = 8627
+    _globals["_MERGEACTION_ACTIONTYPE"]._serialized_end = 8794
+    _globals["_LAZYEXPRESSION"]._serialized_start = 8810
+    _globals["_LAZYEXPRESSION"]._serialized_end = 8875
+    _globals["_SUBQUERYEXPRESSION"]._serialized_start = 8878
+    _globals["_SUBQUERYEXPRESSION"]._serialized_end = 9103
+    _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_start = 9010
+    _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_end = 9103
 # @@protoc_insertion_point(module_scope)
diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.pyi b/python/pyspark/sql/connect/proto/expressions_pb2.pyi
index 1a8c60f67305..b5a844b8b43b 100644
--- a/python/pyspark/sql/connect/proto/expressions_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/expressions_pb2.pyi
@@ -578,6 +578,82 @@ class Expression(google.protobuf.message.Message):
                 ],
             ) -> None: ...
 
+        class SpecializedArray(google.protobuf.message.Message):
+            DESCRIPTOR: google.protobuf.descriptor.Descriptor
+
+            BOOLS_FIELD_NUMBER: builtins.int
+            INTS_FIELD_NUMBER: builtins.int
+            LONGS_FIELD_NUMBER: builtins.int
+            FLOATS_FIELD_NUMBER: builtins.int
+            DOUBLES_FIELD_NUMBER: builtins.int
+            STRINGS_FIELD_NUMBER: builtins.int
+            @property
+            def bools(self) -> pyspark.sql.connect.proto.common_pb2.Bools: ...
+            @property
+            def ints(self) -> pyspark.sql.connect.proto.common_pb2.Ints: ...
+            @property
+            def longs(self) -> pyspark.sql.connect.proto.common_pb2.Longs: ...
+            @property
+            def floats(self) -> pyspark.sql.connect.proto.common_pb2.Floats: ...
+            @property
+            def doubles(self) -> pyspark.sql.connect.proto.common_pb2.Doubles: ...
+            @property
+            def strings(self) -> pyspark.sql.connect.proto.common_pb2.Strings: ...
+            def __init__(
+                self,
+                *,
+                bools: pyspark.sql.connect.proto.common_pb2.Bools | None = ...,
+                ints: pyspark.sql.connect.proto.common_pb2.Ints | None = ...,
+                longs: pyspark.sql.connect.proto.common_pb2.Longs | None = ...,
+                floats: pyspark.sql.connect.proto.common_pb2.Floats | None = ...,
+                doubles: pyspark.sql.connect.proto.common_pb2.Doubles | None = ...,
+                strings: pyspark.sql.connect.proto.common_pb2.Strings | None = ...,
+            ) -> None: ...
+            def HasField(
+                self,
+                field_name: typing_extensions.Literal[
+                    "bools",
+                    b"bools",
+                    "doubles",
+                    b"doubles",
+                    "floats",
+                    b"floats",
+                    "ints",
+                    b"ints",
+                    "longs",
+                    b"longs",
+                    "strings",
+                    b"strings",
+                    "value_type",
+                    b"value_type",
+                ],
+            ) -> builtins.bool: ...
+            def ClearField(
+                self,
+                field_name: typing_extensions.Literal[
+                    "bools",
+                    b"bools",
+                    "doubles",
+                    b"doubles",
+                    "floats",
+                    b"floats",
+                    "ints",
+                    b"ints",
+                    "longs",
+                    b"longs",
+                    "strings",
+                    b"strings",
+                    "value_type",
+                    b"value_type",
+                ],
+            ) -> None: ...
+            def WhichOneof(
+                self, oneof_group: typing_extensions.Literal["value_type", b"value_type"]
+            ) -> (
+                typing_extensions.Literal["bools", "ints", "longs", "floats", "doubles", "strings"]
+                | None
+            ): ...
+
         NULL_FIELD_NUMBER: builtins.int
         BINARY_FIELD_NUMBER: builtins.int
         BOOLEAN_FIELD_NUMBER: builtins.int
@@ -598,6 +674,7 @@ class Expression(google.protobuf.message.Message):
         ARRAY_FIELD_NUMBER: builtins.int
         MAP_FIELD_NUMBER: builtins.int
         STRUCT_FIELD_NUMBER: builtins.int
+        SPECIALIZED_ARRAY_FIELD_NUMBER: builtins.int
         @property
         def null(self) -> pyspark.sql.connect.proto.types_pb2.DataType: ...
         binary: builtins.bytes
@@ -627,6 +704,8 @@ class Expression(google.protobuf.message.Message):
         def map(self) -> global___Expression.Literal.Map: ...
         @property
         def struct(self) -> global___Expression.Literal.Struct: ...
+        @property
+        def specialized_array(self) -> global___Expression.Literal.SpecializedArray: ...
         def __init__(
             self,
             *,
@@ -650,6 +729,7 @@ class Expression(google.protobuf.message.Message):
             array: global___Expression.Literal.Array | None = ...,
             map: global___Expression.Literal.Map | None = ...,
             struct: global___Expression.Literal.Struct | None = ...,
+            specialized_array: global___Expression.Literal.SpecializedArray | None = ...,
         ) -> None: ...
         def HasField(
             self,
@@ -686,6 +766,8 @@ class Expression(google.protobuf.message.Message):
                 b"null",
                 "short",
                 b"short",
+                "specialized_array",
+                b"specialized_array",
                 "string",
                 b"string",
                 "struct",
@@ -733,6 +815,8 @@ class Expression(google.protobuf.message.Message):
                 b"null",
                 "short",
                 b"short",
+                "specialized_array",
+                b"specialized_array",
                 "string",
                 b"string",
                 "struct",
@@ -769,6 +853,7 @@ class Expression(google.protobuf.message.Message):
                 "array",
                 "map",
                 "struct",
+                "specialized_array",
             ]
             | None
         ): ...
diff --git a/python/pyspark/sql/connect/proto/ml_common_pb2.py b/python/pyspark/sql/connect/proto/ml_common_pb2.py
index 70e0e9165289..43d6a512f48f 100644
--- a/python/pyspark/sql/connect/proto/ml_common_pb2.py
+++ b/python/pyspark/sql/connect/proto/ml_common_pb2.py
@@ -38,7 +38,7 @@ from pyspark.sql.connect.proto import expressions_pb2 as spark_dot_connect_dot_e
 
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\n\x1dspark/connect/ml_common.proto\x12\rspark.connect\x1a\x1fspark/connect/expressions.proto"\x98\x01\n\x08MlParams\x12;\n\x06params\x18\x01 \x03(\x0b\x32#.spark.connect.MlParams.ParamsEntryR\x06params\x1aO\n\x0bParamsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x14.spark.connect.ParamR\x05value:\x02\x38\x01"\xb6\x01\n\x05Param\x12=\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralH\x00R\x07literal\x12/\n\x06vector\x18\x02 [...]
+    b'\n\x1dspark/connect/ml_common.proto\x12\rspark.connect\x1a\x1fspark/connect/expressions.proto"\xa5\x01\n\x08MlParams\x12;\n\x06params\x18\x01 \x03(\x0b\x32#.spark.connect.MlParams.ParamsEntryR\x06params\x1a\\\n\x0bParamsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12\x37\n\x05value\x18\x02 \x01(\x0b\x32!.spark.connect.Expression.LiteralR\x05value:\x02\x38\x01"\xc9\x01\n\nMlOperator\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x10\n\x03uid\x18\x02 \x01(\tR\x03uid\x12:\n\x04type [...]
 )
 
 _globals = globals()
@@ -54,27 +54,13 @@ if not _descriptor._USE_C_DESCRIPTORS:
     _globals["_MLPARAMS_PARAMSENTRY"]._loaded_options = None
     _globals["_MLPARAMS_PARAMSENTRY"]._serialized_options = b"8\001"
     _globals["_MLPARAMS"]._serialized_start = 82
-    _globals["_MLPARAMS"]._serialized_end = 234
+    _globals["_MLPARAMS"]._serialized_end = 247
     _globals["_MLPARAMS_PARAMSENTRY"]._serialized_start = 155
-    _globals["_MLPARAMS_PARAMSENTRY"]._serialized_end = 234
-    _globals["_PARAM"]._serialized_start = 237
-    _globals["_PARAM"]._serialized_end = 419
-    _globals["_MLOPERATOR"]._serialized_start = 422
-    _globals["_MLOPERATOR"]._serialized_end = 623
-    _globals["_MLOPERATOR_OPERATORTYPE"]._serialized_start = 534
-    _globals["_MLOPERATOR_OPERATORTYPE"]._serialized_end = 623
-    _globals["_OBJECTREF"]._serialized_start = 625
-    _globals["_OBJECTREF"]._serialized_end = 652
-    _globals["_VECTOR"]._serialized_start = 655
-    _globals["_VECTOR"]._serialized_end = 892
-    _globals["_VECTOR_DENSE"]._serialized_start = 774
-    _globals["_VECTOR_DENSE"]._serialized_end = 803
-    _globals["_VECTOR_SPARSE"]._serialized_start = 805
-    _globals["_VECTOR_SPARSE"]._serialized_end = 877
-    _globals["_MATRIX"]._serialized_start = 895
-    _globals["_MATRIX"]._serialized_end = 1326
-    _globals["_MATRIX_DENSE"]._serialized_start = 1014
-    _globals["_MATRIX_DENSE"]._serialized_end = 1134
-    _globals["_MATRIX_SPARSE"]._serialized_start = 1137
-    _globals["_MATRIX_SPARSE"]._serialized_end = 1311
+    _globals["_MLPARAMS_PARAMSENTRY"]._serialized_end = 247
+    _globals["_MLOPERATOR"]._serialized_start = 250
+    _globals["_MLOPERATOR"]._serialized_end = 451
+    _globals["_MLOPERATOR_OPERATORTYPE"]._serialized_start = 362
+    _globals["_MLOPERATOR_OPERATORTYPE"]._serialized_end = 451
+    _globals["_OBJECTREF"]._serialized_start = 453
+    _globals["_OBJECTREF"]._serialized_end = 480
 # @@protoc_insertion_point(module_scope)
diff --git a/python/pyspark/sql/connect/proto/ml_common_pb2.pyi b/python/pyspark/sql/connect/proto/ml_common_pb2.pyi
index 64029b6679f1..f4688e94c3d5 100644
--- a/python/pyspark/sql/connect/proto/ml_common_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/ml_common_pb2.pyi
@@ -62,12 +62,12 @@ class MlParams(google.protobuf.message.Message):
         VALUE_FIELD_NUMBER: builtins.int
         key: builtins.str
         @property
-        def value(self) -> global___Param: ...
+        def value(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression.Literal: ...
         def __init__(
             self,
             *,
             key: builtins.str = ...,
-            value: global___Param | None = ...,
+            value: pyspark.sql.connect.proto.expressions_pb2.Expression.Literal | None = ...,
         ) -> None: ...
         def HasField(
             self, field_name: typing_extensions.Literal["value", b"value"]
@@ -80,72 +80,22 @@ class MlParams(google.protobuf.message.Message):
     @property
     def params(
         self,
-    ) -> google.protobuf.internal.containers.MessageMap[builtins.str, global___Param]:
+    ) -> google.protobuf.internal.containers.MessageMap[
+        builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+    ]:
         """User-supplied params"""
     def __init__(
         self,
         *,
-        params: collections.abc.Mapping[builtins.str, global___Param] | None = ...,
+        params: collections.abc.Mapping[
+            builtins.str, pyspark.sql.connect.proto.expressions_pb2.Expression.Literal
+        ]
+        | None = ...,
     ) -> None: ...
     def ClearField(self, field_name: typing_extensions.Literal["params", b"params"]) -> None: ...
 
 global___MlParams = MlParams
 
-class Param(google.protobuf.message.Message):
-    """Represents the parameter type of the ML instance, or the returned value
-    of the attribute
-    """
-
-    DESCRIPTOR: google.protobuf.descriptor.Descriptor
-
-    LITERAL_FIELD_NUMBER: builtins.int
-    VECTOR_FIELD_NUMBER: builtins.int
-    MATRIX_FIELD_NUMBER: builtins.int
-    @property
-    def literal(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression.Literal: ...
-    @property
-    def vector(self) -> global___Vector: ...
-    @property
-    def matrix(self) -> global___Matrix: ...
-    def __init__(
-        self,
-        *,
-        literal: pyspark.sql.connect.proto.expressions_pb2.Expression.Literal | None = ...,
-        vector: global___Vector | None = ...,
-        matrix: global___Matrix | None = ...,
-    ) -> None: ...
-    def HasField(
-        self,
-        field_name: typing_extensions.Literal[
-            "literal",
-            b"literal",
-            "matrix",
-            b"matrix",
-            "param_type",
-            b"param_type",
-            "vector",
-            b"vector",
-        ],
-    ) -> builtins.bool: ...
-    def ClearField(
-        self,
-        field_name: typing_extensions.Literal[
-            "literal",
-            b"literal",
-            "matrix",
-            b"matrix",
-            "param_type",
-            b"param_type",
-            "vector",
-            b"vector",
-        ],
-    ) -> None: ...
-    def WhichOneof(
-        self, oneof_group: typing_extensions.Literal["param_type", b"param_type"]
-    ) -> typing_extensions.Literal["literal", "vector", "matrix"] | None: ...
-
-global___Param = Param
-
 class MlOperator(google.protobuf.message.Message):
     """MLOperator represents the ML operators like (Estimator, Transformer or Evaluator)"""
 
@@ -215,213 +165,3 @@ class ObjectRef(google.protobuf.message.Message):
     def ClearField(self, field_name: typing_extensions.Literal["id", b"id"]) -> None: ...
 
 global___ObjectRef = ObjectRef
-
-class Vector(google.protobuf.message.Message):
-    """See pyspark.ml.linalg.Vector"""
-
-    DESCRIPTOR: google.protobuf.descriptor.Descriptor
-
-    class Dense(google.protobuf.message.Message):
-        """See pyspark.ml.linalg.DenseVector"""
-
-        DESCRIPTOR: google.protobuf.descriptor.Descriptor
-
-        VALUE_FIELD_NUMBER: builtins.int
-        @property
-        def value(
-            self,
-        ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.float]: ...
-        def __init__(
-            self,
-            *,
-            value: collections.abc.Iterable[builtins.float] | None = ...,
-        ) -> None: ...
-        def ClearField(self, field_name: typing_extensions.Literal["value", b"value"]) -> None: ...
-
-    class Sparse(google.protobuf.message.Message):
-        """See pyspark.ml.linalg.SparseVector"""
-
-        DESCRIPTOR: google.protobuf.descriptor.Descriptor
-
-        SIZE_FIELD_NUMBER: builtins.int
-        INDEX_FIELD_NUMBER: builtins.int
-        VALUE_FIELD_NUMBER: builtins.int
-        size: builtins.int
-        @property
-        def index(
-            self,
-        ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: ...
-        @property
-        def value(
-            self,
-        ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.float]: ...
-        def __init__(
-            self,
-            *,
-            size: builtins.int = ...,
-            index: collections.abc.Iterable[builtins.int] | None = ...,
-            value: collections.abc.Iterable[builtins.float] | None = ...,
-        ) -> None: ...
-        def ClearField(
-            self,
-            field_name: typing_extensions.Literal[
-                "index", b"index", "size", b"size", "value", b"value"
-            ],
-        ) -> None: ...
-
-    DENSE_FIELD_NUMBER: builtins.int
-    SPARSE_FIELD_NUMBER: builtins.int
-    @property
-    def dense(self) -> global___Vector.Dense: ...
-    @property
-    def sparse(self) -> global___Vector.Sparse: ...
-    def __init__(
-        self,
-        *,
-        dense: global___Vector.Dense | None = ...,
-        sparse: global___Vector.Sparse | None = ...,
-    ) -> None: ...
-    def HasField(
-        self,
-        field_name: typing_extensions.Literal[
-            "dense", b"dense", "sparse", b"sparse", "vector_type", b"vector_type"
-        ],
-    ) -> builtins.bool: ...
-    def ClearField(
-        self,
-        field_name: typing_extensions.Literal[
-            "dense", b"dense", "sparse", b"sparse", "vector_type", b"vector_type"
-        ],
-    ) -> None: ...
-    def WhichOneof(
-        self, oneof_group: typing_extensions.Literal["vector_type", b"vector_type"]
-    ) -> typing_extensions.Literal["dense", "sparse"] | None: ...
-
-global___Vector = Vector
-
-class Matrix(google.protobuf.message.Message):
-    """See pyspark.ml.linalg.Matrix"""
-
-    DESCRIPTOR: google.protobuf.descriptor.Descriptor
-
-    class Dense(google.protobuf.message.Message):
-        """See pyspark.ml.linalg.DenseMatrix"""
-
-        DESCRIPTOR: google.protobuf.descriptor.Descriptor
-
-        NUM_ROWS_FIELD_NUMBER: builtins.int
-        NUM_COLS_FIELD_NUMBER: builtins.int
-        VALUE_FIELD_NUMBER: builtins.int
-        IS_TRANSPOSED_FIELD_NUMBER: builtins.int
-        num_rows: builtins.int
-        num_cols: builtins.int
-        @property
-        def value(
-            self,
-        ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.float]: ...
-        is_transposed: builtins.bool
-        def __init__(
-            self,
-            *,
-            num_rows: builtins.int = ...,
-            num_cols: builtins.int = ...,
-            value: collections.abc.Iterable[builtins.float] | None = ...,
-            is_transposed: builtins.bool = ...,
-        ) -> None: ...
-        def ClearField(
-            self,
-            field_name: typing_extensions.Literal[
-                "is_transposed",
-                b"is_transposed",
-                "num_cols",
-                b"num_cols",
-                "num_rows",
-                b"num_rows",
-                "value",
-                b"value",
-            ],
-        ) -> None: ...
-
-    class Sparse(google.protobuf.message.Message):
-        """See pyspark.ml.linalg.SparseMatrix"""
-
-        DESCRIPTOR: google.protobuf.descriptor.Descriptor
-
-        NUM_ROWS_FIELD_NUMBER: builtins.int
-        NUM_COLS_FIELD_NUMBER: builtins.int
-        COLPTR_FIELD_NUMBER: builtins.int
-        ROW_INDEX_FIELD_NUMBER: builtins.int
-        VALUE_FIELD_NUMBER: builtins.int
-        IS_TRANSPOSED_FIELD_NUMBER: builtins.int
-        num_rows: builtins.int
-        num_cols: builtins.int
-        @property
-        def colptr(
-            self,
-        ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: ...
-        @property
-        def row_index(
-            self,
-        ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.int]: ...
-        @property
-        def value(
-            self,
-        ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.float]: ...
-        is_transposed: builtins.bool
-        def __init__(
-            self,
-            *,
-            num_rows: builtins.int = ...,
-            num_cols: builtins.int = ...,
-            colptr: collections.abc.Iterable[builtins.int] | None = ...,
-            row_index: collections.abc.Iterable[builtins.int] | None = ...,
-            value: collections.abc.Iterable[builtins.float] | None = ...,
-            is_transposed: builtins.bool = ...,
-        ) -> None: ...
-        def ClearField(
-            self,
-            field_name: typing_extensions.Literal[
-                "colptr",
-                b"colptr",
-                "is_transposed",
-                b"is_transposed",
-                "num_cols",
-                b"num_cols",
-                "num_rows",
-                b"num_rows",
-                "row_index",
-                b"row_index",
-                "value",
-                b"value",
-            ],
-        ) -> None: ...
-
-    DENSE_FIELD_NUMBER: builtins.int
-    SPARSE_FIELD_NUMBER: builtins.int
-    @property
-    def dense(self) -> global___Matrix.Dense: ...
-    @property
-    def sparse(self) -> global___Matrix.Sparse: ...
-    def __init__(
-        self,
-        *,
-        dense: global___Matrix.Dense | None = ...,
-        sparse: global___Matrix.Sparse | None = ...,
-    ) -> None: ...
-    def HasField(
-        self,
-        field_name: typing_extensions.Literal[
-            "dense", b"dense", "matrix_type", b"matrix_type", "sparse", b"sparse"
-        ],
-    ) -> builtins.bool: ...
-    def ClearField(
-        self,
-        field_name: typing_extensions.Literal[
-            "dense", b"dense", "matrix_type", b"matrix_type", "sparse", b"sparse"
-        ],
-    ) -> None: ...
-    def WhichOneof(
-        self, oneof_group: typing_extensions.Literal["matrix_type", b"matrix_type"]
-    ) -> typing_extensions.Literal["dense", "sparse"] | None: ...
-
-global___Matrix = Matrix
diff --git a/python/pyspark/sql/connect/proto/ml_pb2.py b/python/pyspark/sql/connect/proto/ml_pb2.py
index 5005f82d5d53..c29d33db547c 100644
--- a/python/pyspark/sql/connect/proto/ml_pb2.py
+++ b/python/pyspark/sql/connect/proto/ml_pb2.py
@@ -35,11 +35,12 @@ _sym_db = _symbol_database.Default()
 
 
 from pyspark.sql.connect.proto import relations_pb2 as spark_dot_connect_dot_relations__pb2
+from pyspark.sql.connect.proto import expressions_pb2 as spark_dot_connect_dot_expressions__pb2
 from pyspark.sql.connect.proto import ml_common_pb2 as spark_dot_connect_dot_ml__common__pb2
 
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\n\x16spark/connect/ml.proto\x12\rspark.connect\x1a\x1dspark/connect/relations.proto\x1a\x1dspark/connect/ml_common.proto"\xc6\x07\n\tMlCommand\x12\x30\n\x03\x66it\x18\x01 \x01(\x0b\x32\x1c.spark.connect.MlCommand.FitH\x00R\x03\x66it\x12,\n\x05\x66\x65tch\x18\x02 \x01(\x0b\x32\x14.spark.connect.FetchH\x00R\x05\x66\x65tch\x12\x39\n\x06\x64\x65lete\x18\x03 \x01(\x0b\x32\x1f.spark.connect.MlCommand.DeleteH\x00R\x06\x64\x65lete\x12\x36\n\x05write\x18\x04 \x01(\x0b\x32\x1e.spark.connect [...]
+    b'\n\x16spark/connect/ml.proto\x12\rspark.connect\x1a\x1dspark/connect/relations.proto\x1a\x1fspark/connect/expressions.proto\x1a\x1dspark/connect/ml_common.proto"\xc6\x07\n\tMlCommand\x12\x30\n\x03\x66it\x18\x01 \x01(\x0b\x32\x1c.spark.connect.MlCommand.FitH\x00R\x03\x66it\x12,\n\x05\x66\x65tch\x18\x02 \x01(\x0b\x32\x14.spark.connect.FetchH\x00R\x05\x66\x65tch\x12\x39\n\x06\x64\x65lete\x18\x03 \x01(\x0b\x32\x1f.spark.connect.MlCommand.DeleteH\x00R\x06\x64\x65lete\x12\x36\n\x05write\ [...]
 )
 
 _globals = globals()
@@ -52,20 +53,20 @@ if not _descriptor._USE_C_DESCRIPTORS:
     ]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
     _globals["_MLCOMMAND_WRITE_OPTIONSENTRY"]._loaded_options = None
     _globals["_MLCOMMAND_WRITE_OPTIONSENTRY"]._serialized_options = b"8\001"
-    _globals["_MLCOMMAND"]._serialized_start = 104
-    _globals["_MLCOMMAND"]._serialized_end = 1070
-    _globals["_MLCOMMAND_FIT"]._serialized_start = 382
-    _globals["_MLCOMMAND_FIT"]._serialized_end = 544
-    _globals["_MLCOMMAND_DELETE"]._serialized_start = 546
-    _globals["_MLCOMMAND_DELETE"]._serialized_end = 605
-    _globals["_MLCOMMAND_WRITE"]._serialized_start = 608
-    _globals["_MLCOMMAND_WRITE"]._serialized_end = 976
-    _globals["_MLCOMMAND_WRITE_OPTIONSENTRY"]._serialized_start = 910
-    _globals["_MLCOMMAND_WRITE_OPTIONSENTRY"]._serialized_end = 968
-    _globals["_MLCOMMAND_READ"]._serialized_start = 978
-    _globals["_MLCOMMAND_READ"]._serialized_end = 1059
-    _globals["_MLCOMMANDRESULT"]._serialized_start = 1073
-    _globals["_MLCOMMANDRESULT"]._serialized_end = 1434
-    _globals["_MLCOMMANDRESULT_MLOPERATORINFO"]._serialized_start = 1253
-    _globals["_MLCOMMANDRESULT_MLOPERATORINFO"]._serialized_end = 1419
+    _globals["_MLCOMMAND"]._serialized_start = 137
+    _globals["_MLCOMMAND"]._serialized_end = 1103
+    _globals["_MLCOMMAND_FIT"]._serialized_start = 415
+    _globals["_MLCOMMAND_FIT"]._serialized_end = 577
+    _globals["_MLCOMMAND_DELETE"]._serialized_start = 579
+    _globals["_MLCOMMAND_DELETE"]._serialized_end = 638
+    _globals["_MLCOMMAND_WRITE"]._serialized_start = 641
+    _globals["_MLCOMMAND_WRITE"]._serialized_end = 1009
+    _globals["_MLCOMMAND_WRITE_OPTIONSENTRY"]._serialized_start = 943
+    _globals["_MLCOMMAND_WRITE_OPTIONSENTRY"]._serialized_end = 1001
+    _globals["_MLCOMMAND_READ"]._serialized_start = 1011
+    _globals["_MLCOMMAND_READ"]._serialized_end = 1092
+    _globals["_MLCOMMANDRESULT"]._serialized_start = 1106
+    _globals["_MLCOMMANDRESULT"]._serialized_end = 1480
+    _globals["_MLCOMMANDRESULT_MLOPERATORINFO"]._serialized_start = 1299
+    _globals["_MLCOMMANDRESULT_MLOPERATORINFO"]._serialized_end = 1465
 # @@protoc_insertion_point(module_scope)
diff --git a/python/pyspark/sql/connect/proto/ml_pb2.pyi b/python/pyspark/sql/connect/proto/ml_pb2.pyi
index 95bfefb524e2..6b950e4c67bb 100644
--- a/python/pyspark/sql/connect/proto/ml_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/ml_pb2.pyi
@@ -38,6 +38,7 @@ import collections.abc
 import google.protobuf.descriptor
 import google.protobuf.internal.containers
 import google.protobuf.message
+import pyspark.sql.connect.proto.expressions_pb2
 import pyspark.sql.connect.proto.ml_common_pb2
 import pyspark.sql.connect.proto.relations_pb2
 import sys
@@ -346,7 +347,7 @@ class MlCommandResult(google.protobuf.message.Message):
     SUMMARY_FIELD_NUMBER: builtins.int
     OPERATOR_INFO_FIELD_NUMBER: builtins.int
     @property
-    def param(self) -> pyspark.sql.connect.proto.ml_common_pb2.Param:
+    def param(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression.Literal:
         """The result of the attribute"""
     summary: builtins.str
     """Evaluate a Dataset in a model and return the cached ID of summary"""
@@ -356,7 +357,7 @@ class MlCommandResult(google.protobuf.message.Message):
     def __init__(
         self,
         *,
-        param: pyspark.sql.connect.proto.ml_common_pb2.Param | None = ...,
+        param: pyspark.sql.connect.proto.expressions_pb2.Expression.Literal | None = ...,
         summary: builtins.str = ...,
         operator_info: global___MlCommandResult.MlOperatorInfo | None = ...,
     ) -> None: ...
diff --git a/python/pyspark/sql/connect/proto/relations_pb2.py b/python/pyspark/sql/connect/proto/relations_pb2.py
index 4327d0240b35..422addcf3dfb 100644
--- a/python/pyspark/sql/connect/proto/relations_pb2.py
+++ b/python/pyspark/sql/connect/proto/relations_pb2.py
@@ -43,7 +43,7 @@ from pyspark.sql.connect.proto import ml_common_pb2 as spark_dot_connect_dot_ml_
 
 
 DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-    b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x19spark/connect/types.proto\x1a\x1bspark/connect/catalog.proto\x1a\x1aspark/connect/common.proto\x1a\x1dspark/connect/ml_common.proto"\x9c\x1d\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x [...]
+    b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x19spark/connect/types.proto\x1a\x1bspark/connect/catalog.proto\x1a\x1aspark/connect/common.proto\x1a\x1dspark/connect/ml_common.proto"\x9c\x1d\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01 \x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02 \x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03 \x [...]
 )
 
 _globals = globals()
@@ -85,163 +85,163 @@ if not _descriptor._USE_C_DESCRIPTORS:
     _globals["_MLRELATION_TRANSFORM"]._serialized_start = 4097
     _globals["_MLRELATION_TRANSFORM"]._serialized_end = 4332
     _globals["_FETCH"]._serialized_start = 4346
-    _globals["_FETCH"]._serialized_end = 4664
+    _globals["_FETCH"]._serialized_end = 4677
     _globals["_FETCH_METHOD"]._serialized_start = 4462
-    _globals["_FETCH_METHOD"]._serialized_end = 4664
+    _globals["_FETCH_METHOD"]._serialized_end = 4677
     _globals["_FETCH_METHOD_ARGS"]._serialized_start = 4550
-    _globals["_FETCH_METHOD_ARGS"]._serialized_end = 4664
-    _globals["_UNKNOWN"]._serialized_start = 4666
-    _globals["_UNKNOWN"]._serialized_end = 4675
-    _globals["_RELATIONCOMMON"]._serialized_start = 4678
-    _globals["_RELATIONCOMMON"]._serialized_end = 4820
-    _globals["_SQL"]._serialized_start = 4823
-    _globals["_SQL"]._serialized_end = 5301
-    _globals["_SQL_ARGSENTRY"]._serialized_start = 5117
-    _globals["_SQL_ARGSENTRY"]._serialized_end = 5207
-    _globals["_SQL_NAMEDARGUMENTSENTRY"]._serialized_start = 5209
-    _globals["_SQL_NAMEDARGUMENTSENTRY"]._serialized_end = 5301
-    _globals["_WITHRELATIONS"]._serialized_start = 5303
-    _globals["_WITHRELATIONS"]._serialized_end = 5420
-    _globals["_READ"]._serialized_start = 5423
-    _globals["_READ"]._serialized_end = 6086
-    _globals["_READ_NAMEDTABLE"]._serialized_start = 5601
-    _globals["_READ_NAMEDTABLE"]._serialized_end = 5793
-    _globals["_READ_NAMEDTABLE_OPTIONSENTRY"]._serialized_start = 5735
-    _globals["_READ_NAMEDTABLE_OPTIONSENTRY"]._serialized_end = 5793
-    _globals["_READ_DATASOURCE"]._serialized_start = 5796
-    _globals["_READ_DATASOURCE"]._serialized_end = 6073
-    _globals["_READ_DATASOURCE_OPTIONSENTRY"]._serialized_start = 5735
-    _globals["_READ_DATASOURCE_OPTIONSENTRY"]._serialized_end = 5793
-    _globals["_PROJECT"]._serialized_start = 6088
-    _globals["_PROJECT"]._serialized_end = 6205
-    _globals["_FILTER"]._serialized_start = 6207
-    _globals["_FILTER"]._serialized_end = 6319
-    _globals["_JOIN"]._serialized_start = 6322
-    _globals["_JOIN"]._serialized_end = 6983
-    _globals["_JOIN_JOINDATATYPE"]._serialized_start = 6661
-    _globals["_JOIN_JOINDATATYPE"]._serialized_end = 6753
-    _globals["_JOIN_JOINTYPE"]._serialized_start = 6756
-    _globals["_JOIN_JOINTYPE"]._serialized_end = 6964
-    _globals["_SETOPERATION"]._serialized_start = 6986
-    _globals["_SETOPERATION"]._serialized_end = 7465
-    _globals["_SETOPERATION_SETOPTYPE"]._serialized_start = 7302
-    _globals["_SETOPERATION_SETOPTYPE"]._serialized_end = 7416
-    _globals["_LIMIT"]._serialized_start = 7467
-    _globals["_LIMIT"]._serialized_end = 7543
-    _globals["_OFFSET"]._serialized_start = 7545
-    _globals["_OFFSET"]._serialized_end = 7624
-    _globals["_TAIL"]._serialized_start = 7626
-    _globals["_TAIL"]._serialized_end = 7701
-    _globals["_AGGREGATE"]._serialized_start = 7704
-    _globals["_AGGREGATE"]._serialized_end = 8470
-    _globals["_AGGREGATE_PIVOT"]._serialized_start = 8119
-    _globals["_AGGREGATE_PIVOT"]._serialized_end = 8230
-    _globals["_AGGREGATE_GROUPINGSETS"]._serialized_start = 8232
-    _globals["_AGGREGATE_GROUPINGSETS"]._serialized_end = 8308
-    _globals["_AGGREGATE_GROUPTYPE"]._serialized_start = 8311
-    _globals["_AGGREGATE_GROUPTYPE"]._serialized_end = 8470
-    _globals["_SORT"]._serialized_start = 8473
-    _globals["_SORT"]._serialized_end = 8633
-    _globals["_DROP"]._serialized_start = 8636
-    _globals["_DROP"]._serialized_end = 8777
-    _globals["_DEDUPLICATE"]._serialized_start = 8780
-    _globals["_DEDUPLICATE"]._serialized_end = 9020
-    _globals["_LOCALRELATION"]._serialized_start = 9022
-    _globals["_LOCALRELATION"]._serialized_end = 9111
-    _globals["_CACHEDLOCALRELATION"]._serialized_start = 9113
-    _globals["_CACHEDLOCALRELATION"]._serialized_end = 9185
-    _globals["_CACHEDREMOTERELATION"]._serialized_start = 9187
-    _globals["_CACHEDREMOTERELATION"]._serialized_end = 9242
-    _globals["_SAMPLE"]._serialized_start = 9245
-    _globals["_SAMPLE"]._serialized_end = 9518
-    _globals["_RANGE"]._serialized_start = 9521
-    _globals["_RANGE"]._serialized_end = 9666
-    _globals["_SUBQUERYALIAS"]._serialized_start = 9668
-    _globals["_SUBQUERYALIAS"]._serialized_end = 9782
-    _globals["_REPARTITION"]._serialized_start = 9785
-    _globals["_REPARTITION"]._serialized_end = 9927
-    _globals["_SHOWSTRING"]._serialized_start = 9930
-    _globals["_SHOWSTRING"]._serialized_end = 10072
-    _globals["_HTMLSTRING"]._serialized_start = 10074
-    _globals["_HTMLSTRING"]._serialized_end = 10188
-    _globals["_STATSUMMARY"]._serialized_start = 10190
-    _globals["_STATSUMMARY"]._serialized_end = 10282
-    _globals["_STATDESCRIBE"]._serialized_start = 10284
-    _globals["_STATDESCRIBE"]._serialized_end = 10365
-    _globals["_STATCROSSTAB"]._serialized_start = 10367
-    _globals["_STATCROSSTAB"]._serialized_end = 10468
-    _globals["_STATCOV"]._serialized_start = 10470
-    _globals["_STATCOV"]._serialized_end = 10566
-    _globals["_STATCORR"]._serialized_start = 10569
-    _globals["_STATCORR"]._serialized_end = 10706
-    _globals["_STATAPPROXQUANTILE"]._serialized_start = 10709
-    _globals["_STATAPPROXQUANTILE"]._serialized_end = 10873
-    _globals["_STATFREQITEMS"]._serialized_start = 10875
-    _globals["_STATFREQITEMS"]._serialized_end = 11000
-    _globals["_STATSAMPLEBY"]._serialized_start = 11003
-    _globals["_STATSAMPLEBY"]._serialized_end = 11312
-    _globals["_STATSAMPLEBY_FRACTION"]._serialized_start = 11204
-    _globals["_STATSAMPLEBY_FRACTION"]._serialized_end = 11303
-    _globals["_NAFILL"]._serialized_start = 11315
-    _globals["_NAFILL"]._serialized_end = 11449
-    _globals["_NADROP"]._serialized_start = 11452
-    _globals["_NADROP"]._serialized_end = 11586
-    _globals["_NAREPLACE"]._serialized_start = 11589
-    _globals["_NAREPLACE"]._serialized_end = 11885
-    _globals["_NAREPLACE_REPLACEMENT"]._serialized_start = 11744
-    _globals["_NAREPLACE_REPLACEMENT"]._serialized_end = 11885
-    _globals["_TODF"]._serialized_start = 11887
-    _globals["_TODF"]._serialized_end = 11975
-    _globals["_WITHCOLUMNSRENAMED"]._serialized_start = 11978
-    _globals["_WITHCOLUMNSRENAMED"]._serialized_end = 12360
-    _globals["_WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY"]._serialized_start = 12222
-    _globals["_WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY"]._serialized_end = 12289
-    _globals["_WITHCOLUMNSRENAMED_RENAME"]._serialized_start = 12291
-    _globals["_WITHCOLUMNSRENAMED_RENAME"]._serialized_end = 12360
-    _globals["_WITHCOLUMNS"]._serialized_start = 12362
-    _globals["_WITHCOLUMNS"]._serialized_end = 12481
-    _globals["_WITHWATERMARK"]._serialized_start = 12484
-    _globals["_WITHWATERMARK"]._serialized_end = 12618
-    _globals["_HINT"]._serialized_start = 12621
-    _globals["_HINT"]._serialized_end = 12753
-    _globals["_UNPIVOT"]._serialized_start = 12756
-    _globals["_UNPIVOT"]._serialized_end = 13083
-    _globals["_UNPIVOT_VALUES"]._serialized_start = 13013
-    _globals["_UNPIVOT_VALUES"]._serialized_end = 13072
-    _globals["_TRANSPOSE"]._serialized_start = 13085
-    _globals["_TRANSPOSE"]._serialized_end = 13207
-    _globals["_UNRESOLVEDTABLEVALUEDFUNCTION"]._serialized_start = 13209
-    _globals["_UNRESOLVEDTABLEVALUEDFUNCTION"]._serialized_end = 13334
-    _globals["_TOSCHEMA"]._serialized_start = 13336
-    _globals["_TOSCHEMA"]._serialized_end = 13442
-    _globals["_REPARTITIONBYEXPRESSION"]._serialized_start = 13445
-    _globals["_REPARTITIONBYEXPRESSION"]._serialized_end = 13648
-    _globals["_MAPPARTITIONS"]._serialized_start = 13651
-    _globals["_MAPPARTITIONS"]._serialized_end = 13883
-    _globals["_GROUPMAP"]._serialized_start = 13886
-    _globals["_GROUPMAP"]._serialized_end = 14603
-    _globals["_COGROUPMAP"]._serialized_start = 14606
-    _globals["_COGROUPMAP"]._serialized_end = 15132
-    _globals["_APPLYINPANDASWITHSTATE"]._serialized_start = 15135
-    _globals["_APPLYINPANDASWITHSTATE"]._serialized_end = 15492
-    _globals["_COMMONINLINEUSERDEFINEDTABLEFUNCTION"]._serialized_start = 15495
-    _globals["_COMMONINLINEUSERDEFINEDTABLEFUNCTION"]._serialized_end = 15739
-    _globals["_PYTHONUDTF"]._serialized_start = 15742
-    _globals["_PYTHONUDTF"]._serialized_end = 15919
-    _globals["_COMMONINLINEUSERDEFINEDDATASOURCE"]._serialized_start = 15922
-    _globals["_COMMONINLINEUSERDEFINEDDATASOURCE"]._serialized_end = 16073
-    _globals["_PYTHONDATASOURCE"]._serialized_start = 16075
-    _globals["_PYTHONDATASOURCE"]._serialized_end = 16150
-    _globals["_COLLECTMETRICS"]._serialized_start = 16153
-    _globals["_COLLECTMETRICS"]._serialized_end = 16289
-    _globals["_PARSE"]._serialized_start = 16292
-    _globals["_PARSE"]._serialized_end = 16680
-    _globals["_PARSE_OPTIONSENTRY"]._serialized_start = 5735
-    _globals["_PARSE_OPTIONSENTRY"]._serialized_end = 5793
-    _globals["_PARSE_PARSEFORMAT"]._serialized_start = 16581
-    _globals["_PARSE_PARSEFORMAT"]._serialized_end = 16669
-    _globals["_ASOFJOIN"]._serialized_start = 16683
-    _globals["_ASOFJOIN"]._serialized_end = 17158
-    _globals["_LATERALJOIN"]._serialized_start = 17161
-    _globals["_LATERALJOIN"]._serialized_end = 17391
+    _globals["_FETCH_METHOD_ARGS"]._serialized_end = 4677
+    _globals["_UNKNOWN"]._serialized_start = 4679
+    _globals["_UNKNOWN"]._serialized_end = 4688
+    _globals["_RELATIONCOMMON"]._serialized_start = 4691
+    _globals["_RELATIONCOMMON"]._serialized_end = 4833
+    _globals["_SQL"]._serialized_start = 4836
+    _globals["_SQL"]._serialized_end = 5314
+    _globals["_SQL_ARGSENTRY"]._serialized_start = 5130
+    _globals["_SQL_ARGSENTRY"]._serialized_end = 5220
+    _globals["_SQL_NAMEDARGUMENTSENTRY"]._serialized_start = 5222
+    _globals["_SQL_NAMEDARGUMENTSENTRY"]._serialized_end = 5314
+    _globals["_WITHRELATIONS"]._serialized_start = 5316
+    _globals["_WITHRELATIONS"]._serialized_end = 5433
+    _globals["_READ"]._serialized_start = 5436
+    _globals["_READ"]._serialized_end = 6099
+    _globals["_READ_NAMEDTABLE"]._serialized_start = 5614
+    _globals["_READ_NAMEDTABLE"]._serialized_end = 5806
+    _globals["_READ_NAMEDTABLE_OPTIONSENTRY"]._serialized_start = 5748
+    _globals["_READ_NAMEDTABLE_OPTIONSENTRY"]._serialized_end = 5806
+    _globals["_READ_DATASOURCE"]._serialized_start = 5809
+    _globals["_READ_DATASOURCE"]._serialized_end = 6086
+    _globals["_READ_DATASOURCE_OPTIONSENTRY"]._serialized_start = 5748
+    _globals["_READ_DATASOURCE_OPTIONSENTRY"]._serialized_end = 5806
+    _globals["_PROJECT"]._serialized_start = 6101
+    _globals["_PROJECT"]._serialized_end = 6218
+    _globals["_FILTER"]._serialized_start = 6220
+    _globals["_FILTER"]._serialized_end = 6332
+    _globals["_JOIN"]._serialized_start = 6335
+    _globals["_JOIN"]._serialized_end = 6996
+    _globals["_JOIN_JOINDATATYPE"]._serialized_start = 6674
+    _globals["_JOIN_JOINDATATYPE"]._serialized_end = 6766
+    _globals["_JOIN_JOINTYPE"]._serialized_start = 6769
+    _globals["_JOIN_JOINTYPE"]._serialized_end = 6977
+    _globals["_SETOPERATION"]._serialized_start = 6999
+    _globals["_SETOPERATION"]._serialized_end = 7478
+    _globals["_SETOPERATION_SETOPTYPE"]._serialized_start = 7315
+    _globals["_SETOPERATION_SETOPTYPE"]._serialized_end = 7429
+    _globals["_LIMIT"]._serialized_start = 7480
+    _globals["_LIMIT"]._serialized_end = 7556
+    _globals["_OFFSET"]._serialized_start = 7558
+    _globals["_OFFSET"]._serialized_end = 7637
+    _globals["_TAIL"]._serialized_start = 7639
+    _globals["_TAIL"]._serialized_end = 7714
+    _globals["_AGGREGATE"]._serialized_start = 7717
+    _globals["_AGGREGATE"]._serialized_end = 8483
+    _globals["_AGGREGATE_PIVOT"]._serialized_start = 8132
+    _globals["_AGGREGATE_PIVOT"]._serialized_end = 8243
+    _globals["_AGGREGATE_GROUPINGSETS"]._serialized_start = 8245
+    _globals["_AGGREGATE_GROUPINGSETS"]._serialized_end = 8321
+    _globals["_AGGREGATE_GROUPTYPE"]._serialized_start = 8324
+    _globals["_AGGREGATE_GROUPTYPE"]._serialized_end = 8483
+    _globals["_SORT"]._serialized_start = 8486
+    _globals["_SORT"]._serialized_end = 8646
+    _globals["_DROP"]._serialized_start = 8649
+    _globals["_DROP"]._serialized_end = 8790
+    _globals["_DEDUPLICATE"]._serialized_start = 8793
+    _globals["_DEDUPLICATE"]._serialized_end = 9033
+    _globals["_LOCALRELATION"]._serialized_start = 9035
+    _globals["_LOCALRELATION"]._serialized_end = 9124
+    _globals["_CACHEDLOCALRELATION"]._serialized_start = 9126
+    _globals["_CACHEDLOCALRELATION"]._serialized_end = 9198
+    _globals["_CACHEDREMOTERELATION"]._serialized_start = 9200
+    _globals["_CACHEDREMOTERELATION"]._serialized_end = 9255
+    _globals["_SAMPLE"]._serialized_start = 9258
+    _globals["_SAMPLE"]._serialized_end = 9531
+    _globals["_RANGE"]._serialized_start = 9534
+    _globals["_RANGE"]._serialized_end = 9679
+    _globals["_SUBQUERYALIAS"]._serialized_start = 9681
+    _globals["_SUBQUERYALIAS"]._serialized_end = 9795
+    _globals["_REPARTITION"]._serialized_start = 9798
+    _globals["_REPARTITION"]._serialized_end = 9940
+    _globals["_SHOWSTRING"]._serialized_start = 9943
+    _globals["_SHOWSTRING"]._serialized_end = 10085
+    _globals["_HTMLSTRING"]._serialized_start = 10087
+    _globals["_HTMLSTRING"]._serialized_end = 10201
+    _globals["_STATSUMMARY"]._serialized_start = 10203
+    _globals["_STATSUMMARY"]._serialized_end = 10295
+    _globals["_STATDESCRIBE"]._serialized_start = 10297
+    _globals["_STATDESCRIBE"]._serialized_end = 10378
+    _globals["_STATCROSSTAB"]._serialized_start = 10380
+    _globals["_STATCROSSTAB"]._serialized_end = 10481
+    _globals["_STATCOV"]._serialized_start = 10483
+    _globals["_STATCOV"]._serialized_end = 10579
+    _globals["_STATCORR"]._serialized_start = 10582
+    _globals["_STATCORR"]._serialized_end = 10719
+    _globals["_STATAPPROXQUANTILE"]._serialized_start = 10722
+    _globals["_STATAPPROXQUANTILE"]._serialized_end = 10886
+    _globals["_STATFREQITEMS"]._serialized_start = 10888
+    _globals["_STATFREQITEMS"]._serialized_end = 11013
+    _globals["_STATSAMPLEBY"]._serialized_start = 11016
+    _globals["_STATSAMPLEBY"]._serialized_end = 11325
+    _globals["_STATSAMPLEBY_FRACTION"]._serialized_start = 11217
+    _globals["_STATSAMPLEBY_FRACTION"]._serialized_end = 11316
+    _globals["_NAFILL"]._serialized_start = 11328
+    _globals["_NAFILL"]._serialized_end = 11462
+    _globals["_NADROP"]._serialized_start = 11465
+    _globals["_NADROP"]._serialized_end = 11599
+    _globals["_NAREPLACE"]._serialized_start = 11602
+    _globals["_NAREPLACE"]._serialized_end = 11898
+    _globals["_NAREPLACE_REPLACEMENT"]._serialized_start = 11757
+    _globals["_NAREPLACE_REPLACEMENT"]._serialized_end = 11898
+    _globals["_TODF"]._serialized_start = 11900
+    _globals["_TODF"]._serialized_end = 11988
+    _globals["_WITHCOLUMNSRENAMED"]._serialized_start = 11991
+    _globals["_WITHCOLUMNSRENAMED"]._serialized_end = 12373
+    _globals["_WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY"]._serialized_start = 12235
+    _globals["_WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY"]._serialized_end = 12302
+    _globals["_WITHCOLUMNSRENAMED_RENAME"]._serialized_start = 12304
+    _globals["_WITHCOLUMNSRENAMED_RENAME"]._serialized_end = 12373
+    _globals["_WITHCOLUMNS"]._serialized_start = 12375
+    _globals["_WITHCOLUMNS"]._serialized_end = 12494
+    _globals["_WITHWATERMARK"]._serialized_start = 12497
+    _globals["_WITHWATERMARK"]._serialized_end = 12631
+    _globals["_HINT"]._serialized_start = 12634
+    _globals["_HINT"]._serialized_end = 12766
+    _globals["_UNPIVOT"]._serialized_start = 12769
+    _globals["_UNPIVOT"]._serialized_end = 13096
+    _globals["_UNPIVOT_VALUES"]._serialized_start = 13026
+    _globals["_UNPIVOT_VALUES"]._serialized_end = 13085
+    _globals["_TRANSPOSE"]._serialized_start = 13098
+    _globals["_TRANSPOSE"]._serialized_end = 13220
+    _globals["_UNRESOLVEDTABLEVALUEDFUNCTION"]._serialized_start = 13222
+    _globals["_UNRESOLVEDTABLEVALUEDFUNCTION"]._serialized_end = 13347
+    _globals["_TOSCHEMA"]._serialized_start = 13349
+    _globals["_TOSCHEMA"]._serialized_end = 13455
+    _globals["_REPARTITIONBYEXPRESSION"]._serialized_start = 13458
+    _globals["_REPARTITIONBYEXPRESSION"]._serialized_end = 13661
+    _globals["_MAPPARTITIONS"]._serialized_start = 13664
+    _globals["_MAPPARTITIONS"]._serialized_end = 13896
+    _globals["_GROUPMAP"]._serialized_start = 13899
+    _globals["_GROUPMAP"]._serialized_end = 14616
+    _globals["_COGROUPMAP"]._serialized_start = 14619
+    _globals["_COGROUPMAP"]._serialized_end = 15145
+    _globals["_APPLYINPANDASWITHSTATE"]._serialized_start = 15148
+    _globals["_APPLYINPANDASWITHSTATE"]._serialized_end = 15505
+    _globals["_COMMONINLINEUSERDEFINEDTABLEFUNCTION"]._serialized_start = 15508
+    _globals["_COMMONINLINEUSERDEFINEDTABLEFUNCTION"]._serialized_end = 15752
+    _globals["_PYTHONUDTF"]._serialized_start = 15755
+    _globals["_PYTHONUDTF"]._serialized_end = 15932
+    _globals["_COMMONINLINEUSERDEFINEDDATASOURCE"]._serialized_start = 15935
+    _globals["_COMMONINLINEUSERDEFINEDDATASOURCE"]._serialized_end = 16086
+    _globals["_PYTHONDATASOURCE"]._serialized_start = 16088
+    _globals["_PYTHONDATASOURCE"]._serialized_end = 16163
+    _globals["_COLLECTMETRICS"]._serialized_start = 16166
+    _globals["_COLLECTMETRICS"]._serialized_end = 16302
+    _globals["_PARSE"]._serialized_start = 16305
+    _globals["_PARSE"]._serialized_end = 16693
+    _globals["_PARSE_OPTIONSENTRY"]._serialized_start = 5748
+    _globals["_PARSE_OPTIONSENTRY"]._serialized_end = 5806
+    _globals["_PARSE_PARSEFORMAT"]._serialized_start = 16594
+    _globals["_PARSE_PARSEFORMAT"]._serialized_end = 16682
+    _globals["_ASOFJOIN"]._serialized_start = 16696
+    _globals["_ASOFJOIN"]._serialized_end = 17171
+    _globals["_LATERALJOIN"]._serialized_start = 17174
+    _globals["_LATERALJOIN"]._serialized_end = 17404
 # @@protoc_insertion_point(module_scope)
diff --git a/python/pyspark/sql/connect/proto/relations_pb2.pyi b/python/pyspark/sql/connect/proto/relations_pb2.pyi
index 0c8cf8dd3eda..d4f1233e45f9 100644
--- a/python/pyspark/sql/connect/proto/relations_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/relations_pb2.pyi
@@ -756,13 +756,13 @@ class Fetch(google.protobuf.message.Message):
             PARAM_FIELD_NUMBER: builtins.int
             INPUT_FIELD_NUMBER: builtins.int
             @property
-            def param(self) -> pyspark.sql.connect.proto.ml_common_pb2.Param: ...
+            def param(self) -> pyspark.sql.connect.proto.expressions_pb2.Expression.Literal: ...
             @property
             def input(self) -> global___Relation: ...
             def __init__(
                 self,
                 *,
-                param: pyspark.sql.connect.proto.ml_common_pb2.Param | None = ...,
+                param: pyspark.sql.connect.proto.expressions_pb2.Expression.Literal | None = ...,
                 input: global___Relation | None = ...,
             ) -> None: ...
             def HasField(
diff --git a/sql/connect/common/src/main/protobuf/spark/connect/common.proto b/sql/connect/common/src/main/protobuf/spark/connect/common.proto
index b2848370b01d..dfa30d151f88 100644
--- a/sql/connect/common/src/main/protobuf/spark/connect/common.proto
+++ b/sql/connect/common/src/main/protobuf/spark/connect/common.proto
@@ -96,3 +96,27 @@ message PythonOrigin {
   // (Required) Callsite to show to end users, for example, stacktrace.
   string call_site = 2;
 }
+
+message Bools {
+  repeated bool values = 1;
+}
+
+message Ints {
+  repeated int32 values = 1;
+}
+
+message Longs {
+  repeated int64 values = 1;
+}
+
+message Floats {
+  repeated float values = 1;
+}
+
+message Doubles {
+  repeated double values = 1;
+}
+
+message Strings {
+  repeated string values = 1;
+}
diff --git a/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto b/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto
index bbe605a47f4f..a5fadd913622 100644
--- a/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto
+++ b/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto
@@ -194,6 +194,8 @@ message Expression {
       Array array = 22;
       Map map = 23;
       Struct struct = 24;
+
+      SpecializedArray specialized_array = 25;
     }
 
     message Decimal {
@@ -228,6 +230,17 @@ message Expression {
       DataType struct_type = 1;
       repeated Literal elements = 2;
     }
+
+    message SpecializedArray {
+      oneof value_type {
+        Bools bools = 1;
+        Ints ints = 2;
+        Longs longs = 3;
+        Floats floats = 4;
+        Doubles doubles = 5;
+        Strings strings = 6;
+      }
+    }
   }
 
   // An unresolved attribute that is not explicitly bound to a specific column, but the column
diff --git a/sql/connect/common/src/main/protobuf/spark/connect/ml.proto b/sql/connect/common/src/main/protobuf/spark/connect/ml.proto
index 48b04a6e14cd..3198bbbe9c11 100644
--- a/sql/connect/common/src/main/protobuf/spark/connect/ml.proto
+++ b/sql/connect/common/src/main/protobuf/spark/connect/ml.proto
@@ -20,6 +20,7 @@ syntax = 'proto3';
 package spark.connect;
 
 import "spark/connect/relations.proto";
+import "spark/connect/expressions.proto";
 import "spark/connect/ml_common.proto";
 
 option java_multiple_files = true;
@@ -84,7 +85,7 @@ message MlCommand {
 message MlCommandResult {
   oneof result_type {
     // The result of the attribute
-    Param param = 1;
+    Expression.Literal param = 1;
     // Evaluate a Dataset in a model and return the cached ID of summary
     string summary = 2;
     // Operator information
diff --git a/sql/connect/common/src/main/protobuf/spark/connect/ml_common.proto b/sql/connect/common/src/main/protobuf/spark/connect/ml_common.proto
index f91c2489ed94..48b5fa8135cc 100644
--- a/sql/connect/common/src/main/protobuf/spark/connect/ml_common.proto
+++ b/sql/connect/common/src/main/protobuf/spark/connect/ml_common.proto
@@ -28,17 +28,7 @@ option go_package = "internal/generated";
 // MlParams stores param settings for ML Estimator / Transformer / Evaluator
 message MlParams {
   // User-supplied params
-  map<string, Param> params = 1;
-}
-
-// Represents the parameter type of the ML instance, or the returned value
-// of the attribute
-message Param {
-  oneof param_type {
-    Expression.Literal literal = 1;
-    Vector vector = 2;
-    Matrix matrix = 3;
-  }
+  map<string, Expression.Literal> params = 1;
 }
 
 // MLOperator represents the ML operators like (Estimator, Transformer or Evaluator)
@@ -64,45 +54,3 @@ message ObjectRef {
   // The ID is used to lookup the object on the server side.
   string id = 1;
 }
-
-// See pyspark.ml.linalg.Vector
-message Vector {
-  oneof vector_type {
-    Dense dense = 1;
-    Sparse sparse = 2;
-  }
-  // See pyspark.ml.linalg.DenseVector
-  message Dense {
-    repeated double value = 1;
-  }
-  // See pyspark.ml.linalg.SparseVector
-  message Sparse {
-    int32 size = 1;
-    repeated int32 index = 2;
-    repeated double value = 3;
-  }
-}
-
-// See pyspark.ml.linalg.Matrix
-message Matrix {
-  oneof matrix_type {
-    Dense dense = 1;
-    Sparse sparse = 2;
-  }
-  // See pyspark.ml.linalg.DenseMatrix
-  message Dense {
-    int32 num_rows = 1;
-    int32 num_cols = 2;
-    repeated double value = 3;
-    bool is_transposed = 4;
-  }
-  // See pyspark.ml.linalg.SparseMatrix
-  message Sparse {
-    int32 num_rows = 1;
-    int32 num_cols = 2;
-    repeated int32 colptr = 3;
-    repeated int32 row_index = 4;
-    repeated double value = 5;
-    bool is_transposed = 6;
-  }
-}
diff --git a/sql/connect/common/src/main/protobuf/spark/connect/relations.proto b/sql/connect/common/src/main/protobuf/spark/connect/relations.proto
index c2cbed0dd22b..58ce05785978 100644
--- a/sql/connect/common/src/main/protobuf/spark/connect/relations.proto
+++ b/sql/connect/common/src/main/protobuf/spark/connect/relations.proto
@@ -151,7 +151,7 @@ message Fetch {
 
     message Args {
       oneof args_type {
-        Param param = 1;
+        Expression.Literal param = 1;
         Relation input = 2;
       }
     }
diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLHandler.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLHandler.scala
index 1f0981e048af..e53276dcf7df 100644
--- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLHandler.scala
+++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLHandler.scala
@@ -153,11 +153,7 @@ private[connect] object MLHandler extends Logging {
         }
         proto.MlCommandResult
           .newBuilder()
-          .setParam(
-            proto.Param
-              .newBuilder()
-              .setLiteral(LiteralValueProtoConverter.toLiteralProto(result))
-              .build())
+          .setParam(LiteralValueProtoConverter.toLiteralProto(result))
           .build()
 
       case proto.MlCommand.CommandCase.WRITE =>
diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLUtils.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLUtils.scala
index 2c006d86dfac..4dbfb063eabb 100644
--- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLUtils.scala
+++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLUtils.scala
@@ -26,11 +26,11 @@ import org.apache.commons.lang3.reflect.MethodUtils.invokeMethod
 
 import org.apache.spark.connect.proto
 import org.apache.spark.ml.{Estimator, Transformer}
-import org.apache.spark.ml.linalg.{Matrices, Matrix, Vector, Vectors}
+import org.apache.spark.ml.linalg._
 import org.apache.spark.ml.param.Params
 import org.apache.spark.ml.util.{MLReadable, MLWritable}
 import org.apache.spark.sql.{DataFrame, Dataset}
-import org.apache.spark.sql.connect.common.LiteralValueProtoConverter
+import org.apache.spark.sql.connect.common.{DataTypeProtoConverter, LiteralValueProtoConverter}
 import org.apache.spark.sql.connect.planner.SparkConnectPlanner
 import org.apache.spark.sql.connect.plugin.SparkConnectPluginRegistry
 import org.apache.spark.sql.connect.service.SessionHolder
@@ -53,28 +53,65 @@ private[ml] object MLUtils {
     providers.map(est => est.getClass.getName -> est.getClass).toMap
   }
 
-  def deserializeVector(vector: proto.Vector): Vector = {
-    if (vector.hasDense) {
-      val values = vector.getDense.getValueList.asScala.map(_.toDouble).toArray
-      Vectors.dense(values)
-    } else {
-      val size = vector.getSparse.getSize
-      val indices = vector.getSparse.getIndexList.asScala.map(_.toInt).toArray
-      val values = vector.getSparse.getValueList.asScala.map(_.toDouble).toArray
-      Vectors.sparse(size, indices, values)
+  private def parseInts(ints: proto.Ints): Array[Int] = {
+    val size = ints.getValuesCount
+    val values = Array.ofDim[Int](size)
+    var i = 0
+    while (i < size) {
+      values(i) = ints.getValues(i)
+      i += 1
     }
+    values
   }
 
-  def deserializeMatrix(matrix: proto.Matrix): Matrix = {
-    if (matrix.hasDense) {
-      val values = matrix.getDense.getValueList.asScala.map(_.toDouble).toArray
-      Matrices.dense(matrix.getDense.getNumRows, matrix.getDense.getNumCols, values)
-    } else {
-      val sparse = matrix.getSparse
-      val colPtrs = sparse.getColptrList.asScala.map(_.toInt).toArray
-      val rowIndices = sparse.getRowIndexList.asScala.map(_.toInt).toArray
-      val values = sparse.getValueList.asScala.map(_.toDouble).toArray
-      Matrices.sparse(sparse.getNumRows, sparse.getNumCols, colPtrs, rowIndices, values)
+  private def parseDoubles(doubles: proto.Doubles): Array[Double] = {
+    val size = doubles.getValuesCount
+    val values = Array.ofDim[Double](size)
+    var i = 0
+    while (i < size) {
+      values(i) = doubles.getValues(i)
+      i += 1
+    }
+    values
+  }
+
+  def deserializeVector(s: proto.Expression.Literal.Struct): Vector = {
+    assert(s.getElementsCount == 4)
+    s.getElements(0).getByte match {
+      case 0 =>
+        val size = s.getElements(1).getInteger
+        val indices = parseInts(s.getElements(2).getSpecializedArray.getInts)
+        val values = parseDoubles(s.getElements(3).getSpecializedArray.getDoubles)
+        Vectors.sparse(size, indices, values)
+
+      case 1 =>
+        val values = parseDoubles(s.getElements(3).getSpecializedArray.getDoubles)
+        Vectors.dense(values)
+
+      case o => throw MlUnsupportedException(s"Unknown Vector type $o")
+    }
+  }
+
+  def deserializeMatrix(s: proto.Expression.Literal.Struct): Matrix = {
+    assert(s.getElementsCount == 7)
+    s.getElements(0).getByte match {
+      case 0 =>
+        val numRows = s.getElements(1).getInteger
+        val numCols = s.getElements(2).getInteger
+        val colPtrs = parseInts(s.getElements(3).getSpecializedArray.getInts)
+        val rowIndices = parseInts(s.getElements(4).getSpecializedArray.getInts)
+        val values = parseDoubles(s.getElements(5).getSpecializedArray.getDoubles)
+        val isTransposed = s.getElements(6).getBoolean
+        new SparseMatrix(numRows, numCols, colPtrs, rowIndices, values, isTransposed)
+
+      case 1 =>
+        val numRows = s.getElements(1).getInteger
+        val numCols = s.getElements(2).getInteger
+        val values = parseDoubles(s.getElements(5).getSpecializedArray.getDoubles)
+        val isTransposed = s.getElements(6).getBoolean
+        new DenseMatrix(numRows, numCols, values, isTransposed)
+
+      case o => throw MlUnsupportedException(s"Unknown Matrix type $o")
     }
   }
 
@@ -87,18 +124,24 @@ private[ml] object MLUtils {
    *   the parameters of the ML operator
    */
   def setInstanceParams(instance: Params, params: proto.MlParams): Unit = {
-    params.getParamsMap.asScala.foreach { case (name, paramProto) =>
+    params.getParamsMap.asScala.foreach { case (name, literal) =>
       val p = instance.getParam(name)
-      val value = if (paramProto.hasLiteral) {
-        reconcileParam(
-          p.paramValueClassTag.runtimeClass,
-          LiteralValueProtoConverter.toCatalystValue(paramProto.getLiteral))
-      } else if (paramProto.hasVector) {
-        deserializeVector(paramProto.getVector)
-      } else if (paramProto.hasMatrix) {
-        deserializeMatrix(paramProto.getMatrix)
-      } else {
-        throw MlUnsupportedException(s"Unsupported parameter type for ${name}")
+      val value = literal.getLiteralTypeCase match {
+        case proto.Expression.Literal.LiteralTypeCase.STRUCT =>
+          val s = literal.getStruct
+          val schema = DataTypeProtoConverter.toCatalystType(s.getStructType)
+          if (schema == VectorUDT.sqlType) {
+            deserializeVector(s)
+          } else if (schema == MatrixUDT.sqlType) {
+            deserializeMatrix(s)
+          } else {
+            throw MlUnsupportedException(s"Unsupported parameter struct ${schema} for ${name}")
+          }
+
+        case _ =>
+          reconcileParam(
+            p.paramValueClassTag.runtimeClass,
+            LiteralValueProtoConverter.toCatalystValue(literal))
       }
       instance.set(p, value)
     }
diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/Serializer.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/Serializer.scala
index ad6735997f83..ee0812a1a98c 100644
--- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/Serializer.scala
+++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/Serializer.scala
@@ -18,10 +18,10 @@
 package org.apache.spark.sql.connect.ml
 
 import org.apache.spark.connect.proto
-import org.apache.spark.ml.linalg.{DenseMatrix, DenseVector, SparseMatrix, SparseVector}
+import org.apache.spark.ml.linalg._
 import org.apache.spark.ml.param.Params
 import org.apache.spark.sql.Dataset
-import org.apache.spark.sql.connect.common.LiteralValueProtoConverter
+import org.apache.spark.sql.connect.common.{DataTypeProtoConverter, LiteralValueProtoConverter, ProtoDataTypes}
 import org.apache.spark.sql.connect.service.SessionHolder
 
 private[ml] object Serializer {
@@ -31,97 +31,141 @@ private[ml] object Serializer {
    * @param data
    *   the value of parameter
    * @return
-   *   proto.Param
+   *   proto.Expression.Literal
    */
-  def serializeParam(data: Any): proto.Param = {
+  def serializeParam(data: Any): proto.Expression.Literal = {
     data match {
-      case v: DenseVector =>
-        val denseBuilder = proto.Vector.Dense.newBuilder()
-        v.values.foreach(denseBuilder.addValue)
-        proto.Param
-          .newBuilder()
-          .setVector(proto.Vector.newBuilder().setDense(denseBuilder))
-          .build()
       case v: SparseVector =>
-        val sparseBuilder = proto.Vector.Sparse.newBuilder().setSize(v.size)
-        v.indices.foreach(sparseBuilder.addIndex)
-        v.values.foreach(sparseBuilder.addValue)
-        proto.Param
-          .newBuilder()
-          .setVector(proto.Vector.newBuilder().setSparse(sparseBuilder))
-          .build()
-      case v: DenseMatrix =>
-        val denseBuilder = proto.Matrix.Dense.newBuilder()
-        v.values.foreach(denseBuilder.addValue)
-        denseBuilder.setNumCols(v.numCols)
-        denseBuilder.setNumRows(v.numRows)
-        denseBuilder.setIsTransposed(v.isTransposed)
-        proto.Param
-          .newBuilder()
-          .setMatrix(proto.Matrix.newBuilder().setDense(denseBuilder))
-          .build()
-      case v: SparseMatrix =>
-        val sparseBuilder = proto.Matrix.Sparse
-          .newBuilder()
-          .setNumCols(v.numCols)
-          .setNumRows(v.numRows)
-        v.values.foreach(sparseBuilder.addValue)
-        v.colPtrs.foreach(sparseBuilder.addColptr)
-        v.rowIndices.foreach(sparseBuilder.addRowIndex)
-        proto.Param
-          .newBuilder()
-          .setMatrix(proto.Matrix.newBuilder().setSparse(sparseBuilder))
-          .build()
+        val builder = proto.Expression.Literal.Struct.newBuilder()
+        builder.setStructType(DataTypeProtoConverter.toConnectProtoType(VectorUDT.sqlType))
+        // type = 0
+        builder.addElements(proto.Expression.Literal.newBuilder().setByte(0))
+        // size
+        builder.addElements(proto.Expression.Literal.newBuilder().setInteger(v.size))
+        // indices
+        builder.addElements(buildIntArray(v.indices))
+        // values
+        builder.addElements(buildDoubleArray(v.values))
+        proto.Expression.Literal.newBuilder().setStruct(builder).build()
+
+      case v: DenseVector =>
+        val builder = proto.Expression.Literal.Struct.newBuilder()
+        builder.setStructType(DataTypeProtoConverter.toConnectProtoType(VectorUDT.sqlType))
+        // type = 1
+        builder.addElements(proto.Expression.Literal.newBuilder().setByte(1))
+        // size = null
+        builder.addElements(
+          proto.Expression.Literal.newBuilder().setNull(ProtoDataTypes.NullType))
+        // indices = null
+        builder.addElements(
+          proto.Expression.Literal.newBuilder().setNull(ProtoDataTypes.NullType))
+        // values
+        builder.addElements(buildDoubleArray(v.values))
+        proto.Expression.Literal.newBuilder().setStruct(builder).build()
+
+      case m: SparseMatrix =>
+        val builder = proto.Expression.Literal.Struct.newBuilder()
+        builder.setStructType(DataTypeProtoConverter.toConnectProtoType(MatrixUDT.sqlType))
+        // type = 0
+        builder.addElements(proto.Expression.Literal.newBuilder().setByte(0))
+        // numRows
+        builder.addElements(proto.Expression.Literal.newBuilder().setInteger(m.numRows))
+        // numCols
+        builder.addElements(proto.Expression.Literal.newBuilder().setInteger(m.numCols))
+        // colPtrs
+        builder.addElements(buildIntArray(m.colPtrs))
+        // rowIndices
+        builder.addElements(buildIntArray(m.rowIndices))
+        // values
+        builder.addElements(buildDoubleArray(m.values))
+        // isTransposed
+        builder.addElements(proto.Expression.Literal.newBuilder().setBoolean(m.isTransposed))
+        proto.Expression.Literal.newBuilder().setStruct(builder).build()
+
+      case m: DenseMatrix =>
+        val builder = proto.Expression.Literal.Struct.newBuilder()
+        builder.setStructType(DataTypeProtoConverter.toConnectProtoType(MatrixUDT.sqlType))
+        // type = 1
+        builder.addElements(proto.Expression.Literal.newBuilder().setByte(1))
+        // numRows
+        builder.addElements(proto.Expression.Literal.newBuilder().setInteger(m.numRows))
+        // numCols
+        builder.addElements(proto.Expression.Literal.newBuilder().setInteger(m.numCols))
+        // colPtrs = null
+        builder.addElements(
+          proto.Expression.Literal.newBuilder().setNull(ProtoDataTypes.NullType))
+        // rowIndices = null
+        builder.addElements(
+          proto.Expression.Literal.newBuilder().setNull(ProtoDataTypes.NullType))
+        // values
+        builder.addElements(buildDoubleArray(m.values))
+        // isTransposed
+        builder.addElements(proto.Expression.Literal.newBuilder().setBoolean(m.isTransposed))
+        proto.Expression.Literal.newBuilder().setStruct(builder).build()
+
       case _: Byte | _: Short | _: Int | _: Long | _: Float | _: Double | _: Boolean | _: String |
           _: Array[_] =>
-        proto.Param
-          .newBuilder()
-          .setLiteral(LiteralValueProtoConverter.toLiteralProto(data))
-          .build()
+        LiteralValueProtoConverter.toLiteralProto(data)
 
       case other => throw MlUnsupportedException(s"$other not supported")
     }
   }
 
+  private def buildIntArray(values: Array[Int]): proto.Expression.Literal = {
+    val builder = proto.Ints.newBuilder()
+    values.foreach(builder.addValues)
+    proto.Expression.Literal
+      .newBuilder()
+      .setSpecializedArray(
+        proto.Expression.Literal.SpecializedArray
+          .newBuilder()
+          .setInts(builder)
+          .build())
+      .build()
+  }
+
+  private def buildDoubleArray(values: Array[Double]): proto.Expression.Literal = {
+    val builder = proto.Doubles.newBuilder()
+    values.foreach(builder.addValues)
+    proto.Expression.Literal
+      .newBuilder()
+      .setSpecializedArray(
+        proto.Expression.Literal.SpecializedArray
+          .newBuilder()
+          .setDoubles(builder)
+          .build())
+      .build()
+  }
+
   def deserializeMethodArguments(
       args: Array[proto.Fetch.Method.Args],
       sessionHolder: SessionHolder): Array[(Object, Class[_])] = {
     args.map { arg =>
       if (arg.hasParam) {
-        val param = arg.getParam
-        if (param.hasLiteral) {
-          param.getLiteral.getLiteralTypeCase match {
-            case proto.Expression.Literal.LiteralTypeCase.INTEGER =>
-              (param.getLiteral.getInteger.asInstanceOf[Object], classOf[Int])
-            case proto.Expression.Literal.LiteralTypeCase.FLOAT =>
-              (param.getLiteral.getFloat.toDouble.asInstanceOf[Object], classOf[Double])
-            case proto.Expression.Literal.LiteralTypeCase.STRING =>
-              (param.getLiteral.getString, classOf[String])
-            case proto.Expression.Literal.LiteralTypeCase.DOUBLE =>
-              (param.getLiteral.getDouble.asInstanceOf[Object], classOf[Double])
-            case proto.Expression.Literal.LiteralTypeCase.BOOLEAN =>
-              (param.getLiteral.getBoolean.asInstanceOf[Object], classOf[Boolean])
-            case other =>
-              throw MlUnsupportedException(s"$other not supported")
-          }
-        } else if (param.hasVector) {
-          val vector = MLUtils.deserializeVector(param.getVector)
-          val vectorType = if (param.getVector.hasDense) {
-            classOf[DenseVector]
-          } else {
-            classOf[SparseVector]
-          }
-          (vector, vectorType)
-        } else if (param.hasMatrix) {
-          val matrix = MLUtils.deserializeMatrix(param.getMatrix)
-          val matrixType = if (param.getMatrix.hasDense) {
-            classOf[DenseMatrix]
-          } else {
-            classOf[SparseMatrix]
-          }
-          (matrix, matrixType)
-        } else {
-          throw MlUnsupportedException(s"$param not supported")
+        val literal = arg.getParam
+        literal.getLiteralTypeCase match {
+          case proto.Expression.Literal.LiteralTypeCase.STRUCT =>
+            val struct = literal.getStruct
+            val schema = DataTypeProtoConverter.toCatalystType(struct.getStructType)
+            if (schema == VectorUDT.sqlType) {
+              (MLUtils.deserializeVector(struct), classOf[Vector])
+            } else if (schema == MatrixUDT.sqlType) {
+              (MLUtils.deserializeMatrix(struct), classOf[Matrix])
+            } else {
+              throw MlUnsupportedException(s"$schema not supported")
+            }
+          case proto.Expression.Literal.LiteralTypeCase.INTEGER =>
+            (literal.getInteger.asInstanceOf[Object], classOf[Int])
+          case proto.Expression.Literal.LiteralTypeCase.FLOAT =>
+            (literal.getFloat.toDouble.asInstanceOf[Object], classOf[Double])
+          case proto.Expression.Literal.LiteralTypeCase.STRING =>
+            (literal.getString, classOf[String])
+          case proto.Expression.Literal.LiteralTypeCase.DOUBLE =>
+            (literal.getDouble.asInstanceOf[Object], classOf[Double])
+          case proto.Expression.Literal.LiteralTypeCase.BOOLEAN =>
+            (literal.getBoolean.asInstanceOf[Object], classOf[Boolean])
+          case other =>
+            throw MlUnsupportedException(s"$other not supported")
         }
       } else if (arg.hasInput) {
         (MLUtils.parseRelationProto(arg.getInput, sessionHolder), classOf[Dataset[_]])
diff --git a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLBackendSuite.scala b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLBackendSuite.scala
index 2e1d3d95c486..f4db4077d1f3 100644
--- a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLBackendSuite.scala
+++ b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLBackendSuite.scala
@@ -51,12 +51,9 @@ class MLBackendSuite extends MLHelper {
       .newBuilder()
       .putParams(
         "maxIter",
-        proto.Param
+        proto.Expression.Literal
           .newBuilder()
-          .setLiteral(
-            proto.Expression.Literal
-              .newBuilder()
-              .setInteger(2))
+          .setInteger(2)
           .build())
   }
 
@@ -95,10 +92,10 @@ class MLBackendSuite extends MLHelper {
         assert(ret.getOperatorInfo.getParams.getParamsMap.containsKey("fakeParam"))
         assert(ret.getOperatorInfo.getParams.getParamsMap.containsKey("maxIter"))
         assert(
-          ret.getOperatorInfo.getParams.getParamsMap.get("maxIter").getLiteral.getInteger
+          ret.getOperatorInfo.getParams.getParamsMap.get("maxIter").getInteger
             == 2)
         assert(
-          ret.getOperatorInfo.getParams.getParamsMap.get("fakeParam").getLiteral.getInteger
+          ret.getOperatorInfo.getParams.getParamsMap.get("fakeParam").getInteger
             == 101010)
       } finally {
         Utils.deleteRecursively(tempDir)
@@ -157,10 +154,10 @@ class MLBackendSuite extends MLHelper {
         assert(ret.getOperatorInfo.getParams.getParamsMap.containsKey("fakeParam"))
         assert(ret.getOperatorInfo.getParams.getParamsMap.containsKey("maxIter"))
         assert(
-          ret.getOperatorInfo.getParams.getParamsMap.get("maxIter").getLiteral.getInteger
+          ret.getOperatorInfo.getParams.getParamsMap.get("maxIter").getInteger
             == 2)
         assert(
-          ret.getOperatorInfo.getParams.getParamsMap.get("fakeParam").getLiteral.getInteger
+          ret.getOperatorInfo.getParams.getParamsMap.get("fakeParam").getInteger
             == 101010)
       } finally {
         Utils.deleteRecursively(tempDir)
diff --git a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLSuite.scala b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLSuite.scala
index 280ced32a3f8..b537f1c702bd 100644
--- a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLSuite.scala
+++ b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLSuite.scala
@@ -72,85 +72,53 @@ class MLSuite extends MLHelper {
     val fakedML = new FakedML
     val params = proto.MlParams
       .newBuilder()
-      .putParams(
-        "boolean",
-        proto.Param
-          .newBuilder()
-          .setLiteral(proto.Expression.Literal.newBuilder().setBoolean(true))
-          .build())
-      .putParams(
-        "double",
-        proto.Param
-          .newBuilder()
-          .setLiteral(proto.Expression.Literal.newBuilder().setDouble(1.0))
-          .build())
-      .putParams(
-        "int",
-        proto.Param
-          .newBuilder()
-          .setLiteral(proto.Expression.Literal.newBuilder().setInteger(10))
-          .build())
-      .putParams(
-        "float",
-        proto.Param
-          .newBuilder()
-          .setLiteral(proto.Expression.Literal.newBuilder().setFloat(10.0f))
-          .build())
+      .putParams("boolean", proto.Expression.Literal.newBuilder().setBoolean(true).build())
+      .putParams("double", proto.Expression.Literal.newBuilder().setDouble(1.0).build())
+      .putParams("int", proto.Expression.Literal.newBuilder().setInteger(10).build())
+      .putParams("float", proto.Expression.Literal.newBuilder().setFloat(10.0f).build())
       .putParams(
         "arrayString",
-        proto.Param
+        proto.Expression.Literal
           .newBuilder()
-          .setLiteral(
-            proto.Expression.Literal
+          .setArray(
+            proto.Expression.Literal.Array
               .newBuilder()
-              .setArray(
-                proto.Expression.Literal.Array
-                  .newBuilder()
-                  .setElementType(proto.DataType
-                    .newBuilder()
-                    .setString(proto.DataType.String.getDefaultInstance)
-                    .build())
-                  .addElements(proto.Expression.Literal.newBuilder().setString("hello"))
-                  .addElements(proto.Expression.Literal.newBuilder().setString("world"))
-                  .build())
+              .setElementType(proto.DataType
+                .newBuilder()
+                .setString(proto.DataType.String.getDefaultInstance)
+                .build())
+              .addElements(proto.Expression.Literal.newBuilder().setString("hello"))
+              .addElements(proto.Expression.Literal.newBuilder().setString("world"))
               .build())
           .build())
       .putParams(
         "arrayInt",
-        proto.Param
+        proto.Expression.Literal
           .newBuilder()
-          .setLiteral(
-            proto.Expression.Literal
+          .setArray(
+            proto.Expression.Literal.Array
               .newBuilder()
-              .setArray(
-                proto.Expression.Literal.Array
-                  .newBuilder()
-                  .setElementType(proto.DataType
-                    .newBuilder()
-                    .setInteger(proto.DataType.Integer.getDefaultInstance)
-                    .build())
-                  .addElements(proto.Expression.Literal.newBuilder().setInteger(1))
-                  .addElements(proto.Expression.Literal.newBuilder().setInteger(2))
-                  .build())
+              .setElementType(proto.DataType
+                .newBuilder()
+                .setInteger(proto.DataType.Integer.getDefaultInstance)
+                .build())
+              .addElements(proto.Expression.Literal.newBuilder().setInteger(1))
+              .addElements(proto.Expression.Literal.newBuilder().setInteger(2))
               .build())
           .build())
       .putParams(
         "arrayDouble",
-        proto.Param
+        proto.Expression.Literal
           .newBuilder()
-          .setLiteral(
-            proto.Expression.Literal
+          .setArray(
+            proto.Expression.Literal.Array
               .newBuilder()
-              .setArray(
-                proto.Expression.Literal.Array
-                  .newBuilder()
-                  .setElementType(proto.DataType
-                    .newBuilder()
-                    .setDouble(proto.DataType.Double.getDefaultInstance)
-                    .build())
-                  .addElements(proto.Expression.Literal.newBuilder().setDouble(11.0))
-                  .addElements(proto.Expression.Literal.newBuilder().setDouble(12.0))
-                  .build())
+              .setElementType(proto.DataType
+                .newBuilder()
+                .setDouble(proto.DataType.Double.getDefaultInstance)
+                .build())
+              .addElements(proto.Expression.Literal.newBuilder().setDouble(11.0))
+              .addElements(proto.Expression.Literal.newBuilder().setDouble(12.0))
               .build())
           .build())
       .build()
@@ -193,11 +161,9 @@ class MLSuite extends MLHelper {
               .newBuilder()
               .putParams(
                 "maxIter",
-                proto.Param
+                proto.Expression.Literal
                   .newBuilder()
-                  .setLiteral(proto.Expression.Literal
-                    .newBuilder()
-                    .setInteger(2))
+                  .setInteger(2)
                   .build())))
       .build()
     val fitResult = MLHandler.handleMlCommand(sessionHolder, fitCommand)
@@ -218,13 +184,13 @@ class MLSuite extends MLHelper {
       // Fetch double attribute
       val interceptCommand = fetchCommand(modelId, "intercept")
       val interceptResult = MLHandler.handleMlCommand(sessionHolder, interceptCommand)
-      assert(interceptResult.getParam.getLiteral.getDouble === lrModel.intercept)
+      assert(interceptResult.getParam.getDouble === lrModel.intercept)
 
       // Fetch Vector attribute
       val coefficientsCommand = fetchCommand(modelId, "coefficients")
       val coefficientsResult = MLHandler.handleMlCommand(sessionHolder, coefficientsCommand)
       val deserializedCoefficients =
-        MLUtils.deserializeVector(coefficientsResult.getParam.getVector)
+        MLUtils.deserializeVector(coefficientsResult.getParam.getStruct)
       assert(deserializedCoefficients === lrModel.coefficients)
 
       // Fetch Matrix attribute
@@ -232,7 +198,7 @@ class MLSuite extends MLHelper {
       val coefficientsMatrixResult =
         MLHandler.handleMlCommand(sessionHolder, coefficientsMatrixCommand)
       val deserializedCoefficientsMatrix =
-        MLUtils.deserializeMatrix(coefficientsMatrixResult.getParam.getMatrix)
+        MLUtils.deserializeMatrix(coefficientsMatrixResult.getParam.getStruct)
       assert(lrModel.coefficientMatrix === deserializedCoefficientsMatrix)
 
       // Predict with sparse vector
@@ -252,7 +218,7 @@ class MLSuite extends MLHelper {
                   .setParam(Serializer.serializeParam(sparseVector)))))
         .build()
       val predictResult = MLHandler.handleMlCommand(sessionHolder, predictCommand)
-      val predictValue = predictResult.getParam.getLiteral.getDouble
+      val predictValue = predictResult.getParam.getDouble
       assert(lrModel.predict(sparseVector) === predictValue)
 
       // The loaded model doesn't have summary
@@ -268,7 +234,7 @@ class MLSuite extends MLHelper {
               .addMethods(proto.Fetch.Method.newBuilder().setMethod("accuracy")))
           .build()
         val accuracyResult = MLHandler.handleMlCommand(sessionHolder, accuracyCommand)
-        assert(lrModel.summary.accuracy === accuracyResult.getParam.getLiteral.getDouble)
+        assert(lrModel.summary.accuracy === accuracyResult.getParam.getDouble)
 
         val weightedFMeasureCommand = proto.MlCommand
           .newBuilder()
@@ -289,7 +255,7 @@ class MLSuite extends MLHelper {
           MLHandler.handleMlCommand(sessionHolder, weightedFMeasureCommand)
         assert(
           lrModel.summary.weightedFMeasure(2.5) ===
-            weightedFMeasureResult.getParam.getLiteral.getDouble)
+            weightedFMeasureResult.getParam.getDouble)
       }
     }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org