You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ue...@apache.org on 2025/02/27 19:21:53 UTC
(spark) branch master updated: [SPARK-51326][CONNECT] Remove LazyExpression proto message
This is an automated email from the ASF dual-hosted git repository.
ueshin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 89e05a88deb9 [SPARK-51326][CONNECT] Remove LazyExpression proto message
89e05a88deb9 is described below
commit 89e05a88deb9bc348f09a4bb09eed8aae9320459
Author: Takuya Ueshin <ue...@databricks.com>
AuthorDate: Thu Feb 27 11:21:43 2025 -0800
[SPARK-51326][CONNECT] Remove LazyExpression proto message
### What changes were proposed in this pull request?
Removes `LazyExpression` proto message.
As any feature using this proto message is not released yet, it's safe to remove it from master and branch-4.0.
### Why are the changes needed?
Discussed offline and found out that Spark Connect should work without it.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
The existing tests should pass.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #50093 from ueshin/issues/SPARK-51326/lazy_expression.
Authored-by: Takuya Ueshin <ue...@databricks.com>
Signed-off-by: Takuya Ueshin <ue...@databricks.com>
---
python/pyspark/sql/connect/column.py | 3 +-
python/pyspark/sql/connect/expressions.py | 19 ---
.../pyspark/sql/connect/proto/expressions_pb2.py | 166 ++++++++++-----------
.../pyspark/sql/connect/proto/expressions_pb2.pyi | 28 ----
.../main/protobuf/spark/connect/expressions.proto | 8 +-
.../spark/sql/connect/columnNodeSupport.scala | 2 +-
.../sql/connect/planner/SparkConnectPlanner.scala | 8 +-
7 files changed, 86 insertions(+), 148 deletions(-)
diff --git a/python/pyspark/sql/connect/column.py b/python/pyspark/sql/connect/column.py
index e6d58aefbf2f..15d943175850 100644
--- a/python/pyspark/sql/connect/column.py
+++ b/python/pyspark/sql/connect/column.py
@@ -44,7 +44,6 @@ from pyspark.sql.connect.expressions import (
Expression,
UnresolvedFunction,
UnresolvedExtractValue,
- LazyExpression,
LiteralExpression,
CaseWhen,
SortOrder,
@@ -459,7 +458,7 @@ class Column(ParentColumn):
return Column(WindowExpression(windowFunction=self._expr, windowSpec=window))
def outer(self) -> ParentColumn:
- return Column(LazyExpression(self._expr))
+ return Column(self._expr)
def isin(self, *cols: Any) -> ParentColumn:
if len(cols) == 1 and isinstance(cols[0], (list, set)):
diff --git a/python/pyspark/sql/connect/expressions.py b/python/pyspark/sql/connect/expressions.py
index 32d176c3e1b8..e5b10be41963 100644
--- a/python/pyspark/sql/connect/expressions.py
+++ b/python/pyspark/sql/connect/expressions.py
@@ -1233,25 +1233,6 @@ class NamedArgumentExpression(Expression):
return f"{self._key} => {self._value}"
-class LazyExpression(Expression):
- def __init__(self, expr: Expression):
- assert isinstance(expr, Expression)
- super().__init__()
- self._expr = expr
-
- def to_plan(self, session: "SparkConnectClient") -> proto.Expression:
- expr = self._create_proto_expression()
- expr.lazy_expression.child.CopyFrom(self._expr.to_plan(session))
- return expr
-
- @property
- def children(self) -> Sequence["Expression"]:
- return [self._expr]
-
- def __repr__(self) -> str:
- return f"lazy({self._expr})"
-
-
class SubqueryExpression(Expression):
def __init__(
self,
diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.py b/python/pyspark/sql/connect/proto/expressions_pb2.py
index 24d2d6157334..f5fdd162a708 100644
--- a/python/pyspark/sql/connect/proto/expressions_pb2.py
+++ b/python/pyspark/sql/connect/proto/expressions_pb2.py
@@ -40,7 +40,7 @@ from pyspark.sql.connect.proto import common_pb2 as spark_dot_connect_dot_common
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
- b'\n\x1fspark/connect/expressions.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x19spark/connect/types.proto\x1a\x1aspark/connect/common.proto"\xbd\x35\n\nExpression\x12\x37\n\x06\x63ommon\x18\x12 \x01(\x0b\x32\x1f.spark.connect.ExpressionCommonR\x06\x63ommon\x12=\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralH\x00R\x07literal\x12\x62\n\x14unresolved_attribute\x18\x02 \x01(\x0b\x32-.spark.connect.Expression.UnresolvedAttributeH\x00R\x13unresolved [...]
+ b'\n\x1fspark/connect/expressions.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x19spark/connect/types.proto\x1a\x1aspark/connect/common.proto"\xf3\x34\n\nExpression\x12\x37\n\x06\x63ommon\x18\x12 \x01(\x0b\x32\x1f.spark.connect.ExpressionCommonR\x06\x63ommon\x12=\n\x07literal\x18\x01 \x01(\x0b\x32!.spark.connect.Expression.LiteralH\x00R\x07literal\x12\x62\n\x14unresolved_attribute\x18\x02 \x01(\x0b\x32-.spark.connect.Expression.UnresolvedAttributeH\x00R\x13unresolved [...]
)
_globals = globals()
@@ -54,87 +54,85 @@ if not _descriptor._USE_C_DESCRIPTORS:
"DESCRIPTOR"
]._serialized_options = b"\n\036org.apache.spark.connect.protoP\001Z\022internal/generated"
_globals["_EXPRESSION"]._serialized_start = 133
- _globals["_EXPRESSION"]._serialized_end = 6978
- _globals["_EXPRESSION_WINDOW"]._serialized_start = 2060
- _globals["_EXPRESSION_WINDOW"]._serialized_end = 2843
- _globals["_EXPRESSION_WINDOW_WINDOWFRAME"]._serialized_start = 2350
- _globals["_EXPRESSION_WINDOW_WINDOWFRAME"]._serialized_end = 2843
- _globals["_EXPRESSION_WINDOW_WINDOWFRAME_FRAMEBOUNDARY"]._serialized_start = 2617
- _globals["_EXPRESSION_WINDOW_WINDOWFRAME_FRAMEBOUNDARY"]._serialized_end = 2762
- _globals["_EXPRESSION_WINDOW_WINDOWFRAME_FRAMETYPE"]._serialized_start = 2764
- _globals["_EXPRESSION_WINDOW_WINDOWFRAME_FRAMETYPE"]._serialized_end = 2843
- _globals["_EXPRESSION_SORTORDER"]._serialized_start = 2846
- _globals["_EXPRESSION_SORTORDER"]._serialized_end = 3271
- _globals["_EXPRESSION_SORTORDER_SORTDIRECTION"]._serialized_start = 3076
- _globals["_EXPRESSION_SORTORDER_SORTDIRECTION"]._serialized_end = 3184
- _globals["_EXPRESSION_SORTORDER_NULLORDERING"]._serialized_start = 3186
- _globals["_EXPRESSION_SORTORDER_NULLORDERING"]._serialized_end = 3271
- _globals["_EXPRESSION_CAST"]._serialized_start = 3274
- _globals["_EXPRESSION_CAST"]._serialized_end = 3589
- _globals["_EXPRESSION_CAST_EVALMODE"]._serialized_start = 3475
- _globals["_EXPRESSION_CAST_EVALMODE"]._serialized_end = 3573
- _globals["_EXPRESSION_LITERAL"]._serialized_start = 3592
- _globals["_EXPRESSION_LITERAL"]._serialized_end = 5577
- _globals["_EXPRESSION_LITERAL_DECIMAL"]._serialized_start = 4526
- _globals["_EXPRESSION_LITERAL_DECIMAL"]._serialized_end = 4643
- _globals["_EXPRESSION_LITERAL_CALENDARINTERVAL"]._serialized_start = 4645
- _globals["_EXPRESSION_LITERAL_CALENDARINTERVAL"]._serialized_end = 4743
- _globals["_EXPRESSION_LITERAL_ARRAY"]._serialized_start = 4746
- _globals["_EXPRESSION_LITERAL_ARRAY"]._serialized_end = 4876
- _globals["_EXPRESSION_LITERAL_MAP"]._serialized_start = 4879
- _globals["_EXPRESSION_LITERAL_MAP"]._serialized_end = 5106
- _globals["_EXPRESSION_LITERAL_STRUCT"]._serialized_start = 5109
- _globals["_EXPRESSION_LITERAL_STRUCT"]._serialized_end = 5238
- _globals["_EXPRESSION_LITERAL_SPECIALIZEDARRAY"]._serialized_start = 5241
- _globals["_EXPRESSION_LITERAL_SPECIALIZEDARRAY"]._serialized_end = 5561
- _globals["_EXPRESSION_UNRESOLVEDATTRIBUTE"]._serialized_start = 5580
- _globals["_EXPRESSION_UNRESOLVEDATTRIBUTE"]._serialized_end = 5766
- _globals["_EXPRESSION_UNRESOLVEDFUNCTION"]._serialized_start = 5769
- _globals["_EXPRESSION_UNRESOLVEDFUNCTION"]._serialized_end = 6027
- _globals["_EXPRESSION_EXPRESSIONSTRING"]._serialized_start = 6029
- _globals["_EXPRESSION_EXPRESSIONSTRING"]._serialized_end = 6079
- _globals["_EXPRESSION_UNRESOLVEDSTAR"]._serialized_start = 6081
- _globals["_EXPRESSION_UNRESOLVEDSTAR"]._serialized_end = 6205
- _globals["_EXPRESSION_UNRESOLVEDREGEX"]._serialized_start = 6207
- _globals["_EXPRESSION_UNRESOLVEDREGEX"]._serialized_end = 6293
- _globals["_EXPRESSION_UNRESOLVEDEXTRACTVALUE"]._serialized_start = 6296
- _globals["_EXPRESSION_UNRESOLVEDEXTRACTVALUE"]._serialized_end = 6428
- _globals["_EXPRESSION_UPDATEFIELDS"]._serialized_start = 6431
- _globals["_EXPRESSION_UPDATEFIELDS"]._serialized_end = 6618
- _globals["_EXPRESSION_ALIAS"]._serialized_start = 6620
- _globals["_EXPRESSION_ALIAS"]._serialized_end = 6740
- _globals["_EXPRESSION_LAMBDAFUNCTION"]._serialized_start = 6743
- _globals["_EXPRESSION_LAMBDAFUNCTION"]._serialized_end = 6901
- _globals["_EXPRESSION_UNRESOLVEDNAMEDLAMBDAVARIABLE"]._serialized_start = 6903
- _globals["_EXPRESSION_UNRESOLVEDNAMEDLAMBDAVARIABLE"]._serialized_end = 6965
- _globals["_EXPRESSIONCOMMON"]._serialized_start = 6980
- _globals["_EXPRESSIONCOMMON"]._serialized_end = 7045
- _globals["_COMMONINLINEUSERDEFINEDFUNCTION"]._serialized_start = 7048
- _globals["_COMMONINLINEUSERDEFINEDFUNCTION"]._serialized_end = 7445
- _globals["_PYTHONUDF"]._serialized_start = 7448
- _globals["_PYTHONUDF"]._serialized_end = 7652
- _globals["_SCALARSCALAUDF"]._serialized_start = 7655
- _globals["_SCALARSCALAUDF"]._serialized_end = 7869
- _globals["_JAVAUDF"]._serialized_start = 7872
- _globals["_JAVAUDF"]._serialized_end = 8021
- _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_start = 8023
- _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_end = 8122
- _globals["_CALLFUNCTION"]._serialized_start = 8124
- _globals["_CALLFUNCTION"]._serialized_end = 8232
- _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_start = 8234
- _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_end = 8326
- _globals["_MERGEACTION"]._serialized_start = 8329
- _globals["_MERGEACTION"]._serialized_end = 8841
- _globals["_MERGEACTION_ASSIGNMENT"]._serialized_start = 8551
- _globals["_MERGEACTION_ASSIGNMENT"]._serialized_end = 8657
- _globals["_MERGEACTION_ACTIONTYPE"]._serialized_start = 8660
- _globals["_MERGEACTION_ACTIONTYPE"]._serialized_end = 8827
- _globals["_LAZYEXPRESSION"]._serialized_start = 8843
- _globals["_LAZYEXPRESSION"]._serialized_end = 8908
- _globals["_SUBQUERYEXPRESSION"]._serialized_start = 8911
- _globals["_SUBQUERYEXPRESSION"]._serialized_end = 9524
- _globals["_SUBQUERYEXPRESSION_TABLEARGOPTIONS"]._serialized_start = 9144
- _globals["_SUBQUERYEXPRESSION_TABLEARGOPTIONS"]._serialized_end = 9378
- _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_start = 9380
- _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_end = 9502
+ _globals["_EXPRESSION"]._serialized_end = 6904
+ _globals["_EXPRESSION_WINDOW"]._serialized_start = 1986
+ _globals["_EXPRESSION_WINDOW"]._serialized_end = 2769
+ _globals["_EXPRESSION_WINDOW_WINDOWFRAME"]._serialized_start = 2276
+ _globals["_EXPRESSION_WINDOW_WINDOWFRAME"]._serialized_end = 2769
+ _globals["_EXPRESSION_WINDOW_WINDOWFRAME_FRAMEBOUNDARY"]._serialized_start = 2543
+ _globals["_EXPRESSION_WINDOW_WINDOWFRAME_FRAMEBOUNDARY"]._serialized_end = 2688
+ _globals["_EXPRESSION_WINDOW_WINDOWFRAME_FRAMETYPE"]._serialized_start = 2690
+ _globals["_EXPRESSION_WINDOW_WINDOWFRAME_FRAMETYPE"]._serialized_end = 2769
+ _globals["_EXPRESSION_SORTORDER"]._serialized_start = 2772
+ _globals["_EXPRESSION_SORTORDER"]._serialized_end = 3197
+ _globals["_EXPRESSION_SORTORDER_SORTDIRECTION"]._serialized_start = 3002
+ _globals["_EXPRESSION_SORTORDER_SORTDIRECTION"]._serialized_end = 3110
+ _globals["_EXPRESSION_SORTORDER_NULLORDERING"]._serialized_start = 3112
+ _globals["_EXPRESSION_SORTORDER_NULLORDERING"]._serialized_end = 3197
+ _globals["_EXPRESSION_CAST"]._serialized_start = 3200
+ _globals["_EXPRESSION_CAST"]._serialized_end = 3515
+ _globals["_EXPRESSION_CAST_EVALMODE"]._serialized_start = 3401
+ _globals["_EXPRESSION_CAST_EVALMODE"]._serialized_end = 3499
+ _globals["_EXPRESSION_LITERAL"]._serialized_start = 3518
+ _globals["_EXPRESSION_LITERAL"]._serialized_end = 5503
+ _globals["_EXPRESSION_LITERAL_DECIMAL"]._serialized_start = 4452
+ _globals["_EXPRESSION_LITERAL_DECIMAL"]._serialized_end = 4569
+ _globals["_EXPRESSION_LITERAL_CALENDARINTERVAL"]._serialized_start = 4571
+ _globals["_EXPRESSION_LITERAL_CALENDARINTERVAL"]._serialized_end = 4669
+ _globals["_EXPRESSION_LITERAL_ARRAY"]._serialized_start = 4672
+ _globals["_EXPRESSION_LITERAL_ARRAY"]._serialized_end = 4802
+ _globals["_EXPRESSION_LITERAL_MAP"]._serialized_start = 4805
+ _globals["_EXPRESSION_LITERAL_MAP"]._serialized_end = 5032
+ _globals["_EXPRESSION_LITERAL_STRUCT"]._serialized_start = 5035
+ _globals["_EXPRESSION_LITERAL_STRUCT"]._serialized_end = 5164
+ _globals["_EXPRESSION_LITERAL_SPECIALIZEDARRAY"]._serialized_start = 5167
+ _globals["_EXPRESSION_LITERAL_SPECIALIZEDARRAY"]._serialized_end = 5487
+ _globals["_EXPRESSION_UNRESOLVEDATTRIBUTE"]._serialized_start = 5506
+ _globals["_EXPRESSION_UNRESOLVEDATTRIBUTE"]._serialized_end = 5692
+ _globals["_EXPRESSION_UNRESOLVEDFUNCTION"]._serialized_start = 5695
+ _globals["_EXPRESSION_UNRESOLVEDFUNCTION"]._serialized_end = 5953
+ _globals["_EXPRESSION_EXPRESSIONSTRING"]._serialized_start = 5955
+ _globals["_EXPRESSION_EXPRESSIONSTRING"]._serialized_end = 6005
+ _globals["_EXPRESSION_UNRESOLVEDSTAR"]._serialized_start = 6007
+ _globals["_EXPRESSION_UNRESOLVEDSTAR"]._serialized_end = 6131
+ _globals["_EXPRESSION_UNRESOLVEDREGEX"]._serialized_start = 6133
+ _globals["_EXPRESSION_UNRESOLVEDREGEX"]._serialized_end = 6219
+ _globals["_EXPRESSION_UNRESOLVEDEXTRACTVALUE"]._serialized_start = 6222
+ _globals["_EXPRESSION_UNRESOLVEDEXTRACTVALUE"]._serialized_end = 6354
+ _globals["_EXPRESSION_UPDATEFIELDS"]._serialized_start = 6357
+ _globals["_EXPRESSION_UPDATEFIELDS"]._serialized_end = 6544
+ _globals["_EXPRESSION_ALIAS"]._serialized_start = 6546
+ _globals["_EXPRESSION_ALIAS"]._serialized_end = 6666
+ _globals["_EXPRESSION_LAMBDAFUNCTION"]._serialized_start = 6669
+ _globals["_EXPRESSION_LAMBDAFUNCTION"]._serialized_end = 6827
+ _globals["_EXPRESSION_UNRESOLVEDNAMEDLAMBDAVARIABLE"]._serialized_start = 6829
+ _globals["_EXPRESSION_UNRESOLVEDNAMEDLAMBDAVARIABLE"]._serialized_end = 6891
+ _globals["_EXPRESSIONCOMMON"]._serialized_start = 6906
+ _globals["_EXPRESSIONCOMMON"]._serialized_end = 6971
+ _globals["_COMMONINLINEUSERDEFINEDFUNCTION"]._serialized_start = 6974
+ _globals["_COMMONINLINEUSERDEFINEDFUNCTION"]._serialized_end = 7371
+ _globals["_PYTHONUDF"]._serialized_start = 7374
+ _globals["_PYTHONUDF"]._serialized_end = 7578
+ _globals["_SCALARSCALAUDF"]._serialized_start = 7581
+ _globals["_SCALARSCALAUDF"]._serialized_end = 7795
+ _globals["_JAVAUDF"]._serialized_start = 7798
+ _globals["_JAVAUDF"]._serialized_end = 7947
+ _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_start = 7949
+ _globals["_TYPEDAGGREGATEEXPRESSION"]._serialized_end = 8048
+ _globals["_CALLFUNCTION"]._serialized_start = 8050
+ _globals["_CALLFUNCTION"]._serialized_end = 8158
+ _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_start = 8160
+ _globals["_NAMEDARGUMENTEXPRESSION"]._serialized_end = 8252
+ _globals["_MERGEACTION"]._serialized_start = 8255
+ _globals["_MERGEACTION"]._serialized_end = 8767
+ _globals["_MERGEACTION_ASSIGNMENT"]._serialized_start = 8477
+ _globals["_MERGEACTION_ASSIGNMENT"]._serialized_end = 8583
+ _globals["_MERGEACTION_ACTIONTYPE"]._serialized_start = 8586
+ _globals["_MERGEACTION_ACTIONTYPE"]._serialized_end = 8753
+ _globals["_SUBQUERYEXPRESSION"]._serialized_start = 8770
+ _globals["_SUBQUERYEXPRESSION"]._serialized_end = 9383
+ _globals["_SUBQUERYEXPRESSION_TABLEARGOPTIONS"]._serialized_start = 9003
+ _globals["_SUBQUERYEXPRESSION_TABLEARGOPTIONS"]._serialized_end = 9237
+ _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_start = 9239
+ _globals["_SUBQUERYEXPRESSION_SUBQUERYTYPE"]._serialized_end = 9361
# @@protoc_insertion_point(module_scope)
diff --git a/python/pyspark/sql/connect/proto/expressions_pb2.pyi b/python/pyspark/sql/connect/proto/expressions_pb2.pyi
index 714be5227b20..f6aada59a2d8 100644
--- a/python/pyspark/sql/connect/proto/expressions_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/expressions_pb2.pyi
@@ -1289,7 +1289,6 @@ class Expression(google.protobuf.message.Message):
NAMED_ARGUMENT_EXPRESSION_FIELD_NUMBER: builtins.int
MERGE_ACTION_FIELD_NUMBER: builtins.int
TYPED_AGGREGATE_EXPRESSION_FIELD_NUMBER: builtins.int
- LAZY_EXPRESSION_FIELD_NUMBER: builtins.int
SUBQUERY_EXPRESSION_FIELD_NUMBER: builtins.int
EXTENSION_FIELD_NUMBER: builtins.int
@property
@@ -1335,8 +1334,6 @@ class Expression(google.protobuf.message.Message):
@property
def typed_aggregate_expression(self) -> global___TypedAggregateExpression: ...
@property
- def lazy_expression(self) -> global___LazyExpression: ...
- @property
def subquery_expression(self) -> global___SubqueryExpression: ...
@property
def extension(self) -> google.protobuf.any_pb2.Any:
@@ -1367,7 +1364,6 @@ class Expression(google.protobuf.message.Message):
named_argument_expression: global___NamedArgumentExpression | None = ...,
merge_action: global___MergeAction | None = ...,
typed_aggregate_expression: global___TypedAggregateExpression | None = ...,
- lazy_expression: global___LazyExpression | None = ...,
subquery_expression: global___SubqueryExpression | None = ...,
extension: google.protobuf.any_pb2.Any | None = ...,
) -> None: ...
@@ -1392,8 +1388,6 @@ class Expression(google.protobuf.message.Message):
b"extension",
"lambda_function",
b"lambda_function",
- "lazy_expression",
- b"lazy_expression",
"literal",
b"literal",
"merge_action",
@@ -1445,8 +1439,6 @@ class Expression(google.protobuf.message.Message):
b"extension",
"lambda_function",
b"lambda_function",
- "lazy_expression",
- b"lazy_expression",
"literal",
b"literal",
"merge_action",
@@ -1500,7 +1492,6 @@ class Expression(google.protobuf.message.Message):
"named_argument_expression",
"merge_action",
"typed_aggregate_expression",
- "lazy_expression",
"subquery_expression",
"extension",
]
@@ -1931,25 +1922,6 @@ class MergeAction(google.protobuf.message.Message):
global___MergeAction = MergeAction
-class LazyExpression(google.protobuf.message.Message):
- DESCRIPTOR: google.protobuf.descriptor.Descriptor
-
- CHILD_FIELD_NUMBER: builtins.int
- @property
- def child(self) -> global___Expression:
- """(Required) The expression to be marked as lazy."""
- def __init__(
- self,
- *,
- child: global___Expression | None = ...,
- ) -> None: ...
- def HasField(
- self, field_name: typing_extensions.Literal["child", b"child"]
- ) -> builtins.bool: ...
- def ClearField(self, field_name: typing_extensions.Literal["child", b"child"]) -> None: ...
-
-global___LazyExpression = LazyExpression
-
class SubqueryExpression(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
diff --git a/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto b/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto
index 77fb3d423a81..4dcaa9a40142 100644
--- a/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto
+++ b/sql/connect/common/src/main/protobuf/spark/connect/expressions.proto
@@ -52,8 +52,7 @@ message Expression {
NamedArgumentExpression named_argument_expression = 17;
MergeAction merge_action = 19;
TypedAggregateExpression typed_aggregate_expression = 20;
- LazyExpression lazy_expression = 21;
- SubqueryExpression subquery_expression = 22;
+ SubqueryExpression subquery_expression = 21;
// This field is used to mark extensions to the protocol. When plugins generate arbitrary
// relations they can add them here. During the planning the correct resolution is done.
@@ -474,11 +473,6 @@ message MergeAction {
}
}
-message LazyExpression {
- // (Required) The expression to be marked as lazy.
- Expression child = 1;
-}
-
message SubqueryExpression {
// (Required) The ID of the corresponding connect plan.
int64 plan_id = 1;
diff --git a/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/columnNodeSupport.scala b/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/columnNodeSupport.scala
index 5e15eb318999..54f45a434826 100644
--- a/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/columnNodeSupport.scala
+++ b/sql/connect/common/src/main/scala/org/apache/spark/sql/connect/columnNodeSupport.scala
@@ -216,7 +216,7 @@ object ColumnNodeToProtoConverter extends (ColumnNode => proto.Expression) {
}
case LazyExpression(child, _) =>
- builder.getLazyExpressionBuilder.setChild(apply(child, e))
+ return apply(child, e)
case SubqueryExpressionNode(relation, subqueryType, _) =>
val b = builder.getSubqueryExpressionBuilder
diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
index 7228bdfa7a90..24fc1275d482 100644
--- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
+++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/planner/SparkConnectPlanner.scala
@@ -46,7 +46,7 @@ import org.apache.spark.internal.LogKeys.{DATAFRAME_ID, SESSION_ID}
import org.apache.spark.resource.{ExecutorResourceRequest, ResourceProfile, TaskResourceProfile, TaskResourceRequest}
import org.apache.spark.sql.{Column, Encoders, ForeachWriter, Observation, Row}
import org.apache.spark.sql.catalyst.{expressions, AliasIdentifier, FunctionIdentifier, QueryPlanningTracker}
-import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, GlobalTempView, LazyExpression, LocalTempView, MultiAlias, NameParameterizedQuery, PosParameterizedQuery, UnresolvedAlias, UnresolvedAttribute, UnresolvedDataFrameStar, UnresolvedDeserializer, UnresolvedExtractValue, UnresolvedFunction, UnresolvedPlanId, UnresolvedRegex, UnresolvedRelation, UnresolvedStar, UnresolvedStarWithColumns, UnresolvedStarWithColumnsRenames, UnresolvedSubqueryColumnAliases, UnresolvedTableValuedFunc [...]
+import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, GlobalTempView, LocalTempView, MultiAlias, NameParameterizedQuery, PosParameterizedQuery, UnresolvedAlias, UnresolvedAttribute, UnresolvedDataFrameStar, UnresolvedDeserializer, UnresolvedExtractValue, UnresolvedFunction, UnresolvedPlanId, UnresolvedRegex, UnresolvedRelation, UnresolvedStar, UnresolvedStarWithColumns, UnresolvedStarWithColumnsRenames, UnresolvedSubqueryColumnAliases, UnresolvedTableValuedFunction, Unresolved [...]
import org.apache.spark.sql.catalyst.encoders.{encoderFor, AgnosticEncoder, ExpressionEncoder, RowEncoder}
import org.apache.spark.sql.catalyst.encoders.AgnosticEncoders.{ProductEncoder, RowEncoder => AgnosticRowEncoder, StringEncoder, UnboundRowEncoder}
import org.apache.spark.sql.catalyst.expressions._
@@ -1694,8 +1694,6 @@ class SparkConnectPlanner(
transformMergeAction(exp.getMergeAction)
case proto.Expression.ExprTypeCase.TYPED_AGGREGATE_EXPRESSION =>
transformTypedAggregateExpression(exp.getTypedAggregateExpression, baseRelationOpt)
- case proto.Expression.ExprTypeCase.LAZY_EXPRESSION =>
- transformLazyExpression(exp.getLazyExpression)
case proto.Expression.ExprTypeCase.SUBQUERY_EXPRESSION =>
transformSubqueryExpression(exp.getSubqueryExpression)
case _ =>
@@ -3939,10 +3937,6 @@ class SparkConnectPlanner(
}
}
- private def transformLazyExpression(getLazyExpression: proto.LazyExpression): Expression = {
- LazyExpression(transformExpression(getLazyExpression.getChild))
- }
-
private def transformSubqueryExpression(
getSubqueryExpression: proto.SubqueryExpression): Expression = {
val planId = getSubqueryExpression.getPlanId
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org