You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2026/08/07 20:48:22 UTC
(spark-connect-swift) branch main updated: [SPARK-58652] Update `Spark Connect`-generated `Swift` source code with `branch-4.3 (2026-08-07)`
This is an automated email from the ASF dual-hosted git repository.
dongjoon-hyun pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/spark-connect-swift.git
The following commit(s) were added to refs/heads/main by this push:
new 2d2c053 [SPARK-58652] Update `Spark Connect`-generated `Swift` source code with `branch-4.3 (2026-08-07)`
2d2c053 is described below
commit 2d2c053aab41b63f17cde11b2095b74ddd4142be
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Fri Aug 7 13:48:15 2026 -0700
[SPARK-58652] Update `Spark Connect`-generated `Swift` source code with `branch-4.3 (2026-08-07)`
### What changes were proposed in this pull request?
This PR updates the `Spark Connect`-generated Swift source code by regenerating with `branch-4.3 (2026-08-07)`
- apache/spark#56300
- apache/spark#56909
- apache/spark#57412
### Why are the changes needed?
To keep the generated Swift source code in sync with Apache Spark `branch-4.3` protobuf definitions.
```
$ git clone -b branch-4.3 https://github.com/apache/spark.git
$ cd spark/sql/connect/common/src/main/protobuf/
$ protoc --swift_out=. spark/connect/*.proto
$ protoc --grpc-swift_out=. spark/connect/*.proto
// Remove empty GRPC files
$ cd spark/connect
$ grep 'This file contained no services' * | awk -F: '{print $1}' | xargs rm
```
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5
Closes #490 from dongjoon-hyun/SPARK-58652.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
Sources/SparkConnect/DataType.swift | 3 +
Sources/SparkConnect/expressions.pb.swift | 207 +++++++++++++++++++++++++++++-
Sources/SparkConnect/pipelines.pb.swift | 26 +++-
Sources/SparkConnect/relations.pb.swift | 141 +++++++++++++++++++-
Sources/SparkConnect/types.pb.swift | 186 ++++++++++++++++++++++++++-
5 files changed, 558 insertions(+), 5 deletions(-)
diff --git a/Sources/SparkConnect/DataType.swift b/Sources/SparkConnect/DataType.swift
index 5611204..28b93af 100644
--- a/Sources/SparkConnect/DataType.swift
+++ b/Sources/SparkConnect/DataType.swift
@@ -344,6 +344,9 @@ extension DataType {
self = .timestamp
case .timestampNtz:
self = .timestampNtz
+ case .timestampNtzNanos, .timestampLtzNanos:
+ // Nanosecond-precision timestamps are not supported by this client yet.
+ throw SparkConnectError.InvalidType
case .time(let time):
self = .time(precision: time.hasPrecision ? time.precision : 6)
case .calendarInterval:
diff --git a/Sources/SparkConnect/expressions.pb.swift b/Sources/SparkConnect/expressions.pb.swift
index e5b71dc..36314be 100644
--- a/Sources/SparkConnect/expressions.pb.swift
+++ b/Sources/SparkConnect/expressions.pb.swift
@@ -881,6 +881,24 @@ nonisolated struct Spark_Connect_Expression: @unchecked Sendable {
set {literalType = .time(newValue)}
}
+ /// Nanosecond-capable timestamp literals (precision 7..9). NTZ and LTZ are distinct
+ /// arms so the literal kind is self-describing.
+ var timestampNtzNanos: Spark_Connect_Expression.Literal.TimestampNTZNanos {
+ get {
+ if case .timestampNtzNanos(let v)? = literalType {return v}
+ return Spark_Connect_Expression.Literal.TimestampNTZNanos()
+ }
+ set {literalType = .timestampNtzNanos(newValue)}
+ }
+
+ var timestampLtzNanos: Spark_Connect_Expression.Literal.TimestampLTZNanos {
+ get {
+ if case .timestampLtzNanos(let v)? = literalType {return v}
+ return Spark_Connect_Expression.Literal.TimestampLTZNanos()
+ }
+ set {literalType = .timestampLtzNanos(newValue)}
+ }
+
/// Data type information for the literal.
/// This field is required only in the root literal message for null values or
/// for data types (e.g., array, map, or struct) with non-trivial information.
@@ -923,6 +941,10 @@ nonisolated struct Spark_Connect_Expression: @unchecked Sendable {
case `struct`(Spark_Connect_Expression.Literal.Struct)
case specializedArray(Spark_Connect_Expression.Literal.SpecializedArray)
case time(Spark_Connect_Expression.Literal.Time)
+ /// Nanosecond-capable timestamp literals (precision 7..9). NTZ and LTZ are distinct
+ /// arms so the literal kind is self-describing.
+ case timestampNtzNanos(Spark_Connect_Expression.Literal.TimestampNTZNanos)
+ case timestampLtzNanos(Spark_Connect_Expression.Literal.TimestampLTZNanos)
}
@@ -1180,6 +1202,67 @@ nonisolated struct Spark_Connect_Expression: @unchecked Sendable {
fileprivate var _precision: Int32? = nil
}
+ /// A TIMESTAMP_NTZ literal with nanosecond-capable precision. The physical value is carried
+ /// as microseconds since the UNIX epoch plus the extra nanoseconds within that microsecond,
+ /// because a single int64 of nanoseconds cannot span the supported year range.
+ nonisolated struct TimestampNTZNanos: Sendable {
+ // SwiftProtobuf.Message conformance is added in an extension below. See the
+ // `Message` and `Message+*Additions` files in the SwiftProtobuf library for
+ // methods supported on all messages.
+
+ /// Microseconds since the UNIX epoch (without timezone information).
+ var epochMicros: Int64 = 0
+
+ /// Additional nanoseconds within epoch_micros, in [0, 999].
+ var nanosWithinMicro: Int32 = 0
+
+ /// Number of fractional-second digits (7, 8, or 9). If omitted, defaults to 9 (nanoseconds).
+ var precision: Int32 {
+ get {_precision ?? 0}
+ set {_precision = newValue}
+ }
+ /// Returns true if `precision` has been explicitly set.
+ var hasPrecision: Bool {self._precision != nil}
+ /// Clears the value of `precision`. Subsequent reads from it will return its default value.
+ mutating func clearPrecision() {self._precision = nil}
+
+ var unknownFields = SwiftProtobuf.UnknownStorage()
+
+ init() {}
+
+ fileprivate var _precision: Int32? = nil
+ }
+
+ /// A TIMESTAMP_LTZ literal with nanosecond-capable precision. See TimestampNTZNanos for the
+ /// rationale behind the two-component physical value.
+ nonisolated struct TimestampLTZNanos: Sendable {
+ // SwiftProtobuf.Message conformance is added in an extension below. See the
+ // `Message` and `Message+*Additions` files in the SwiftProtobuf library for
+ // methods supported on all messages.
+
+ /// Microseconds since the UNIX epoch.
+ var epochMicros: Int64 = 0
+
+ /// Additional nanoseconds within epoch_micros, in [0, 999].
+ var nanosWithinMicro: Int32 = 0
+
+ /// Number of fractional-second digits (7, 8, or 9). If omitted, defaults to 9 (nanoseconds).
+ var precision: Int32 {
+ get {_precision ?? 0}
+ set {_precision = newValue}
+ }
+ /// Returns true if `precision` has been explicitly set.
+ var hasPrecision: Bool {self._precision != nil}
+ /// Clears the value of `precision`. Subsequent reads from it will return its default value.
+ mutating func clearPrecision() {self._precision = nil}
+
+ var unknownFields = SwiftProtobuf.UnknownStorage()
+
+ init() {}
+
+ fileprivate var _precision: Int32? = nil
+ }
+
init() {}
fileprivate var _dataType: Spark_Connect_DataType? = nil
@@ -2998,7 +3081,7 @@ nonisolated extension Spark_Connect_Expression.Cast.EvalMode: SwiftProtobuf._Pro
nonisolated extension Spark_Connect_Expression.Literal: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding {
static let protoMessageName: String = Spark_Connect_Expression.protoMessageName + ".Literal"
- static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}null\0\u{1}binary\0\u{1}boolean\0\u{1}byte\0\u{1}short\0\u{1}integer\0\u{1}long\0\u{2}\u{3}float\0\u{1}double\0\u{1}decimal\0\u{1}string\0\u{2}\u{3}date\0\u{1}timestamp\0\u{3}timestamp_ntz\0\u{3}calendar_interval\0\u{3}year_month_interval\0\u{3}day_time_interval\0\u{1}array\0\u{1}map\0\u{1}struct\0\u{3}specialized_array\0\u{1}time\0\u{4}J\u{1}data_type\0\u{c}\u{1b}\u{1}\u{c}\u{1c}\u{1}")
+ static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}null\0\u{1}binary\0\u{1}boolean\0\u{1}byte\0\u{1}short\0\u{1}integer\0\u{1}long\0\u{2}\u{3}float\0\u{1}double\0\u{1}decimal\0\u{1}string\0\u{2}\u{3}date\0\u{1}timestamp\0\u{3}timestamp_ntz\0\u{3}calendar_interval\0\u{3}year_month_interval\0\u{3}day_time_interval\0\u{1}array\0\u{1}map\0\u{1}struct\0\u{3}specialized_array\0\u{1}time\0\u{4}\u{3}timestamp_ntz_nanos\0\u{3}timestamp_ltz_nanos\0\u{4}F\u{1}data_type\0\u{c} [...]
mutating func decodeMessage<D: SwiftProtobuf.Decoder>(decoder: inout D) throws {
while let fieldNumber = try decoder.nextFieldNumber() {
@@ -3222,6 +3305,32 @@ nonisolated extension Spark_Connect_Expression.Literal: SwiftProtobuf.Message, S
self.literalType = .time(v)
}
}()
+ case 29: try {
+ var v: Spark_Connect_Expression.Literal.TimestampNTZNanos?
+ var hadOneofValue = false
+ if let current = self.literalType {
+ hadOneofValue = true
+ if case .timestampNtzNanos(let m) = current {v = m}
+ }
+ try decoder.decodeSingularMessageField(value: &v)
+ if let v = v {
+ if hadOneofValue {try decoder.handleConflictingOneOf()}
+ self.literalType = .timestampNtzNanos(v)
+ }
+ }()
+ case 30: try {
+ var v: Spark_Connect_Expression.Literal.TimestampLTZNanos?
+ var hadOneofValue = false
+ if let current = self.literalType {
+ hadOneofValue = true
+ if case .timestampLtzNanos(let m) = current {v = m}
+ }
+ try decoder.decodeSingularMessageField(value: &v)
+ if let v = v {
+ if hadOneofValue {try decoder.handleConflictingOneOf()}
+ self.literalType = .timestampLtzNanos(v)
+ }
+ }()
case 100: try { try decoder.decodeSingularMessageField(value: &self._dataType) }()
default: break
}
@@ -3322,6 +3431,14 @@ nonisolated extension Spark_Connect_Expression.Literal: SwiftProtobuf.Message, S
guard case .time(let v)? = self.literalType else { preconditionFailure() }
try visitor.visitSingularMessageField(value: v, fieldNumber: 26)
}()
+ case .timestampNtzNanos?: try {
+ guard case .timestampNtzNanos(let v)? = self.literalType else { preconditionFailure() }
+ try visitor.visitSingularMessageField(value: v, fieldNumber: 29)
+ }()
+ case .timestampLtzNanos?: try {
+ guard case .timestampLtzNanos(let v)? = self.literalType else { preconditionFailure() }
+ try visitor.visitSingularMessageField(value: v, fieldNumber: 30)
+ }()
case nil: break
}
try { if let v = self._dataType {
@@ -3723,6 +3840,94 @@ nonisolated extension Spark_Connect_Expression.Literal.Time: SwiftProtobuf.Messa
}
}
+nonisolated extension Spark_Connect_Expression.Literal.TimestampNTZNanos: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding {
+ static let protoMessageName: String = Spark_Connect_Expression.Literal.protoMessageName + ".TimestampNTZNanos"
+ static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{3}epoch_micros\0\u{3}nanos_within_micro\0\u{1}precision\0")
+
+ mutating func decodeMessage<D: SwiftProtobuf.Decoder>(decoder: inout D) throws {
+ while let fieldNumber = try decoder.nextFieldNumber() {
+ // The use of inline closures is to circumvent an issue where the compiler
+ // allocates stack space for every case branch when no optimizations are
+ // enabled. https://github.com/apple/swift-protobuf/issues/1034
+ switch fieldNumber {
+ case 1: try { try decoder.decodeSingularInt64Field(value: &self.epochMicros) }()
+ case 2: try { try decoder.decodeSingularInt32Field(value: &self.nanosWithinMicro) }()
+ case 3: try { try decoder.decodeSingularInt32Field(value: &self._precision) }()
+ default: break
+ }
+ }
+ }
+
+ func traverse<V: SwiftProtobuf.Visitor>(visitor: inout V) throws {
+ // The use of inline closures is to circumvent an issue where the compiler
+ // allocates stack space for every if/case branch local when no optimizations
+ // are enabled. https://github.com/apple/swift-protobuf/issues/1034 and
+ // https://github.com/apple/swift-protobuf/issues/1182
+ if self.epochMicros != 0 {
+ try visitor.visitSingularInt64Field(value: self.epochMicros, fieldNumber: 1)
+ }
+ if self.nanosWithinMicro != 0 {
+ try visitor.visitSingularInt32Field(value: self.nanosWithinMicro, fieldNumber: 2)
+ }
+ try { if let v = self._precision {
+ try visitor.visitSingularInt32Field(value: v, fieldNumber: 3)
+ } }()
+ try unknownFields.traverse(visitor: &visitor)
+ }
+
+ static func ==(lhs: Spark_Connect_Expression.Literal.TimestampNTZNanos, rhs: Spark_Connect_Expression.Literal.TimestampNTZNanos) -> Bool {
+ if lhs.epochMicros != rhs.epochMicros {return false}
+ if lhs.nanosWithinMicro != rhs.nanosWithinMicro {return false}
+ if lhs._precision != rhs._precision {return false}
+ if lhs.unknownFields != rhs.unknownFields {return false}
+ return true
+ }
+}
+
+nonisolated extension Spark_Connect_Expression.Literal.TimestampLTZNanos: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding {
+ static let protoMessageName: String = Spark_Connect_Expression.Literal.protoMessageName + ".TimestampLTZNanos"
+ static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{3}epoch_micros\0\u{3}nanos_within_micro\0\u{1}precision\0")
+
+ mutating func decodeMessage<D: SwiftProtobuf.Decoder>(decoder: inout D) throws {
+ while let fieldNumber = try decoder.nextFieldNumber() {
+ // The use of inline closures is to circumvent an issue where the compiler
+ // allocates stack space for every case branch when no optimizations are
+ // enabled. https://github.com/apple/swift-protobuf/issues/1034
+ switch fieldNumber {
+ case 1: try { try decoder.decodeSingularInt64Field(value: &self.epochMicros) }()
+ case 2: try { try decoder.decodeSingularInt32Field(value: &self.nanosWithinMicro) }()
+ case 3: try { try decoder.decodeSingularInt32Field(value: &self._precision) }()
+ default: break
+ }
+ }
+ }
+
+ func traverse<V: SwiftProtobuf.Visitor>(visitor: inout V) throws {
+ // The use of inline closures is to circumvent an issue where the compiler
+ // allocates stack space for every if/case branch local when no optimizations
+ // are enabled. https://github.com/apple/swift-protobuf/issues/1034 and
+ // https://github.com/apple/swift-protobuf/issues/1182
+ if self.epochMicros != 0 {
+ try visitor.visitSingularInt64Field(value: self.epochMicros, fieldNumber: 1)
+ }
+ if self.nanosWithinMicro != 0 {
+ try visitor.visitSingularInt32Field(value: self.nanosWithinMicro, fieldNumber: 2)
+ }
+ try { if let v = self._precision {
+ try visitor.visitSingularInt32Field(value: v, fieldNumber: 3)
+ } }()
+ try unknownFields.traverse(visitor: &visitor)
+ }
+
+ static func ==(lhs: Spark_Connect_Expression.Literal.TimestampLTZNanos, rhs: Spark_Connect_Expression.Literal.TimestampLTZNanos) -> Bool {
+ if lhs.epochMicros != rhs.epochMicros {return false}
+ if lhs.nanosWithinMicro != rhs.nanosWithinMicro {return false}
+ if lhs._precision != rhs._precision {return false}
+ if lhs.unknownFields != rhs.unknownFields {return false}
+ return true
+ }
+}
+
nonisolated extension Spark_Connect_Expression.UnresolvedAttribute: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding {
static let protoMessageName: String = Spark_Connect_Expression.protoMessageName + ".UnresolvedAttribute"
static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{3}unparsed_identifier\0\u{3}plan_id\0\u{3}is_metadata_column\0")
diff --git a/Sources/SparkConnect/pipelines.pb.swift b/Sources/SparkConnect/pipelines.pb.swift
index 72e3efe..f71e573 100644
--- a/Sources/SparkConnect/pipelines.pb.swift
+++ b/Sources/SparkConnect/pipelines.pb.swift
@@ -569,6 +569,7 @@ nonisolated struct Spark_Connect_PipelineCommand: Sendable {
typealias RawValue = Int
case unspecified // = 0
case scdType1 // = 1
+ case scdType2 // = 2
case UNRECOGNIZED(Int)
init() {
@@ -579,6 +580,7 @@ nonisolated struct Spark_Connect_PipelineCommand: Sendable {
switch rawValue {
case 0: self = .unspecified
case 1: self = .scdType1
+ case 2: self = .scdType2
default: self = .UNRECOGNIZED(rawValue)
}
}
@@ -587,6 +589,7 @@ nonisolated struct Spark_Connect_PipelineCommand: Sendable {
switch self {
case .unspecified: return 0
case .scdType1: return 1
+ case .scdType2: return 2
case .UNRECOGNIZED(let i): return i
}
}
@@ -595,6 +598,7 @@ nonisolated struct Spark_Connect_PipelineCommand: Sendable {
static let allCases: [Spark_Connect_PipelineCommand.DefineFlow.SCDType] = [
.unspecified,
.scdType1,
+ .scdType2,
]
}
@@ -681,6 +685,14 @@ nonisolated struct Spark_Connect_PipelineCommand: Sendable {
/// SCD Type for target table.
var storedAsScdType: Spark_Connect_PipelineCommand.DefineFlow.SCDType = .unspecified
+ /// SCD2 only. Columns whose value change opens a new history record. When empty, every
+ /// eligible selected user column is tracked.
+ var trackHistoryColumnList: [Spark_Connect_Expression] = []
+
+ /// SCD2 only. Columns excluded from history tracking. Mutually exclusive with
+ /// track_history_column_list.
+ var trackHistoryExceptColumnList: [Spark_Connect_Expression] = []
+
/// Subset of columns to ignore null in updates.
var ignoreNullUpdatesColumnList: [Spark_Connect_Expression] = []
@@ -1976,7 +1988,7 @@ nonisolated extension Spark_Connect_PipelineCommand.DefineFlow: SwiftProtobuf.Me
}
nonisolated extension Spark_Connect_PipelineCommand.DefineFlow.SCDType: SwiftProtobuf._ProtoNameProviding {
- static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{2}\0SCD_TYPE_UNSPECIFIED\0\u{1}SCD_TYPE_1\0")
+ static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{2}\0SCD_TYPE_UNSPECIFIED\0\u{1}SCD_TYPE_1\0\u{1}SCD_TYPE_2\0")
}
nonisolated extension Spark_Connect_PipelineCommand.DefineFlow.WriteRelationFlowDetails: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding {
@@ -2015,7 +2027,7 @@ nonisolated extension Spark_Connect_PipelineCommand.DefineFlow.WriteRelationFlow
nonisolated extension Spark_Connect_PipelineCommand.DefineFlow.AutoCdcFlowDetails: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding {
static let protoMessageName: String = Spark_Connect_PipelineCommand.DefineFlow.protoMessageName + ".AutoCdcFlowDetails"
- static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}source\0\u{1}keys\0\u{3}sequence_by\0\u{4}\u{3}apply_as_deletes\0\u{3}apply_as_truncates\0\u{3}column_list\0\u{3}except_column_list\0\u{3}stored_as_scd_type\0\u{4}\u{4}ignore_null_updates_column_list\0\u{3}ignore_null_updates_except_column_list\0")
+ static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}source\0\u{1}keys\0\u{3}sequence_by\0\u{4}\u{3}apply_as_deletes\0\u{3}apply_as_truncates\0\u{3}column_list\0\u{3}except_column_list\0\u{3}stored_as_scd_type\0\u{3}track_history_column_list\0\u{3}track_history_except_column_list\0\u{4}\u{2}ignore_null_updates_column_list\0\u{3}ignore_null_updates_except_column_list\0")
mutating func decodeMessage<D: SwiftProtobuf.Decoder>(decoder: inout D) throws {
while let fieldNumber = try decoder.nextFieldNumber() {
@@ -2031,6 +2043,8 @@ nonisolated extension Spark_Connect_PipelineCommand.DefineFlow.AutoCdcFlowDetail
case 8: try { try decoder.decodeRepeatedMessageField(value: &self.columnList) }()
case 9: try { try decoder.decodeRepeatedMessageField(value: &self.exceptColumnList) }()
case 10: try { try decoder.decodeSingularEnumField(value: &self.storedAsScdType) }()
+ case 11: try { try decoder.decodeRepeatedMessageField(value: &self.trackHistoryColumnList) }()
+ case 12: try { try decoder.decodeRepeatedMessageField(value: &self.trackHistoryExceptColumnList) }()
case 14: try { try decoder.decodeRepeatedMessageField(value: &self.ignoreNullUpdatesColumnList) }()
case 15: try { try decoder.decodeRepeatedMessageField(value: &self.ignoreNullUpdatesExceptColumnList) }()
default: break
@@ -2067,6 +2081,12 @@ nonisolated extension Spark_Connect_PipelineCommand.DefineFlow.AutoCdcFlowDetail
if self.storedAsScdType != .unspecified {
try visitor.visitSingularEnumField(value: self.storedAsScdType, fieldNumber: 10)
}
+ if !self.trackHistoryColumnList.isEmpty {
+ try visitor.visitRepeatedMessageField(value: self.trackHistoryColumnList, fieldNumber: 11)
+ }
+ if !self.trackHistoryExceptColumnList.isEmpty {
+ try visitor.visitRepeatedMessageField(value: self.trackHistoryExceptColumnList, fieldNumber: 12)
+ }
if !self.ignoreNullUpdatesColumnList.isEmpty {
try visitor.visitRepeatedMessageField(value: self.ignoreNullUpdatesColumnList, fieldNumber: 14)
}
@@ -2085,6 +2105,8 @@ nonisolated extension Spark_Connect_PipelineCommand.DefineFlow.AutoCdcFlowDetail
if lhs.columnList != rhs.columnList {return false}
if lhs.exceptColumnList != rhs.exceptColumnList {return false}
if lhs.storedAsScdType != rhs.storedAsScdType {return false}
+ if lhs.trackHistoryColumnList != rhs.trackHistoryColumnList {return false}
+ if lhs.trackHistoryExceptColumnList != rhs.trackHistoryExceptColumnList {return false}
if lhs.ignoreNullUpdatesColumnList != rhs.ignoreNullUpdatesColumnList {return false}
if lhs.ignoreNullUpdatesExceptColumnList != rhs.ignoreNullUpdatesExceptColumnList {return false}
if lhs.unknownFields != rhs.unknownFields {return false}
diff --git a/Sources/SparkConnect/relations.pb.swift b/Sources/SparkConnect/relations.pb.swift
index d57bd55..427519d 100644
--- a/Sources/SparkConnect/relations.pb.swift
+++ b/Sources/SparkConnect/relations.pb.swift
@@ -432,6 +432,14 @@ nonisolated struct Spark_Connect_Relation: @unchecked Sendable {
set {_uniqueStorage()._relType = .nearestByJoin(newValue)}
}
+ var zip: Spark_Connect_Zip {
+ get {
+ if case .zip(let v)? = _storage._relType {return v}
+ return Spark_Connect_Zip()
+ }
+ set {_uniqueStorage()._relType = .zip(newValue)}
+ }
+
/// NA functions
var fillNa: Spark_Connect_NAFill {
get {
@@ -607,6 +615,7 @@ nonisolated struct Spark_Connect_Relation: @unchecked Sendable {
case chunkedCachedLocalRelation(Spark_Connect_ChunkedCachedLocalRelation)
case relationChanges(Spark_Connect_RelationChanges)
case nearestByJoin(Spark_Connect_NearestByJoin)
+ case zip(Spark_Connect_Zip)
/// NA functions
case fillNa(Spark_Connect_NAFill)
case dropNa(Spark_Connect_NADrop)
@@ -3868,13 +3877,49 @@ nonisolated struct Spark_Connect_NearestByJoin: @unchecked Sendable {
fileprivate var _storage = _StorageClass.defaultInstance
}
+/// Relation of type [[Zip]].
+///
+/// Combines the columns of two DataFrames side-by-side. Both DataFrames must produce the same
+/// canonicalized plan after stripping outer Project chains.
+nonisolated struct Spark_Connect_Zip: @unchecked Sendable {
+ // SwiftProtobuf.Message conformance is added in an extension below. See the
+ // `Message` and `Message+*Additions` files in the SwiftProtobuf library for
+ // methods supported on all messages.
+
+ /// (Required) Left input relation.
+ var left: Spark_Connect_Relation {
+ get {_storage._left ?? Spark_Connect_Relation()}
+ set {_uniqueStorage()._left = newValue}
+ }
+ /// Returns true if `left` has been explicitly set.
+ var hasLeft: Bool {_storage._left != nil}
+ /// Clears the value of `left`. Subsequent reads from it will return its default value.
+ mutating func clearLeft() {_uniqueStorage()._left = nil}
+
+ /// (Required) Right input relation.
+ var right: Spark_Connect_Relation {
+ get {_storage._right ?? Spark_Connect_Relation()}
+ set {_uniqueStorage()._right = newValue}
+ }
+ /// Returns true if `right` has been explicitly set.
+ var hasRight: Bool {_storage._right != nil}
+ /// Clears the value of `right`. Subsequent reads from it will return its default value.
+ mutating func clearRight() {_uniqueStorage()._right = nil}
+
+ var unknownFields = SwiftProtobuf.UnknownStorage()
+
+ init() {}
+
+ fileprivate var _storage = _StorageClass.defaultInstance
+}
+
// MARK: - Code below here is support for the SwiftProtobuf runtime.
fileprivate nonisolated let _protobuf_package = "spark.connect"
nonisolated extension Spark_Connect_Relation: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding {
static let protoMessageName: String = _protobuf_package + ".Relation"
- static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}common\0\u{1}read\0\u{1}project\0\u{1}filter\0\u{1}join\0\u{3}set_op\0\u{1}sort\0\u{1}limit\0\u{1}aggregate\0\u{1}sql\0\u{3}local_relation\0\u{1}sample\0\u{1}offset\0\u{1}deduplicate\0\u{1}range\0\u{3}subquery_alias\0\u{1}repartition\0\u{3}to_df\0\u{3}with_columns_renamed\0\u{3}show_string\0\u{1}drop\0\u{1}tail\0\u{3}with_columns\0\u{1}hint\0\u{1}unpivot\0\u{3}to_schema\0\u{3}repartition_by_expression\0\u{3}map_par [...]
+ static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}common\0\u{1}read\0\u{1}project\0\u{1}filter\0\u{1}join\0\u{3}set_op\0\u{1}sort\0\u{1}limit\0\u{1}aggregate\0\u{1}sql\0\u{3}local_relation\0\u{1}sample\0\u{1}offset\0\u{1}deduplicate\0\u{1}range\0\u{3}subquery_alias\0\u{1}repartition\0\u{3}to_df\0\u{3}with_columns_renamed\0\u{3}show_string\0\u{1}drop\0\u{1}tail\0\u{3}with_columns\0\u{1}hint\0\u{1}unpivot\0\u{3}to_schema\0\u{3}repartition_by_expression\0\u{3}map_par [...]
fileprivate class _StorageClass {
var _common: Spark_Connect_RelationCommon? = nil
@@ -4508,6 +4553,19 @@ nonisolated extension Spark_Connect_Relation: SwiftProtobuf.Message, SwiftProtob
_storage._relType = .nearestByJoin(v)
}
}()
+ case 48: try {
+ var v: Spark_Connect_Zip?
+ var hadOneofValue = false
+ if let current = _storage._relType {
+ hadOneofValue = true
+ if case .zip(let m) = current {v = m}
+ }
+ try decoder.decodeSingularMessageField(value: &v)
+ if let v = v {
+ if hadOneofValue {try decoder.handleConflictingOneOf()}
+ _storage._relType = .zip(v)
+ }
+ }()
case 90: try {
var v: Spark_Connect_NAFill?
var hadOneofValue = false
@@ -4903,6 +4961,10 @@ nonisolated extension Spark_Connect_Relation: SwiftProtobuf.Message, SwiftProtob
guard case .nearestByJoin(let v)? = _storage._relType else { preconditionFailure() }
try visitor.visitSingularMessageField(value: v, fieldNumber: 47)
}()
+ case .zip?: try {
+ guard case .zip(let v)? = _storage._relType else { preconditionFailure() }
+ try visitor.visitSingularMessageField(value: v, fieldNumber: 48)
+ }()
case .fillNa?: try {
guard case .fillNa(let v)? = _storage._relType else { preconditionFailure() }
try visitor.visitSingularMessageField(value: v, fieldNumber: 90)
@@ -10519,3 +10581,80 @@ nonisolated extension Spark_Connect_NearestByJoin: SwiftProtobuf.Message, SwiftP
return true
}
}
+
+nonisolated extension Spark_Connect_Zip: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding {
+ static let protoMessageName: String = _protobuf_package + ".Zip"
+ static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}left\0\u{1}right\0")
+
+ fileprivate class _StorageClass {
+ var _left: Spark_Connect_Relation? = nil
+ var _right: Spark_Connect_Relation? = nil
+
+ // This property is used as the initial default value for new instances of the type.
+ // The type itself is protecting the reference to its storage via CoW semantics.
+ // This will force a copy to be made of this reference when the first mutation occurs;
+ // hence, it is safe to mark this as `nonisolated(unsafe)`.
+ static nonisolated(unsafe) let defaultInstance = _StorageClass()
+
+ private init() {}
+
+ init(copying source: _StorageClass) {
+ _left = source._left
+ _right = source._right
+ }
+ }
+
+ fileprivate mutating func _uniqueStorage() -> _StorageClass {
+ if !isKnownUniquelyReferenced(&_storage) {
+ _storage = _StorageClass(copying: _storage)
+ }
+ return _storage
+ }
+
+ mutating func decodeMessage<D: SwiftProtobuf.Decoder>(decoder: inout D) throws {
+ _ = _uniqueStorage()
+ try withExtendedLifetime(_storage) { (_storage: _StorageClass) in
+ while let fieldNumber = try decoder.nextFieldNumber() {
+ // The use of inline closures is to circumvent an issue where the compiler
+ // allocates stack space for every case branch when no optimizations are
+ // enabled. https://github.com/apple/swift-protobuf/issues/1034
+ switch fieldNumber {
+ case 1: try { try decoder.decodeSingularMessageField(value: &_storage._left) }()
+ case 2: try { try decoder.decodeSingularMessageField(value: &_storage._right) }()
+ default: break
+ }
+ }
+ }
+ }
+
+ func traverse<V: SwiftProtobuf.Visitor>(visitor: inout V) throws {
+ try withExtendedLifetime(_storage) { (_storage: _StorageClass) in
+ // The use of inline closures is to circumvent an issue where the compiler
+ // allocates stack space for every if/case branch local when no optimizations
+ // are enabled. https://github.com/apple/swift-protobuf/issues/1034 and
+ // https://github.com/apple/swift-protobuf/issues/1182
+ try { if let v = _storage._left {
+ try visitor.visitSingularMessageField(value: v, fieldNumber: 1)
+ } }()
+ try { if let v = _storage._right {
+ try visitor.visitSingularMessageField(value: v, fieldNumber: 2)
+ } }()
+ }
+ try unknownFields.traverse(visitor: &visitor)
+ }
+
+ static func ==(lhs: Spark_Connect_Zip, rhs: Spark_Connect_Zip) -> Bool {
+ if lhs._storage !== rhs._storage {
+ let storagesAreEqual: Bool = withExtendedLifetime((lhs._storage, rhs._storage)) { (_args: (_StorageClass, _StorageClass)) in
+ let _storage = _args.0
+ let rhs_storage = _args.1
+ if _storage._left != rhs_storage._left {return false}
+ if _storage._right != rhs_storage._right {return false}
+ return true
+ }
+ if !storagesAreEqual {return false}
+ }
+ if lhs.unknownFields != rhs.unknownFields {return false}
+ return true
+ }
+}
diff --git a/Sources/SparkConnect/types.pb.swift b/Sources/SparkConnect/types.pb.swift
index 57af744..e2950c9 100644
--- a/Sources/SparkConnect/types.pb.swift
+++ b/Sources/SparkConnect/types.pb.swift
@@ -280,6 +280,24 @@ nonisolated struct Spark_Connect_DataType: @unchecked Sendable {
set {_uniqueStorage()._kind = .time(newValue)}
}
+ /// Nanosecond-capable timestamp types (precision 7..9). NTZ and LTZ are distinct kinds
+ /// even though their physical value is identical, mirroring timestamp vs timestamp_ntz.
+ var timestampNtzNanos: Spark_Connect_DataType.TimestampNTZNanos {
+ get {
+ if case .timestampNtzNanos(let v)? = _storage._kind {return v}
+ return Spark_Connect_DataType.TimestampNTZNanos()
+ }
+ set {_uniqueStorage()._kind = .timestampNtzNanos(newValue)}
+ }
+
+ var timestampLtzNanos: Spark_Connect_DataType.TimestampLTZNanos {
+ get {
+ if case .timestampLtzNanos(let v)? = _storage._kind {return v}
+ return Spark_Connect_DataType.TimestampLTZNanos()
+ }
+ set {_uniqueStorage()._kind = .timestampLtzNanos(newValue)}
+ }
+
var unknownFields = SwiftProtobuf.UnknownStorage()
nonisolated enum OneOf_Kind: Equatable, Sendable {
@@ -319,6 +337,10 @@ nonisolated struct Spark_Connect_DataType: @unchecked Sendable {
/// UnparsedDataType
case unparsed(Spark_Connect_DataType.Unparsed)
case time(Spark_Connect_DataType.Time)
+ /// Nanosecond-capable timestamp types (precision 7..9). NTZ and LTZ are distinct kinds
+ /// even though their physical value is identical, mirroring timestamp vs timestamp_ntz.
+ case timestampNtzNanos(Spark_Connect_DataType.TimestampNTZNanos)
+ case timestampLtzNanos(Spark_Connect_DataType.TimestampLTZNanos)
}
@@ -503,6 +525,56 @@ nonisolated struct Spark_Connect_DataType: @unchecked Sendable {
fileprivate var _precision: Int32? = nil
}
+ /// Timestamp without time zone with nanosecond-capable fractional-second precision.
+ nonisolated struct TimestampNTZNanos: Sendable {
+ // SwiftProtobuf.Message conformance is added in an extension below. See the
+ // `Message` and `Message+*Additions` files in the SwiftProtobuf library for
+ // methods supported on all messages.
+
+ /// Number of fractional-second digits. Valid values are 7, 8, and 9. If omitted, defaults to 9.
+ var precision: Int32 {
+ get {_precision ?? 0}
+ set {_precision = newValue}
+ }
+ /// Returns true if `precision` has been explicitly set.
+ var hasPrecision: Bool {self._precision != nil}
+ /// Clears the value of `precision`. Subsequent reads from it will return its default value.
+ mutating func clearPrecision() {self._precision = nil}
+
+ var typeVariationReference: UInt32 = 0
+
+ var unknownFields = SwiftProtobuf.UnknownStorage()
+
+ init() {}
+
+ fileprivate var _precision: Int32? = nil
+ }
+
+ /// Timestamp with local time zone with nanosecond-capable fractional-second precision.
+ nonisolated struct TimestampLTZNanos: Sendable {
+ // SwiftProtobuf.Message conformance is added in an extension below. See the
+ // `Message` and `Message+*Additions` files in the SwiftProtobuf library for
+ // methods supported on all messages.
+
+ /// Number of fractional-second digits. Valid values are 7, 8, and 9. If omitted, defaults to 9.
+ var precision: Int32 {
+ get {_precision ?? 0}
+ set {_precision = newValue}
+ }
+ /// Returns true if `precision` has been explicitly set.
+ var hasPrecision: Bool {self._precision != nil}
+ /// Clears the value of `precision`. Subsequent reads from it will return its default value.
+ mutating func clearPrecision() {self._precision = nil}
+
+ var typeVariationReference: UInt32 = 0
+
+ var unknownFields = SwiftProtobuf.UnknownStorage()
+
+ init() {}
+
+ fileprivate var _precision: Int32? = nil
+ }
+
nonisolated struct CalendarInterval: Sendable {
// SwiftProtobuf.Message conformance is added in an extension below. See the
// `Message` and `Message+*Additions` files in the SwiftProtobuf library for
@@ -884,7 +956,7 @@ fileprivate nonisolated let _protobuf_package = "spark.connect"
nonisolated extension Spark_Connect_DataType: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding {
static let protoMessageName: String = _protobuf_package + ".DataType"
- static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}null\0\u{1}binary\0\u{1}boolean\0\u{1}byte\0\u{1}short\0\u{1}integer\0\u{1}long\0\u{1}float\0\u{1}double\0\u{1}decimal\0\u{1}string\0\u{1}char\0\u{3}var_char\0\u{1}date\0\u{1}timestamp\0\u{3}timestamp_ntz\0\u{3}calendar_interval\0\u{3}year_month_interval\0\u{3}day_time_interval\0\u{1}array\0\u{1}struct\0\u{1}map\0\u{1}udt\0\u{1}unparsed\0\u{1}variant\0\u{1}geometry\0\u{1}geography\0\u{1}time\0")
+ static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}null\0\u{1}binary\0\u{1}boolean\0\u{1}byte\0\u{1}short\0\u{1}integer\0\u{1}long\0\u{1}float\0\u{1}double\0\u{1}decimal\0\u{1}string\0\u{1}char\0\u{3}var_char\0\u{1}date\0\u{1}timestamp\0\u{3}timestamp_ntz\0\u{3}calendar_interval\0\u{3}year_month_interval\0\u{3}day_time_interval\0\u{1}array\0\u{1}struct\0\u{1}map\0\u{1}udt\0\u{1}unparsed\0\u{1}variant\0\u{1}geometry\0\u{1}geography\0\u{1}time\0\u{3}timestamp_ntz_nan [...]
fileprivate class _StorageClass {
var _kind: Spark_Connect_DataType.OneOf_Kind?
@@ -1281,6 +1353,32 @@ nonisolated extension Spark_Connect_DataType: SwiftProtobuf.Message, SwiftProtob
_storage._kind = .time(v)
}
}()
+ case 29: try {
+ var v: Spark_Connect_DataType.TimestampNTZNanos?
+ var hadOneofValue = false
+ if let current = _storage._kind {
+ hadOneofValue = true
+ if case .timestampNtzNanos(let m) = current {v = m}
+ }
+ try decoder.decodeSingularMessageField(value: &v)
+ if let v = v {
+ if hadOneofValue {try decoder.handleConflictingOneOf()}
+ _storage._kind = .timestampNtzNanos(v)
+ }
+ }()
+ case 30: try {
+ var v: Spark_Connect_DataType.TimestampLTZNanos?
+ var hadOneofValue = false
+ if let current = _storage._kind {
+ hadOneofValue = true
+ if case .timestampLtzNanos(let m) = current {v = m}
+ }
+ try decoder.decodeSingularMessageField(value: &v)
+ if let v = v {
+ if hadOneofValue {try decoder.handleConflictingOneOf()}
+ _storage._kind = .timestampLtzNanos(v)
+ }
+ }()
default: break
}
}
@@ -1406,6 +1504,14 @@ nonisolated extension Spark_Connect_DataType: SwiftProtobuf.Message, SwiftProtob
guard case .time(let v)? = _storage._kind else { preconditionFailure() }
try visitor.visitSingularMessageField(value: v, fieldNumber: 28)
}()
+ case .timestampNtzNanos?: try {
+ guard case .timestampNtzNanos(let v)? = _storage._kind else { preconditionFailure() }
+ try visitor.visitSingularMessageField(value: v, fieldNumber: 29)
+ }()
+ case .timestampLtzNanos?: try {
+ guard case .timestampLtzNanos(let v)? = _storage._kind else { preconditionFailure() }
+ try visitor.visitSingularMessageField(value: v, fieldNumber: 30)
+ }()
case nil: break
}
}
@@ -1861,6 +1967,84 @@ nonisolated extension Spark_Connect_DataType.Time: SwiftProtobuf.Message, SwiftP
}
}
+nonisolated extension Spark_Connect_DataType.TimestampNTZNanos: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding {
+ static let protoMessageName: String = Spark_Connect_DataType.protoMessageName + ".TimestampNTZNanos"
+ static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}precision\0\u{3}type_variation_reference\0")
+
+ mutating func decodeMessage<D: SwiftProtobuf.Decoder>(decoder: inout D) throws {
+ while let fieldNumber = try decoder.nextFieldNumber() {
+ // The use of inline closures is to circumvent an issue where the compiler
+ // allocates stack space for every case branch when no optimizations are
+ // enabled. https://github.com/apple/swift-protobuf/issues/1034
+ switch fieldNumber {
+ case 1: try { try decoder.decodeSingularInt32Field(value: &self._precision) }()
+ case 2: try { try decoder.decodeSingularUInt32Field(value: &self.typeVariationReference) }()
+ default: break
+ }
+ }
+ }
+
+ func traverse<V: SwiftProtobuf.Visitor>(visitor: inout V) throws {
+ // The use of inline closures is to circumvent an issue where the compiler
+ // allocates stack space for every if/case branch local when no optimizations
+ // are enabled. https://github.com/apple/swift-protobuf/issues/1034 and
+ // https://github.com/apple/swift-protobuf/issues/1182
+ try { if let v = self._precision {
+ try visitor.visitSingularInt32Field(value: v, fieldNumber: 1)
+ } }()
+ if self.typeVariationReference != 0 {
+ try visitor.visitSingularUInt32Field(value: self.typeVariationReference, fieldNumber: 2)
+ }
+ try unknownFields.traverse(visitor: &visitor)
+ }
+
+ static func ==(lhs: Spark_Connect_DataType.TimestampNTZNanos, rhs: Spark_Connect_DataType.TimestampNTZNanos) -> Bool {
+ if lhs._precision != rhs._precision {return false}
+ if lhs.typeVariationReference != rhs.typeVariationReference {return false}
+ if lhs.unknownFields != rhs.unknownFields {return false}
+ return true
+ }
+}
+
+nonisolated extension Spark_Connect_DataType.TimestampLTZNanos: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding {
+ static let protoMessageName: String = Spark_Connect_DataType.protoMessageName + ".TimestampLTZNanos"
+ static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{1}precision\0\u{3}type_variation_reference\0")
+
+ mutating func decodeMessage<D: SwiftProtobuf.Decoder>(decoder: inout D) throws {
+ while let fieldNumber = try decoder.nextFieldNumber() {
+ // The use of inline closures is to circumvent an issue where the compiler
+ // allocates stack space for every case branch when no optimizations are
+ // enabled. https://github.com/apple/swift-protobuf/issues/1034
+ switch fieldNumber {
+ case 1: try { try decoder.decodeSingularInt32Field(value: &self._precision) }()
+ case 2: try { try decoder.decodeSingularUInt32Field(value: &self.typeVariationReference) }()
+ default: break
+ }
+ }
+ }
+
+ func traverse<V: SwiftProtobuf.Visitor>(visitor: inout V) throws {
+ // The use of inline closures is to circumvent an issue where the compiler
+ // allocates stack space for every if/case branch local when no optimizations
+ // are enabled. https://github.com/apple/swift-protobuf/issues/1034 and
+ // https://github.com/apple/swift-protobuf/issues/1182
+ try { if let v = self._precision {
+ try visitor.visitSingularInt32Field(value: v, fieldNumber: 1)
+ } }()
+ if self.typeVariationReference != 0 {
+ try visitor.visitSingularUInt32Field(value: self.typeVariationReference, fieldNumber: 2)
+ }
+ try unknownFields.traverse(visitor: &visitor)
+ }
+
+ static func ==(lhs: Spark_Connect_DataType.TimestampLTZNanos, rhs: Spark_Connect_DataType.TimestampLTZNanos) -> Bool {
+ if lhs._precision != rhs._precision {return false}
+ if lhs.typeVariationReference != rhs.typeVariationReference {return false}
+ if lhs.unknownFields != rhs.unknownFields {return false}
+ return true
+ }
+}
+
nonisolated extension Spark_Connect_DataType.CalendarInterval: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding {
static let protoMessageName: String = Spark_Connect_DataType.protoMessageName + ".CalendarInterval"
static let _protobuf_nameMap = SwiftProtobuf._NameMap(bytecode: "\0\u{3}type_variation_reference\0")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org