You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2026/05/07 08:32:45 UTC

(spark) branch SPARK-34591 created (now 3ab5b2ef7d72)

This is an automated email from the ASF dual-hosted git repository.

WeichenXu123 pushed a change to branch SPARK-34591
in repository https://gitbox.apache.org/repos/asf/spark.git


      at 3ab5b2ef7d72 address conflicts

This branch includes the following new commits:

     new 86aa0c20c451 merge master
     new 3ab5b2ef7d72 address conflicts

The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


(spark) 02/02: address conflicts

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

WeichenXu123 pushed a commit to branch SPARK-34591
in repository https://gitbox.apache.org/repos/asf/spark.git

commit 3ab5b2ef7d72a28cdde52121c9aec0f193b4c477
Author: Weichen Xu <we...@databricks.com>
AuthorDate: Thu May 7 16:32:17 2026 +0800

    address conflicts
    
    Signed-off-by: Weichen Xu <we...@databricks.com>
---
 .../apache/spark/ml/tree/impl/RandomForest.scala   |  38 +-------
 .../spark/ml/tree/impl/RandomForestSuite.scala     | 106 ---------------------
 python/pyspark/ml/classification.py                |  71 ++------------
 python/pyspark/ml/tree.py                          |   5 -
 4 files changed, 10 insertions(+), 210 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala
index 9813a351e7ee..c3a16ab3dddd 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/impl/RandomForest.scala
@@ -124,14 +124,9 @@ private[spark] object RandomForest extends Logging with Serializable {
       featureSubsetStrategy: String,
       seed: Long,
       instr: Option[Instrumentation],
-<<<<<<< HEAD
-      parentUID: Option[String] = None): Array[DecisionTreeModel] = {
-=======
-      prune: Boolean = true, // exposed for testing only, real trees are always pruned
       parentUID: Option[String] = None,
       earlyStopModelSizeThresholdInBytes: Long = 0): Array[DecisionTreeModel] = {
     lastEarlyStoppedModelSize = 0
->>>>>>> master
     val timer = new TimeTracker()
     timer.start("total")
 
@@ -178,13 +173,7 @@ private[spark] object RandomForest extends Logging with Serializable {
       // At first, all the rows belong to the root nodes (node Id == 1).
       nodeIds = baggedInput.map { _ => Array.fill(numTrees)(1) }
       nodeIdCheckpointer = new PeriodicRDDCheckpointer[Array[Int]](
-<<<<<<< HEAD
-        strategy.getCheckpointInterval,
-        sc,
-        StorageLevel.MEMORY_AND_DISK)
-=======
         strategy.getCheckpointInterval(), sc, StorageLevel.MEMORY_AND_DISK)
->>>>>>> master
       nodeIdCheckpointer.update(nodeIds)
     }
 
@@ -245,7 +234,7 @@ private[spark] object RandomForest extends Logging with Serializable {
       timer.stop("findBestSplits")
 
       if (earlyStopModelSizeThresholdInBytes > 0) {
-        val nodes = topNodes.map(_.toNode(prune))
+        val nodes = topNodes.map(_.toNode(strategy.pruneTree))
         val estimatedSize = SizeEstimator.estimate(nodes)
         if (estimatedSize > earlyStopModelSizeThresholdInBytes){
           earlyStop = true
@@ -278,16 +267,11 @@ private[spark] object RandomForest extends Logging with Serializable {
       case Some(uid) =>
         if (strategy.algo == OldAlgo.Classification) {
           topNodes.map { rootNode =>
-<<<<<<< HEAD
             new DecisionTreeClassificationModel(
               uid,
               rootNode.toNode(strategy.pruneTree),
               numFeatures,
               strategy.getNumClasses)
-=======
-            new DecisionTreeClassificationModel(uid, rootNode.toNode(prune), numFeatures,
-              strategy.getNumClasses())
->>>>>>> master
           }
         } else {
           topNodes.map { rootNode =>
@@ -297,15 +281,10 @@ private[spark] object RandomForest extends Logging with Serializable {
       case None =>
         if (strategy.algo == OldAlgo.Classification) {
           topNodes.map { rootNode =>
-<<<<<<< HEAD
             new DecisionTreeClassificationModel(
               rootNode.toNode(strategy.pruneTree),
               numFeatures,
               strategy.getNumClasses)
-=======
-            new DecisionTreeClassificationModel(rootNode.toNode(prune), numFeatures,
-              strategy.getNumClasses())
->>>>>>> master
           }
         } else {
           topNodes.map(rootNode =>
@@ -368,7 +347,6 @@ private[spark] object RandomForest extends Logging with Serializable {
       .persist(StorageLevel.MEMORY_AND_DISK)
       .setName("bagged tree points")
 
-<<<<<<< HEAD
     val trees = runBagged(
       baggedInput = baggedInput,
       metadata = metadata,
@@ -378,13 +356,8 @@ private[spark] object RandomForest extends Logging with Serializable {
       featureSubsetStrategy = featureSubsetStrategy,
       seed = seed,
       instr = instr,
-      parentUID = parentUID)
-=======
-    val trees = runBagged(baggedInput = baggedInput, metadata = metadata, bcSplits = bcSplits,
-      strategy = strategy, numTrees = numTrees, featureSubsetStrategy = featureSubsetStrategy,
-      seed = seed, instr = instr, prune = prune, parentUID = parentUID,
+      parentUID = parentUID,
       earlyStopModelSizeThresholdInBytes = earlyStopModelSizeThresholdInBytes)
->>>>>>> master
 
     baggedInput.unpersist()
     bcSplits.destroy()
@@ -1460,17 +1433,10 @@ private[spark] object RandomForest extends Logging with Serializable {
     }
     if (memUsage > maxMemoryUsage) {
       // If maxMemoryUsage is 0, we should still allow splitting 1 node.
-<<<<<<< HEAD
-      logWarning(
-        s"Tree learning is using approximately $memUsage bytes per iteration, which" +
-          s" exceeds requested limit maxMemoryUsage=$maxMemoryUsage. This allows splitting" +
-          s" $numNodesInGroup nodes in this iteration.")
-=======
       logWarning(log"Tree learning is using approximately ${MDC(MEMORY_SIZE, memUsage)} " +
         log"bytes per iteration, which exceeds requested limit " +
         log"maxMemoryUsage=${MDC(MAX_MEMORY_SIZE, maxMemoryUsage)}. This allows splitting " +
         log"${MDC(NUM_NODES, numNodesInGroup)} nodes in this iteration.")
->>>>>>> master
     }
     // Convert mutable maps to immutable ones.
     val nodesForGroup: Map[Int, Array[LearningNode]] =
diff --git a/mllib/src/test/scala/org/apache/spark/ml/tree/impl/RandomForestSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/tree/impl/RandomForestSuite.scala
index 3632b723dd8f..a614e9b3e94f 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/tree/impl/RandomForestSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/tree/impl/RandomForestSuite.scala
@@ -59,20 +59,9 @@ class RandomForestSuite extends SparkFunSuite with MLlibTestSparkContext {
   test("Binary classification with binary (ordered) categorical features: split calculation") {
     val arr = OldDTSuite.generateCategoricalDataPoints().map(_.asML.toInstance)
     assert(arr.length === 1000)
-<<<<<<< HEAD
-    val rdd = sc.parallelize(arr)
-    val strategy = new OldStrategy(
-      OldAlgo.Classification,
-      Gini,
-      maxDepth = 2,
-      numClasses = 2,
-      maxBins = 100,
-      categoricalFeaturesInfo = Map(0 -> 2, 1 -> 2))
-=======
     val rdd = sc.parallelize(arr.toImmutableArraySeq)
     val strategy = new OldStrategy(OldAlgo.Classification, Gini, maxDepth = 2, numClasses = 2,
       maxBins = 100, categoricalFeaturesInfo = Map(0 -> 2, 1 -> 2))
->>>>>>> master
 
     val metadata = DecisionTreeMetadata.buildMetadata(rdd, strategy)
     val splits = RandomForest.findSplits(rdd, metadata, seed = 42)
@@ -88,20 +77,9 @@ class RandomForestSuite extends SparkFunSuite with MLlibTestSparkContext {
       " with no samples for one category: split calculation") {
     val arr = OldDTSuite.generateCategoricalDataPoints().map(_.asML.toInstance)
     assert(arr.length === 1000)
-<<<<<<< HEAD
-    val rdd = sc.parallelize(arr)
-    val strategy = new OldStrategy(
-      OldAlgo.Classification,
-      Gini,
-      maxDepth = 2,
-      numClasses = 2,
-      maxBins = 100,
-      categoricalFeaturesInfo = Map(0 -> 3, 1 -> 3))
-=======
     val rdd = sc.parallelize(arr.toImmutableArraySeq)
     val strategy = new OldStrategy(OldAlgo.Classification, Gini, maxDepth = 2, numClasses = 2,
       maxBins = 100, categoricalFeaturesInfo = Map(0 -> 3, 1 -> 3))
->>>>>>> master
 
     val metadata = DecisionTreeMetadata.buildMetadata(rdd, strategy)
     assert(!metadata.isUnordered(featureIndex = 0))
@@ -115,33 +93,12 @@ class RandomForestSuite extends SparkFunSuite with MLlibTestSparkContext {
   test("find splits for a continuous feature") {
     // find splits for normal case
     {
-<<<<<<< HEAD
-      val fakeMetadata = new DecisionTreeMetadata(
-        1,
-        200000,
-        200000.0,
-        0,
-        0,
-        Map(),
-        Set(),
-        Array(6),
-        Gini,
-        QuantileStrategy.Sort,
-        0,
-        0,
-        0.0,
-        0.0,
-        0,
-        0)
-      val featureSamples = Array.fill(10000)((1.0, math.random)).filter(_._2 != 0.0)
-=======
       val fakeMetadata = new DecisionTreeMetadata(1, 200000, 200000.0, 0, 0,
         Map(), Set(),
         Array(6), Gini, QuantileStrategy.Sort,
         0, 0, 0.0, 0.0, 0, 0
       )
       val featureSamples = Array.fill(10000)((1.0, math.random())).filter(_._2 != 0.0)
->>>>>>> master
       val splits = RandomForest.findSplitsForContinuousFeature(featureSamples, fakeMetadata, 0)
       assert(splits.length === 5)
       assert(fakeMetadata.numSplits(0) === 5)
@@ -273,32 +230,6 @@ class RandomForestSuite extends SparkFunSuite with MLlibTestSparkContext {
 
     // find splits for arbitrarily scaled data
     {
-<<<<<<< HEAD
-      val fakeMetadata = new DecisionTreeMetadata(
-        1,
-        0,
-        0.0,
-        0,
-        0,
-        Map(),
-        Set(),
-        Array(6),
-        Gini,
-        QuantileStrategy.Sort,
-        0,
-        0,
-        0.0,
-        0.0,
-        0,
-        0)
-      val featureSamplesUnitWeight = Array.fill(10)((1.0, math.random))
-      val featureSamplesSmallWeight = featureSamplesUnitWeight.map {
-        case (w, x) => (w * 0.001, x)
-      }
-      val featureSamplesLargeWeight = featureSamplesUnitWeight.map {
-        case (w, x) => (w * 1000, x)
-      }
-=======
       val fakeMetadata = new DecisionTreeMetadata(1, 0, 0.0, 0, 0,
         Map(), Set(),
         Array(6), Gini, QuantileStrategy.Sort,
@@ -307,7 +238,6 @@ class RandomForestSuite extends SparkFunSuite with MLlibTestSparkContext {
       val featureSamplesUnitWeight = Array.fill(10)((1.0, math.random()))
       val featureSamplesSmallWeight = featureSamplesUnitWeight.map { case (w, x) => (w * 0.001, x)}
       val featureSamplesLargeWeight = featureSamplesUnitWeight.map { case (w, x) => (w * 1000, x)}
->>>>>>> master
       val splitsUnitWeight = RandomForest
         .findSplitsForContinuousFeature(featureSamplesUnitWeight, fakeMetadata, 0)
       val splitsSmallWeight = RandomForest
@@ -433,20 +363,9 @@ class RandomForestSuite extends SparkFunSuite with MLlibTestSparkContext {
       .generateCategoricalDataPointsForMulticlassForOrderedFeatures()
       .map(_.asML.toInstance)
     assert(arr.length === 3000)
-<<<<<<< HEAD
-    val rdd = sc.parallelize(arr)
-    val strategy = new OldStrategy(
-      OldAlgo.Classification,
-      Gini,
-      maxDepth = 2,
-      numClasses = 100,
-      maxBins = 100,
-      categoricalFeaturesInfo = Map(0 -> 10, 1 -> 10))
-=======
     val rdd = sc.parallelize(arr.toImmutableArraySeq)
     val strategy = new OldStrategy(OldAlgo.Classification, Gini, maxDepth = 2, numClasses = 100,
       maxBins = 100, categoricalFeaturesInfo = Map(0 -> 10, 1 -> 10))
->>>>>>> master
     // 2^(10-1) - 1 > 100, so categorical features will be ordered
 
     val metadata = DecisionTreeMetadata.buildMetadata(rdd, strategy)
@@ -873,14 +792,9 @@ class RandomForestSuite extends SparkFunSuite with MLlibTestSparkContext {
       Instance(0.0, 1.0, Vectors.dense(0.0, 0.0)),
       Instance(1.0, 1.0, Vectors.dense(1.0, 0.0)),
       Instance(0.0, 1.0, Vectors.dense(1.0, 0.0)),
-<<<<<<< HEAD
-      Instance(1.0, 1.0, Vectors.dense(1.0, 1.0)))
-    val rdd = sc.parallelize(arr)
-=======
       Instance(1.0, 1.0, Vectors.dense(1.0, 1.0))
     )
     val rdd = sc.parallelize(arr.toImmutableArraySeq)
->>>>>>> master
 
     val numClasses = 2
     val strategy = new OldStrategy(
@@ -919,13 +833,9 @@ class RandomForestSuite extends SparkFunSuite with MLlibTestSparkContext {
     assert(prunedTree.numNodes === 5)
     assert(unprunedTree.numNodes === 7)
 
-<<<<<<< HEAD
     assert(defaultBehaviorTree.numNodes == unprunedTree.numNodes)
 
-    assert(RandomForestSuite.getSumLeafCounters(List(prunedTree.rootNode)) === arr.size)
-=======
     assert(RandomForestSuite.getSumLeafCounters(List(prunedTree.rootNode)) === arr.length)
->>>>>>> master
   }
 
   test("SPARK-3159 tree model redundancy - regression") {
@@ -939,14 +849,9 @@ class RandomForestSuite extends SparkFunSuite with MLlibTestSparkContext {
       Instance(0.0, 1.0, Vectors.dense(1.0, 0.0)),
       Instance(1.0, 1.0, Vectors.dense(1.0, 1.0)),
       Instance(0.0, 1.0, Vectors.dense(1.0, 1.0)),
-<<<<<<< HEAD
-      Instance(0.5, 1.0, Vectors.dense(1.0, 1.0)))
-    val rdd = sc.parallelize(arr)
-=======
       Instance(0.5, 1.0, Vectors.dense(1.0, 1.0))
     )
     val rdd = sc.parallelize(arr.toImmutableArraySeq)
->>>>>>> master
 
     val strategy = new OldStrategy(
       algo = OldAlgo.Regression,
@@ -983,14 +888,10 @@ class RandomForestSuite extends SparkFunSuite with MLlibTestSparkContext {
 
     assert(prunedTree.numNodes === 3)
     assert(unprunedTree.numNodes === 5)
-<<<<<<< HEAD
 
     assert(defaultBehaviorTree.numNodes == unprunedTree.numNodes)
 
-    assert(RandomForestSuite.getSumLeafCounters(List(prunedTree.rootNode)) === arr.size)
-=======
     assert(RandomForestSuite.getSumLeafCounters(List(prunedTree.rootNode)) === arr.length)
->>>>>>> master
   }
 
   test("weights at arbitrary scale") {
@@ -1024,18 +925,11 @@ class RandomForestSuite extends SparkFunSuite with MLlibTestSparkContext {
       Instance(0.0, 1.0, Vectors.dense(0.0)),
       Instance(0.0, 1.0, Vectors.dense(0.0)),
       Instance(0.0, 1.0, Vectors.dense(0.0)),
-<<<<<<< HEAD
-      Instance(1.0, 0.1, Vectors.dense(1.0)))
-    val rdd = sc.parallelize(data)
-    val strategy =
-      new OldStrategy(OldAlgo.Classification, Gini, 3, 2, minWeightFractionPerNode = 0.5)
-=======
       Instance(1.0, 0.1, Vectors.dense(1.0))
     )
     val rdd = sc.parallelize(data.toImmutableArraySeq)
     val strategy = new OldStrategy(OldAlgo.Classification, Gini, 3, 2,
       minWeightFractionPerNode = 0.5)
->>>>>>> master
     val Array(tree1) = RandomForest.run(rdd, strategy, 1, "all", 42L, None)
     assert(tree1.depth === 0)
 
diff --git a/python/pyspark/ml/classification.py b/python/pyspark/ml/classification.py
index 321da0520766..ae5566cdda03 100644
--- a/python/pyspark/ml/classification.py
+++ b/python/pyspark/ml/classification.py
@@ -1671,13 +1671,6 @@ class _DecisionTreeClassifierParams(_DecisionTreeParams, _TreeClassifierParams):
     Params for :py:class:`DecisionTreeClassifier` and :py:class:`DecisionTreeClassificationModel`.
     """
 
-<<<<<<< HEAD
-    def __init__(self, *args):
-        super(_DecisionTreeClassifierParams, self).__init__(*args)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, pruneTree=True,
-                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
-                         impurity="gini", leafCol="", minWeightFractionPerNode=0.0)
-=======
     def __init__(self, *args: Any):
         super().__init__(*args)
         self._setDefault(
@@ -1685,6 +1678,7 @@ class _DecisionTreeClassifierParams(_DecisionTreeParams, _TreeClassifierParams):
             maxBins=32,
             minInstancesPerNode=1,
             minInfoGain=0.0,
+            pruneTree=True,
             maxMemoryInMB=256,
             cacheNodeIds=False,
             checkpointInterval=10,
@@ -1692,7 +1686,6 @@ class _DecisionTreeClassifierParams(_DecisionTreeParams, _TreeClassifierParams):
             leafCol="",
             minWeightFractionPerNode=0.0,
         )
->>>>>>> master
 
 
 @inherit_doc
@@ -1785,13 +1778,6 @@ class DecisionTreeClassifier(
     _input_kwargs: Dict[str, Any]
 
     @keyword_only
-<<<<<<< HEAD
-    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
-                 probabilityCol="probability", rawPredictionCol="rawPrediction",
-                 maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, pruneTree=True,
-                 maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, impurity="gini",
-                 seed=None, weightCol=None, leafCol="", minWeightFractionPerNode=0.0):
-=======
     def __init__(
         self,
         *,
@@ -1804,6 +1790,7 @@ class DecisionTreeClassifier(
         maxBins: int = 32,
         minInstancesPerNode: int = 1,
         minInfoGain: float = 0.0,
+        pruneTree: bool = True,
         maxMemoryInMB: int = 256,
         cacheNodeIds: bool = False,
         checkpointInterval: int = 10,
@@ -1813,7 +1800,6 @@ class DecisionTreeClassifier(
         leafCol: str = "",
         minWeightFractionPerNode: float = 0.0,
     ):
->>>>>>> master
         """
         __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  probabilityCol="probability", rawPredictionCol="rawPrediction", \
@@ -1830,14 +1816,6 @@ class DecisionTreeClassifier(
 
     @keyword_only
     @since("1.4.0")
-<<<<<<< HEAD
-    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
-                  probabilityCol="probability", rawPredictionCol="rawPrediction",
-                  maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, pruneTree=True,
-                  maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
-                  impurity="gini", seed=None, weightCol=None, leafCol="",
-                  minWeightFractionPerNode=0.0):
-=======
     def setParams(
         self,
         *,
@@ -1850,6 +1828,7 @@ class DecisionTreeClassifier(
         maxBins: int = 32,
         minInstancesPerNode: int = 1,
         minInfoGain: float = 0.0,
+        pruneTree: bool = True,
         maxMemoryInMB: int = 256,
         cacheNodeIds: bool = False,
         checkpointInterval: int = 10,
@@ -1859,7 +1838,6 @@ class DecisionTreeClassifier(
         leafCol: str = "",
         minWeightFractionPerNode: float = 0.0,
     ) -> "DecisionTreeClassifier":
->>>>>>> master
         """
         setParams(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                   probabilityCol="probability", rawPredictionCol="rawPrediction", \
@@ -1886,17 +1864,13 @@ class DecisionTreeClassifier(
         """
         return self._set(maxBins=value)
 
-<<<<<<< HEAD
-    def setPruneTree(self, value):
+    def setPruneTree(self, value: bool) -> "DecisionTreeClassifier":
         """
         Sets the value of :py:attr:`pruneTree`.
         """
         return self._set(pruneTree=value)
 
-    def setMinInstancesPerNode(self, value):
-=======
     def setMinInstancesPerNode(self, value: int) -> "DecisionTreeClassifier":
->>>>>>> master
         """
         Sets the value of :py:attr:`minInstancesPerNode`.
         """
@@ -2000,15 +1974,6 @@ class _RandomForestClassifierParams(_RandomForestParams, _TreeClassifierParams):
     Params for :py:class:`RandomForestClassifier` and :py:class:`RandomForestClassificationModel`.
     """
 
-<<<<<<< HEAD
-    def __init__(self, *args):
-        super(_RandomForestClassifierParams, self).__init__(*args)
-        self._setDefault(maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, pruneTree=True,
-                         maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10,
-                         impurity="gini", numTrees=20, featureSubsetStrategy="auto",
-                         subsamplingRate=1.0, leafCol="", minWeightFractionPerNode=0.0,
-                         bootstrap=True)
-=======
     def __init__(self, *args: Any):
         super().__init__(*args)
         self._setDefault(
@@ -2016,6 +1981,7 @@ class _RandomForestClassifierParams(_RandomForestParams, _TreeClassifierParams):
             maxBins=32,
             minInstancesPerNode=1,
             minInfoGain=0.0,
+            pruneTree=True,
             maxMemoryInMB=256,
             cacheNodeIds=False,
             checkpointInterval=10,
@@ -2027,7 +1993,6 @@ class _RandomForestClassifierParams(_RandomForestParams, _TreeClassifierParams):
             minWeightFractionPerNode=0.0,
             bootstrap=True,
         )
->>>>>>> master
 
 
 @inherit_doc
@@ -2114,14 +2079,6 @@ class RandomForestClassifier(
     _input_kwargs: Dict[str, Any]
 
     @keyword_only
-<<<<<<< HEAD
-    def __init__(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
-                 probabilityCol="probability", rawPredictionCol="rawPrediction",
-                 maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, pruneTree=True,
-                 maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, impurity="gini",
-                 numTrees=20, featureSubsetStrategy="auto", seed=None, subsamplingRate=1.0,
-                 leafCol="", minWeightFractionPerNode=0.0, weightCol=None, bootstrap=True):
-=======
     def __init__(
         self,
         *,
@@ -2134,6 +2091,7 @@ class RandomForestClassifier(
         maxBins: int = 32,
         minInstancesPerNode: int = 1,
         minInfoGain: float = 0.0,
+        pruneTree: bool = True,
         maxMemoryInMB: int = 256,
         cacheNodeIds: bool = False,
         checkpointInterval: int = 10,
@@ -2147,7 +2105,6 @@ class RandomForestClassifier(
         weightCol: Optional[str] = None,
         bootstrap: Optional[bool] = True,
     ):
->>>>>>> master
         """
         __init__(self, \\*, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  probabilityCol="probability", rawPredictionCol="rawPrediction", \
@@ -2165,14 +2122,6 @@ class RandomForestClassifier(
 
     @keyword_only
     @since("1.4.0")
-<<<<<<< HEAD
-    def setParams(self, *, featuresCol="features", labelCol="label", predictionCol="prediction",
-                  probabilityCol="probability", rawPredictionCol="rawPrediction",
-                  maxDepth=5, maxBins=32, minInstancesPerNode=1, minInfoGain=0.0, pruneTree=True,
-                  maxMemoryInMB=256, cacheNodeIds=False, checkpointInterval=10, seed=None,
-                  impurity="gini", numTrees=20, featureSubsetStrategy="auto", subsamplingRate=1.0,
-                  leafCol="", minWeightFractionPerNode=0.0, weightCol=None, bootstrap=True):
-=======
     def setParams(
         self,
         *,
@@ -2185,6 +2134,7 @@ class RandomForestClassifier(
         maxBins: int = 32,
         minInstancesPerNode: int = 1,
         minInfoGain: float = 0.0,
+        pruneTree: bool = True,
         maxMemoryInMB: int = 256,
         cacheNodeIds: bool = False,
         checkpointInterval: int = 10,
@@ -2198,7 +2148,6 @@ class RandomForestClassifier(
         weightCol: Optional[str] = None,
         bootstrap: Optional[bool] = True,
     ) -> "RandomForestClassifier":
->>>>>>> master
         """
         setParams(self, featuresCol="features", labelCol="label", predictionCol="prediction", \
                  probabilityCol="probability", rawPredictionCol="rawPrediction", \
@@ -2226,17 +2175,13 @@ class RandomForestClassifier(
         """
         return self._set(maxBins=value)
 
-<<<<<<< HEAD
-    def setPruneTree(self, value):
+    def setPruneTree(self, value: bool) -> "RandomForestClassifier":
         """
         Sets the value of :py:attr:`pruneTree`.
         """
         return self._set(pruneTree=value)
 
-    def setMinInstancesPerNode(self, value):
-=======
     def setMinInstancesPerNode(self, value: int) -> "RandomForestClassifier":
->>>>>>> master
         """
         Sets the value of :py:attr:`minInstancesPerNode`.
         """
diff --git a/python/pyspark/ml/tree.py b/python/pyspark/ml/tree.py
index 1a9a320b03f8..1083168e68e1 100644
--- a/python/pyspark/ml/tree.py
+++ b/python/pyspark/ml/tree.py
@@ -415,7 +415,6 @@ class _TreeClassifierParams(Params):
         typeConverter=TypeConverters.toString,
     )
 
-<<<<<<< HEAD
     pruneTree = Param(Params._dummy(), "pruneTree", "" +
                       "If true, the trained tree will undergo a 'pruning' process after training in which nodes" +
                       " that have the same class predictions will be merged.  This drawback means that the class" +
@@ -424,12 +423,8 @@ class _TreeClassifierParams(Params):
                       " If false, the post-training tree will undergo no pruning.  The benefit being that you" +
                       " maintain the class prediction probabilities", typeConverter=TypeConverters.toBoolean)
 
-    def __init__(self):
-        super(_TreeClassifierParams, self).__init__()
-=======
     def __init__(self) -> None:
         super().__init__()
->>>>>>> master
 
     @since("1.6.0")
     def getImpurity(self) -> str:


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


(spark) 01/02: merge master

Posted by we...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

WeichenXu123 pushed a commit to branch SPARK-34591
in repository https://gitbox.apache.org/repos/asf/spark.git

commit 86aa0c20c451562fcbb8d35d8511db8849df2408
Merge: f51bbae8eeb7 d6a39df91f25
Author: Weichen Xu <we...@databricks.com>
AuthorDate: Thu May 7 16:23:26 2026 +0800

    merge master
    
    Signed-off-by: Weichen Xu <we...@databricks.com>

 .asf.yaml                                          |    12 +-
 .github/PULL_REQUEST_TEMPLATE                      |    15 +-
 .github/labeler.yml                                |   152 -
 .github/workflows/benchmark.yml                    |   175 +-
 .github/workflows/build_and_test.yml               |  1508 +-
 .github/workflows/build_branch35.yml               |    53 +
 .github/workflows/build_branch35_python.yml        |    47 +
 .github/workflows/build_branch40.yml               |    53 +
 .github/workflows/build_branch40_java21.yml        |    57 +
 .github/workflows/build_branch40_maven.yml         |    35 +
 .github/workflows/build_branch40_maven_java21.yml  |    36 +
 .github/workflows/build_branch40_non_ansi.yml      |    53 +
 .github/workflows/build_branch40_python.yml        |    47 +
 .../workflows/build_branch40_python_pypy3.10.yml   |    47 +
 .github/workflows/build_branch41.yml               |    53 +
 .github/workflows/build_branch41_java21.yml        |    57 +
 .github/workflows/build_branch41_maven.yml         |    35 +
 .github/workflows/build_branch41_maven_java21.yml  |    36 +
 .github/workflows/build_branch41_non_ansi.yml      |    53 +
 .github/workflows/build_branch41_python.yml        |    47 +
 .github/workflows/build_branch41_python_3.14.yml   |    47 +
 .../workflows/build_branch41_python_pypy3.10.yml   |    47 +
 .github/workflows/build_branch42.yml               |    53 +
 .github/workflows/build_branch42_java21.yml        |    57 +
 .github/workflows/build_branch42_maven.yml         |    35 +
 .github/workflows/build_branch42_maven_java21.yml  |    36 +
 .github/workflows/build_branch42_non_ansi.yml      |    53 +
 .github/workflows/build_branch42_python.yml        |    47 +
 .github/workflows/build_branch42_python_3.14.yml   |    47 +
 .../workflows/build_branch42_python_pypy3.10.yml   |    47 +
 .github/workflows/build_branch4x.yml               |    54 +
 .github/workflows/build_branch4x_java21.yml        |    57 +
 .github/workflows/build_branch4x_maven.yml         |    35 +
 .github/workflows/build_branch4x_maven_java21.yml  |    36 +
 .github/workflows/build_branch4x_non_ansi.yml      |    53 +
 .github/workflows/build_branch4x_python.yml        |    47 +
 .github/workflows/build_branch4x_python_3.14.yml   |    47 +
 .../workflows/build_branch4x_python_pypy3.10.yml   |    47 +
 .github/workflows/build_coverage.yml               |    51 +
 .github/workflows/build_infra_images_cache.yml     |   245 +
 .github/workflows/build_java21.yml                 |    57 +
 .github/workflows/build_java25.yml                 |    57 +
 .github/workflows/build_main.yml                   |    32 +
 .github/workflows/build_maven.yml                  |    33 +
 .github/workflows/build_maven_java21.yml           |    35 +
 .github/workflows/build_maven_java21_arm.yml       |    37 +
 .github/workflows/build_maven_java21_macos26.yml   |    44 +
 .github/workflows/build_maven_java25.yml           |    35 +
 .github/workflows/build_non_ansi.yml               |    55 +
 .github/workflows/build_python_3.10.yml            |    47 +
 .github/workflows/build_python_3.11.yml            |    47 +
 .github/workflows/build_python_3.12_arm.yml        |    35 +
 .../workflows/build_python_3.12_classic_only.yml   |    47 +
 .github/workflows/build_python_3.12_macos26.yml    |    35 +
 .github/workflows/build_python_3.12_pandas_3.yml   |    47 +
 .github/workflows/build_python_3.13.yml            |    47 +
 .github/workflows/build_python_3.14.yml            |    47 +
 .github/workflows/build_python_3.14_nogil.yml      |    48 +
 .github/workflows/build_python_connect.yml         |   140 +
 .github/workflows/build_python_connect40.yml       |   120 +
 .github/workflows/build_python_minimum.yml         |    46 +
 .github/workflows/build_python_ps_minimum.yml      |    47 +
 .github/workflows/build_rockdb_as_ui_backend.yml   |    50 +
 .github/workflows/build_sparkr_window.yml          |    93 +
 .github/workflows/build_uds.yml                    |    53 +
 .../workflows/cancel_duplicate_workflow_runs.yml   |    38 -
 .../workflows/images/workflow-enable-button.png    |   Bin 0 -> 79807 bytes
 .github/workflows/labeler.yml                      |    50 -
 .github/workflows/maven_test.yml                   |   252 +
 .github/workflows/notify_test_workflow.yml         |    73 +-
 .github/workflows/pages.yml                        |    98 +
 .github/workflows/publish_snapshot.yml             |    34 +-
 .github/workflows/python_hosted_runner_test.yml    |   186 +
 .github/workflows/release.yml                      |   322 +
 .github/workflows/stale.yml                        |     3 +-
 .github/workflows/test_report.yml                  |    20 +-
 .github/workflows/update_build_status.yml          |    11 +-
 .gitignore                                         |    36 +-
 .idea/vcs.xml                                      |    36 -
 .mvn/jvm.config                                    |     4 +
 .mvn/maven.config                                  |     2 +
 .../sql/parquet_partitioned/_SUCCESS => .nojekyll  |     0
 .pre-commit-config.yaml                            |    33 +
 .sbtopts                                           |    20 +
 AGENTS.md                                          |   130 +
 CLAUDE.md                                          |     1 +
 LICENSE                                            |    18 +-
 LICENSE-binary                                     |   453 +-
 NOTICE-binary                                      |   650 +-
 R/DOCUMENTATION.md                                 |     3 +-
 R/README.md                                        |    73 -
 R/check-cran.sh                                    |     2 +-
 R/create-docs.sh                                   |    18 +-
 R/create-rd.sh                                     |     6 +-
 R/find-r.sh                                        |     2 +-
 R/install-dev.sh                                   |     2 +-
 R/install-source-package.sh                        |     2 +-
 R/log4j.properties                                 |    28 -
 R/log4j2.properties                                |    31 +
 R/pkg/.Rbuildignore                                |     3 +
 R/pkg/.gitignore                                   |     1 +
 R/pkg/DESCRIPTION                                  |    20 +-
 R/pkg/NAMESPACE                                    |    37 +-
 R/pkg/R/DataFrame.R                                |   166 +-
 R/pkg/R/RDD.R                                      |     2 +-
 R/pkg/R/SQLContext.R                               |    35 +-
 R/pkg/R/WindowSpec.R                               |     2 +-
 R/pkg/R/catalog.R                                  |   294 +-
 R/pkg/R/client.R                                   |     6 +-
 R/pkg/R/column.R                                   |     8 +-
 R/pkg/R/context.R                                  |     4 +-
 R/pkg/R/functions.R                                |   405 +-
 R/pkg/R/generics.R                                 |    86 +-
 R/pkg/R/install.R                                  |    15 +-
 R/pkg/R/jobj.R                                     |     1 +
 R/pkg/R/mllib_classification.R                     |     4 +-
 R/pkg/R/pairRDD.R                                  |     2 +-
 R/pkg/R/schema.R                                   |     2 +
 R/pkg/R/serialize.R                                |    15 +-
 R/pkg/R/sparkR.R                                   |   178 +-
 R/pkg/R/utils.R                                    |     1 +
 R/pkg/R/zzz.R                                      |    30 +
 R/pkg/README.md                                    |    73 +
 R/pkg/pkgdown/_pkgdown_template.yml                |   321 +
 R/pkg/pkgdown/extra.css                            |    48 +
 R/pkg/tests/fulltests/test_context.R               |    16 +
 R/pkg/tests/fulltests/test_mllib_fpm.R             |     8 +-
 R/pkg/tests/fulltests/test_sparkR.R                |     6 +-
 R/pkg/tests/fulltests/test_sparkSQL.R              |   357 +-
 R/pkg/tests/fulltests/test_sparkSQL_arrow.R        |    15 +
 R/pkg/tests/fulltests/test_streaming.R             |    10 +-
 R/pkg/tests/fulltests/test_utils.R                 |     2 +-
 R/pkg/vignettes/sparkr-vignettes.Rmd               |   183 +-
 R/run-tests.sh                                     |     8 +-
 README.md                                          |   113 +-
 appveyor.yml                                       |    68 -
 assembly/README                                    |    12 -
 assembly/pom.xml                                   |   209 +-
 assembly/src/main/assembly/assembly.xml            |    93 -
 bin/docker-image-tool.sh                           |    26 +-
 bin/load-spark-env.cmd                             |    30 +-
 bin/load-spark-env.sh                              |    43 +-
 bin/pyspark                                        |    36 +-
 bin/pyspark2.cmd                                   |     2 +-
 bin/spark-class                                    |     3 +-
 bin/spark-class2.cmd                               |    24 +-
 bin/spark-connect-shell                            |    26 +
 bin/spark-pipelines                                |    40 +
 bin/spark-shell                                    |     5 +-
 bin/spark-sql                                      |     1 +
 bin/spark-sql2.cmd                                 |     2 +
 bin/sparkR                                         |     3 +-
 bin/sparkR2.cmd                                    |     2 +
 binder/Dockerfile                                  |    43 +
 binder/apt.txt                                     |     1 -
 binder/postBuild                                   |    35 +-
 build/mvn                                          |    67 +-
 build/sbt                                          |     9 +-
 build/sbt-launch-lib.bash                          |    24 +-
 build/spark-build-info                             |     3 +-
 build/spark-build-info.ps1                         |    47 +
 common/kvstore/pom.xml                             |    38 +-
 .../apache/spark/util/kvstore/ArrayWrappers.java   |    37 +-
 .../apache/spark/util/kvstore/InMemoryStore.java   |    14 +-
 .../org/apache/spark/util/kvstore/KVStore.java     |     4 +-
 .../spark/util/kvstore/KVStoreSerializer.java      |    10 +-
 .../org/apache/spark/util/kvstore/KVStoreView.java |     7 +-
 .../org/apache/spark/util/kvstore/KVTypeInfo.java  |    23 +-
 .../org/apache/spark/util/kvstore/LevelDB.java     |    41 +-
 .../apache/spark/util/kvstore/LevelDBIterator.java |    94 +-
 .../apache/spark/util/kvstore/LevelDBTypeInfo.java |    44 +-
 .../org/apache/spark/util/kvstore/RocksDB.java     |   464 +
 .../apache/spark/util/kvstore/RocksDBIterator.java |   321 +
 .../apache/spark/util/kvstore/RocksDBTypeInfo.java |   512 +
 .../spark/util/kvstore/ArrayKeyIndexType.java      |     3 +-
 .../spark/util/kvstore/ArrayWrappersSuite.java     |     4 +-
 .../org/apache/spark/util/kvstore/CustomType1.java |     3 +-
 .../org/apache/spark/util/kvstore/CustomType2.java |     3 +-
 .../apache/spark/util/kvstore/DBIteratorSuite.java |    34 +-
 .../spark/util/kvstore/InMemoryStoreSuite.java     |    43 +-
 .../org/apache/spark/util/kvstore/IntKeyType.java  |     3 +-
 .../spark/util/kvstore/LevelDBBenchmark.java       |    42 +-
 .../spark/util/kvstore/LevelDBIteratorSuite.java   |    12 +-
 .../apache/spark/util/kvstore/LevelDBSuite.java    |   250 +-
 .../spark/util/kvstore/LevelDBTypeInfoSuite.java   |    48 +-
 .../spark/util/kvstore/RocksDBIteratorSuite.java   |    49 +
 .../apache/spark/util/kvstore/RocksDBSuite.java    |   491 +
 .../spark/util/kvstore/RocksDBTypeInfoSuite.java   |   213 +
 common/kvstore/src/test/resources/log4j.properties |    27 -
 .../kvstore/src/test/resources/log4j2.properties   |    31 +
 common/network-common/pom.xml                      |   111 +-
 .../org/apache/spark/network/TransportContext.java |    95 +-
 .../network/buffer/FileSegmentManagedBuffer.java   |    19 +-
 .../apache/spark/network/buffer/ManagedBuffer.java |    16 +-
 .../spark/network/buffer/NettyManagedBuffer.java   |    11 +-
 .../spark/network/buffer/NioManagedBuffer.java     |    11 +-
 .../spark/network/client/StreamInterceptor.java    |     4 +-
 .../spark/network/client/TransportClient.java      |    47 +-
 .../network/client/TransportClientFactory.java     |    93 +-
 .../network/client/TransportResponseHandler.java   |    93 +-
 .../spark/network/crypto/AuthClientBootstrap.java  |     9 +-
 .../apache/spark/network/crypto/AuthEngine.java    |    61 +-
 .../apache/spark/network/crypto/AuthMessage.java   |    12 +-
 .../spark/network/crypto/AuthRpcHandler.java       |    32 +-
 .../spark/network/crypto/AuthServerBootstrap.java  |     1 +
 .../spark/network/crypto/CtrTransportCipher.java   |   381 +
 .../spark/network/crypto/GcmTransportCipher.java   |   413 +
 .../java/org/apache/spark/network/crypto/README.md |    17 +-
 .../spark/network/crypto/TransportCipher.java      |   359 +-
 .../spark/network/crypto/TransportCipherUtil.java  |    41 +
 .../spark/network/protocol/AbstractMessage.java    |     4 +-
 .../spark/network/protocol/ChunkFetchFailure.java  |    10 +-
 .../spark/network/protocol/ChunkFetchRequest.java  |     9 +-
 .../spark/network/protocol/ChunkFetchSuccess.java  |    10 +-
 .../protocol/EncryptedMessageWithHeader.java       |   146 +
 .../network/protocol/MergedBlockMetaRequest.java   |    21 +-
 .../network/protocol/MergedBlockMetaSuccess.java   |    10 +-
 .../org/apache/spark/network/protocol/Message.java |    34 +-
 .../spark/network/protocol/MessageDecoder.java     |    66 +-
 .../spark/network/protocol/MessageEncoder.java     |    40 +-
 .../spark/network/protocol/MessageWithHeader.java  |    25 +-
 .../spark/network/protocol/OneWayMessage.java      |     9 +-
 .../apache/spark/network/protocol/RpcFailure.java  |    10 +-
 .../apache/spark/network/protocol/RpcRequest.java  |    10 +-
 .../apache/spark/network/protocol/RpcResponse.java |    10 +-
 .../spark/network/protocol/SslMessageEncoder.java  |   110 +
 .../spark/network/protocol/StreamChunkId.java      |    20 +-
 .../spark/network/protocol/StreamFailure.java      |    10 +-
 .../spark/network/protocol/StreamRequest.java      |     9 +-
 .../spark/network/protocol/StreamResponse.java     |    12 +-
 .../spark/network/protocol/UploadStream.java       |    10 +-
 .../spark/network/sasl/SaslClientBootstrap.java    |    20 +-
 .../apache/spark/network/sasl/SaslEncryption.java  |     6 +-
 .../apache/spark/network/sasl/SaslRpcHandler.java  |     6 +-
 .../spark/network/sasl/SaslServerBootstrap.java    |     1 +
 .../spark/network/sasl/SaslTimeoutException.java   |    35 +
 .../apache/spark/network/sasl/SparkSaslClient.java |    23 +-
 .../apache/spark/network/sasl/SparkSaslServer.java |    30 +-
 .../network/server/BlockPushNonFatalFailure.java   |    27 +-
 .../network/server/ChunkFetchRequestHandler.java   |    38 +-
 .../network/server/OneForOneStreamManager.java     |    53 +-
 .../apache/spark/network/server/RpcHandler.java    |     7 +-
 .../network/server/TransportChannelHandler.java    |    35 +-
 .../network/server/TransportRequestHandler.java    |    85 +-
 .../spark/network/server/TransportServer.java      |    24 +-
 .../org/apache/spark/network/shuffledb/DB.java     |    50 +
 .../apache/spark/network/shuffledb/DBBackend.java  |    42 +
 .../apache/spark/network/shuffledb/DBIterator.java |    38 +
 .../apache/spark/network/shuffledb/LevelDB.java    |    53 +
 .../spark/network/shuffledb/LevelDBIterator.java   |    86 +
 .../apache/spark/network/shuffledb/RocksDB.java    |    70 +
 .../spark/network/shuffledb/RocksDBIterator.java   |    94 +
 .../spark/network/shuffledb/StoreVersion.java      |    62 +
 .../network/ssl/ReloadingX509TrustManager.java     |   225 +
 .../org/apache/spark/network/ssl/SSLFactory.java   |   483 +
 .../network/util/ByteBufferWriteableChannel.java   |    59 +
 .../org/apache/spark/network/util/DBProvider.java  |    69 +
 .../java/org/apache/spark/network/util/IOMode.java |    19 +-
 .../org/apache/spark/network/util/JavaUtils.java   |   378 -
 .../apache/spark/network/util/LevelDBProvider.java |    72 +-
 .../spark/network/util/LimitedInputStream.java     |    25 +-
 .../org/apache/spark/network/util/NettyLogger.java |    77 +
 .../org/apache/spark/network/util/NettyUtils.java  |   105 +-
 .../apache/spark/network/util/RocksDBProvider.java |   176 +
 .../apache/spark/network/util/TransportConf.java   |   263 +-
 .../spark/network/util/TransportFrameDecoder.java  |     7 +-
 .../spark/network/ChunkFetchIntegrationSuite.java  |    31 +-
 .../network/ChunkFetchRequestHandlerSuite.java     |    35 +-
 .../org/apache/spark/network/ProtocolSuite.java    |     9 +-
 .../network/RequestTimeoutIntegrationSuite.java    |     8 +-
 .../apache/spark/network/RpcIntegrationSuite.java  |   121 +-
 .../network/SslChunkFetchIntegrationSuite.java     |    32 +
 .../java/org/apache/spark/network/StreamSuite.java |    42 +-
 .../org/apache/spark/network/StreamTestHelper.java |    31 +-
 .../apache/spark/network/TestManagedBuffer.java    |    13 +-
 .../apache/spark/network/TransportConfSuite.java   |   110 +
 .../network/TransportRequestHandlerSuite.java      |    43 +-
 .../network/TransportResponseHandlerSuite.java     |     4 +-
 .../client/SslTransportClientFactorySuite.java     |    39 +
 .../client/TransportClientFactorySuite.java        |    78 +-
 .../spark/network/crypto/AuthEngineSuite.java      |   236 +-
 .../spark/network/crypto/AuthIntegrationSuite.java |   137 +-
 .../spark/network/crypto/AuthMessagesSuite.java    |    12 +-
 .../spark/network/crypto/CtrAuthEngineSuite.java   |   177 +
 .../spark/network/crypto/GcmAuthEngineSuite.java   |   339 +
 .../spark/network/crypto/TransportCipherSuite.java |    33 +-
 .../spark/network/protocol/EncodersSuite.java      |     9 +-
 .../protocol/EncryptedMessageWithHeaderSuite.java  |   154 +
 .../protocol/MergedBlockMetaSuccessSuite.java      |    44 +-
 .../network/protocol/MessageWithHeaderSuite.java   |     6 +-
 .../apache/spark/network/sasl/SparkSaslSuite.java  |    63 +-
 .../server/OneForOneStreamManagerSuite.java        |    71 +-
 .../ssl/ReloadingX509TrustManagerSuite.java        |   314 +
 .../apache/spark/network/ssl/SSLFactorySuite.java  |   104 +
 .../apache/spark/network/ssl/SslSampleConfigs.java |   251 +
 .../spark/network/util/CryptoUtilsSuite.java       |     7 +-
 .../apache/spark/network/util/DBProviderSuite.java |    61 +
 .../apache/spark/network/util/JavaUtilsSuite.java  |    88 +
 .../network/util/NettyMemoryMetricsSuite.java      |    56 +-
 .../network/util/TimerWithCustomUnitSuite.java     |     6 +-
 .../network/util/TransportFrameDecoderSuite.java   |    24 +-
 .../src/test/resources/certchain.pem               |    17 +
 common/network-common/src/test/resources/key.pem   |    29 +
 common/network-common/src/test/resources/keystore  |   Bin 0 -> 2247 bytes
 .../src/test/resources/log4j.properties            |    27 -
 .../src/test/resources/log4j2.properties           |    31 +
 .../network-common/src/test/resources/truststore   |   Bin 0 -> 957 bytes
 .../src/test/resources/unencrypted-certchain.pem   |    21 +
 .../src/test/resources/unencrypted-key.pem         |    28 +
 .../src/test/resources/untrusted-keystore          |   Bin 0 -> 2246 bytes
 common/network-shuffle/pom.xml                     |    39 +-
 .../spark/network/sasl/ShuffleSecretManager.java   |    15 +-
 .../network/shuffle/AppsWithRecoveryDisabled.java  |    62 +
 .../spark/network/shuffle/BlockStoreClient.java    |    42 +-
 .../apache/spark/network/shuffle/Constants.java    |     7 +
 .../shuffle/DownloadFileWritableChannel.java       |     3 +-
 .../apache/spark/network/shuffle/ErrorHandler.java |    19 +-
 .../spark/network/shuffle/ExecutorDiskUtils.java   |     7 +-
 .../network/shuffle/ExternalBlockHandler.java      |    68 +-
 .../network/shuffle/ExternalBlockStoreClient.java  |    70 +-
 .../shuffle/ExternalShuffleBlockResolver.java      |   163 +-
 .../spark/network/shuffle/MergedBlockMeta.java     |     4 +-
 .../network/shuffle/MergedShuffleFileManager.java  |    28 +
 .../shuffle/NoOpMergedShuffleFileManager.java      |     9 +-
 .../network/shuffle/OneForOneBlockFetcher.java     |    31 +-
 .../network/shuffle/OneForOneBlockPusher.java      |    12 +-
 .../network/shuffle/RemoteBlockPushResolver.java   |  1337 +-
 .../network/shuffle/RetryingBlockTransferor.java   |   138 +-
 .../network/shuffle/ShuffleIndexInformation.java   |    23 +-
 .../spark/network/shuffle/ShuffleIndexRecord.java  |    18 +-
 .../network/shuffle/ShuffleTransportContext.java   |   187 +
 .../spark/network/shuffle/SimpleDownloadFile.java  |     3 +-
 .../shuffle/checksum/ShuffleChecksumHelper.java    |    66 +-
 .../protocol/AbstractFetchShuffleBlocks.java       |    14 +-
 .../shuffle/protocol/BlockPushReturnCode.java      |    14 +-
 .../shuffle/protocol/BlockTransferMessage.java     |    47 +-
 .../network/shuffle/protocol/BlocksRemoved.java    |     9 +-
 .../network/shuffle/protocol/CorruptionCause.java  |     6 +-
 .../shuffle/protocol/DiagnoseCorruption.java       |    14 +-
 .../shuffle/protocol/ExecutorShuffleInfo.java      |    12 +-
 .../shuffle/protocol/FetchShuffleBlockChunks.java  |     9 +-
 .../shuffle/protocol/FetchShuffleBlocks.java       |     8 +-
 .../shuffle/protocol/FinalizeShuffleMerge.java     |    20 +-
 .../shuffle/protocol/GetLocalDirsForExecutors.java |    10 +-
 .../shuffle/protocol/LocalDirsForExecutors.java    |    13 +-
 .../network/shuffle/protocol/MergeStatuses.java    |    19 +-
 .../spark/network/shuffle/protocol/OpenBlocks.java |    12 +-
 .../network/shuffle/protocol/PushBlockStream.java  |    25 +-
 .../network/shuffle/protocol/RegisterExecutor.java |    12 +-
 .../network/shuffle/protocol/RemoveBlocks.java     |    12 +-
 .../shuffle/protocol/RemoveShuffleMerge.java       |    96 +
 .../network/shuffle/protocol/StreamHandle.java     |    10 +-
 .../network/shuffle/protocol/UploadBlock.java      |    14 +-
 .../shuffle/protocol/UploadBlockStream.java        |    10 +-
 .../shuffle/protocol/mesos/RegisterDriver.java     |    77 -
 .../protocol/mesos/ShuffleServiceHeartbeat.java    |    53 -
 .../spark/network/sasl/SaslIntegrationSuite.java   |    55 +-
 .../network/sasl/ShuffleSecretManagerSuite.java    |     4 +-
 .../spark/network/shuffle/AppIsolationSuite.java   |    14 +-
 .../shuffle/BlockTransferMessagesSuite.java        |     4 +-
 .../CleanupNonShuffleServiceServedFilesSuite.java  |    45 +-
 .../spark/network/shuffle/ErrorHandlerSuite.java   |     4 +-
 .../network/shuffle/ExternalBlockHandlerSuite.java |    79 +-
 .../shuffle/ExternalShuffleBlockResolverSuite.java |    44 +-
 .../shuffle/ExternalShuffleCleanupSuite.java       |    10 +-
 .../shuffle/ExternalShuffleIntegrationSuite.java   |   227 +-
 .../shuffle/ExternalShuffleSecuritySuite.java      |    44 +-
 .../shuffle/OneForOneBlockFetcherSuite.java        |    41 +-
 .../network/shuffle/OneForOneBlockPusherSuite.java |    13 +-
 .../shuffle/RemoteBlockPushResolverSuite.java      |   863 +-
 .../shuffle/RetryingBlockTransferorSuite.java      |   197 +-
 .../shuffle/ShuffleIndexInformationSuite.java      |    71 +
 .../shuffle/ShuffleTransportContextSuite.java      |   147 +
 .../network/shuffle/SimpleDownloadFileSuite.java   |    48 +
 .../SslExternalShuffleIntegrationSuite.java        |    44 +
 .../shuffle/SslExternalShuffleSecuritySuite.java   |    43 +
 .../shuffle/SslShuffleTransportContextSuite.java   |    38 +
 .../network/shuffle/TestShuffleDataContext.java    |    33 +-
 .../protocol/FetchShuffleBlockChunksSuite.java     |    10 +-
 .../shuffle/protocol/FetchShuffleBlocksSuite.java  |    10 +-
 .../src/test/resources/certchain.pem               |    17 +
 common/network-shuffle/src/test/resources/key.pem  |    29 +
 common/network-shuffle/src/test/resources/keystore |   Bin 0 -> 2247 bytes
 .../src/test/resources/log4j.properties            |    24 -
 .../src/test/resources/log4j2.properties           |    27 +
 .../network-shuffle/src/test/resources/truststore  |   Bin 0 -> 957 bytes
 .../src/test/resources/unencrypted-certchain.pem   |    21 +
 .../src/test/resources/unencrypted-key.pem         |    28 +
 .../src/test/resources/untrusted-keystore          |   Bin 0 -> 2246 bytes
 common/network-yarn/pom.xml                        |    61 +-
 .../spark/network/yarn/YarnShuffleService.java     |   232 +-
 .../network/yarn/YarnShuffleServiceMetrics.java    |    70 +-
 common/sketch/pom.xml                              |    14 +-
 .../org/apache/spark/util/sketch/BitArray.java     |     3 +-
 .../org/apache/spark/util/sketch/BloomFilter.java  |    81 +-
 .../apache/spark/util/sketch/BloomFilterBase.java  |   199 +
 .../apache/spark/util/sketch/BloomFilterImpl.java  |   177 +-
 .../spark/util/sketch/BloomFilterImplV2.java       |   160 +
 .../spark/util/sketch/CountMinSketchImpl.java      |    24 +-
 .../java/org/apache/spark/util/sketch/Utils.java   |    16 +-
 .../spark/util/sketch/SparkBloomFilterSuite.java   |   396 +
 .../spark/util/sketch/BloomFilterSuite.scala       |    16 +-
 .../spark/util/sketch/CountMinSketchSuite.scala    |     2 +-
 common/tags/pom.xml                                |     8 +-
 .../org/apache/spark/annotation/ClassicOnly.java   |    28 +
 .../java/org/apache/spark/tags/AmmoniteTest.java   |    30 +
 .../org/apache/spark/tags/ExtendedLevelDBTest.java |    30 +
 .../java/org/apache/spark/tags/SlowSQLTest.java    |    30 +
 .../java/org/apache/spark/tags/WebBrowserTest.java |    27 +
 common/unsafe/pom.xml                              |    55 +-
 .../catalyst/util/CollationAwareUTF8String.java    |  1607 +
 .../spark/sql/catalyst/util/CollationFactory.java  |  1504 +
 .../spark/sql/catalyst/util/CollationNames.java    |    25 +
 .../spark/sql/catalyst/util/CollationSupport.java  |   721 +
 .../spark/sql/catalyst/util/DateTimeConstants.java |     1 +
 .../catalyst/util/SpecialCodePointConstants.java   |    33 +
 .../java/org/apache/spark/unsafe/Platform.java     |   114 +-
 .../org/apache/spark/unsafe/UTF8StringBuilder.java |    29 +
 .../apache/spark/unsafe/UnsafeAlignedOffset.java   |    22 +-
 .../spark/unsafe/array/ByteArrayMethods.java       |    54 +-
 .../org/apache/spark/unsafe/types/ByteArray.java   |   189 +-
 .../spark/unsafe/types/CalendarInterval.java       |    24 +-
 .../apache/spark/unsafe/types/GeographyVal.java    |    53 +
 .../org/apache/spark/unsafe/types/GeometryVal.java |    53 +
 .../org/apache/spark/unsafe/types/UTF8String.java  |   958 +-
 .../org/apache/spark/unsafe/types/VariantVal.java  |   136 +
 .../org/apache/spark/unsafe/PlatformUtilSuite.java |    63 +-
 .../apache/spark/unsafe/array/ByteArraySuite.java  |   125 +
 .../apache/spark/unsafe/array/LongArraySuite.java  |    14 +-
 .../spark/unsafe/hash/Murmur3_x86_32Suite.java     |    46 +-
 .../spark/unsafe/types/CalendarIntervalSuite.java  |    50 +-
 .../spark/unsafe/types/CollationSupportSuite.java  |  3894 +++
 .../spark/unsafe/types/GeographyValSuite.java      |    57 +
 .../spark/unsafe/types/GeometryValSuite.java       |    57 +
 .../apache/spark/unsafe/types/UTF8StringSuite.java |   630 +-
 .../spark/unsafe/types/CollationFactorySuite.scala |   553 +
 .../types/UTF8StringPropertyCheckSuite.scala       |     4 +-
 common/utils-java/pom.xml                          |    84 +
 .../main/java/org/apache/spark/QueryContext.java   |    57 +
 .../java/org/apache/spark/QueryContextType.java    |    31 +
 .../spark/api/java/function/CoGroupFunction.java   |     0
 .../api/java/function/DoubleFlatMapFunction.java   |     0
 .../spark/api/java/function/DoubleFunction.java    |     0
 .../spark/api/java/function/FilterFunction.java    |     0
 .../spark/api/java/function/FlatMapFunction.java   |     0
 .../spark/api/java/function/FlatMapFunction2.java  |     0
 .../api/java/function/FlatMapGroupsFunction.java   |     0
 .../spark/api/java/function/ForeachFunction.java   |     0
 .../java/function/ForeachPartitionFunction.java    |     0
 .../apache/spark/api/java/function/Function.java   |     0
 .../apache/spark/api/java/function/Function0.java  |     0
 .../apache/spark/api/java/function/Function2.java  |     0
 .../apache/spark/api/java/function/Function3.java  |     0
 .../apache/spark/api/java/function/Function4.java  |     0
 .../spark/api/java/function/MapFunction.java       |     0
 .../spark/api/java/function/MapGroupsFunction.java |     0
 .../api/java/function/MapPartitionsFunction.java   |     0
 .../api/java/function/PairFlatMapFunction.java     |     0
 .../spark/api/java/function/PairFunction.java      |     0
 .../spark/api/java/function/ReduceFunction.java    |     0
 .../spark/api/java/function/VoidFunction.java      |     0
 .../spark/api/java/function/VoidFunction2.java     |     0
 .../spark/api/java/function/package-info.java      |     0
 .../java/org/apache/spark/internal/LogKey.java     |    52 +
 .../java/org/apache/spark/internal/LogKeys.java    |   907 +
 .../main/java/org/apache/spark/internal/MDC.java   |    29 +
 .../org/apache/spark/internal/SparkLogger.java     |   249 +
 .../apache/spark/internal/SparkLoggerFactory.java  |    47 +
 .../java/org/apache/spark/memory/MemoryMode.java   |     0
 .../org/apache/spark/network/util/ByteUnit.java    |     0
 .../org/apache/spark/network/util/JavaUtils.java   |   826 +
 .../apache/spark/unsafe/array/ByteArrayUtils.java  |    27 +
 .../src/main/java/org/apache/spark/util/Pair.java  |    29 +
 .../org/apache/spark/util/UUIDv7Generator.java     |    67 +
 .../resources/org/apache/spark/SparkLayout.json    |    61 +
 .../org/apache/spark/log4j2-defaults.properties    |    55 +
 .../org/apache/spark/log4j2-json-layout.properties |    55 +
 .../java/org/apache/spark/util/CustomLogKeys.java  |    24 +
 .../apache/spark/util/PatternSparkLoggerSuite.java |    96 +
 .../apache/spark/util/SparkLoggerSuiteBase.java    |   246 +
 .../spark/util/StructuredSparkLoggerSuite.java     |   191 +
 .../apache/spark/util/UUIDv7GeneratorSuite.java    |    73 +
 .../src/test/resources/log4j2.properties           |    60 +
 common/utils/pom.xml                               |   178 +
 .../main/java/org/apache/spark/SparkThrowable.java |   102 +
 .../apache/spark/storage/StorageLevelMapper.java   |    47 +
 common/utils/src/main/resources/error/README.md    |   175 +
 .../src/main/resources/error/error-classes.json    |    90 +
 .../src/main/resources/error/error-conditions.json | 11440 +++++++
 .../src/main/resources/error/error-states.json     |  7569 +++++
 .../org/apache/spark/ErrorClassesJSONReader.scala  |   272 +
 .../scala/org/apache/spark/SparkBuildInfo.scala    |    65 +
 .../scala/org/apache/spark/SparkException.scala    |   828 +
 .../org/apache/spark/SparkThrowableHelper.scala    |   194 +
 .../scala/org/apache/spark/StringSubstitutor.scala |   126 +
 .../apache/spark/api/java/function/package.scala   |     0
 .../scala/org/apache/spark/internal/Logging.scala  |   699 +
 .../internal/config/ConfigBindingPolicy.scala      |    54 +
 .../spark/internal/config/ConfigBuilder.scala      |   394 +
 .../apache/spark/internal/config/ConfigEntry.scala |   302 +
 .../spark/internal/config/ConfigProvider.scala     |    47 +
 .../spark/internal/config/ConfigReader.scala       |   123 +
 .../spark/scheduler/SparkListenerEvent.scala       |    28 +
 .../sql/streaming/StreamingQueryException.scala    |   116 +
 .../org/apache/spark/storage/StorageLevel.scala    |   231 +
 .../org/apache/spark/util/ArrayImplicits.scala     |    35 +
 .../org/apache/spark/util/ClosureCleaner.scala     |  1211 +
 .../org/apache/spark/util/CompletionIterator.scala |     0
 .../scala/org/apache/spark/util/JsonUtils.scala    |    44 +
 .../org/apache/spark/util/LexicalThreadLocal.scala |    66 +
 .../scala/org/apache/spark/util/LogUtils.scala     |    50 +
 .../scala/org/apache/spark/util/MavenUtils.scala   |   660 +
 .../org/apache/spark/util/SparkClassUtils.scala    |   169 +
 .../apache/spark/util/SparkCollectionUtils.scala   |    70 +
 .../org/apache/spark/util/SparkEnvUtils.scala      |    30 +
 .../org/apache/spark/util/SparkErrorUtils.scala    |   168 +
 .../apache/spark/util/SparkFatalException.scala    |    28 +
 .../org/apache/spark/util/SparkFileUtils.scala     |   243 +
 .../org/apache/spark/util/SparkSchemaUtils.scala   |    36 +
 .../org/apache/spark/util/SparkSerDeUtils.scala    |    54 +
 .../org/apache/spark/util/SparkStreamUtils.scala   |   118 +
 .../org/apache/spark/util/SparkStringUtils.scala   |   146 +
 .../org/apache/spark/util/SparkSystemUtils.scala   |    79 +
 .../org/apache/spark/util/SparkTestUtils.scala     |   253 +
 .../org/apache/spark/util/SparkThreadUtils.scala   |    81 +
 .../scala/org/apache/spark/util/VersionUtils.scala |     0
 common/utils/src/test/resources/log4j2.properties  |    60 +
 .../streaming/StreamingQueryExceptionSuite.scala   |    63 +
 .../scala/org/apache/spark/util/IvyTestUtils.scala |   416 +
 .../scala/org/apache/spark/util/LogKeysSuite.scala |    90 +
 .../org/apache/spark/util/LogThrottlingSuite.scala |   234 +
 .../scala/org/apache/spark/util/MDCSuite.scala     |    87 +
 .../org/apache/spark/util/MavenUtilsSuite.scala    |   312 +
 .../scala/org/apache/spark/util/MaybeNull.scala    |    36 +
 .../apache/spark/util/PatternLoggingSuite.scala    |    69 +
 .../apache/spark/util/SparkErrorUtilsSuite.scala   |   120 +
 .../apache/spark/util/StructuredLoggingSuite.scala |   493 +
 .../org/apache/spark/util/VersionUtilsSuite.scala  |   115 +
 common/variant/README.md                           |     1 +
 common/variant/pom.xml                             |    75 +
 .../apache/spark/types/variant/ShreddingUtils.java |   173 +
 .../org/apache/spark/types/variant/Variant.java    |   364 +
 .../apache/spark/types/variant/VariantBuilder.java |   586 +
 .../apache/spark/types/variant/VariantSchema.java  |   162 +
 .../types/variant/VariantShreddingWriter.java      |   303 +
 .../types/variant/VariantSizeLimitException.java   |    25 +
 .../apache/spark/types/variant/VariantUtil.java    |   698 +
 conf/log4j.properties.template                     |    48 -
 conf/log4j2-json-layout.properties.template        |    56 +
 conf/log4j2.properties.template                    |    62 +
 conf/metrics.properties.template                   |     2 +
 conf/spark-env.sh.template                         |     5 +-
 .../benchmarks/AvroReadBenchmark-jdk21-results.txt |   141 +
 .../benchmarks/AvroReadBenchmark-jdk25-results.txt |   141 +
 .../avro/benchmarks/AvroReadBenchmark-results.txt  |   141 +
 .../AvroWriteBenchmark-jdk21-results.txt           |    56 +
 .../AvroWriteBenchmark-jdk25-results.txt           |    56 +
 .../avro/benchmarks/AvroWriteBenchmark-results.txt |    56 +
 connector/avro/pom.xml                             |    86 +
 ...org.apache.spark.sql.sources.DataSourceRegister |    18 +
 .../apache/spark/sql/avro/AvroDataToCatalyst.scala |   148 +
 .../spark/sql/avro/AvroExpressionEvalUtils.scala   |    45 +
 .../apache/spark/sql/avro/CatalystDataToAvro.scala |    69 +
 .../org/apache/spark/sql/avro/SchemaOfAvro.scala   |    69 +
 .../scala/org/apache/spark/sql/avro/package.scala  |     0
 .../spark/sql/v2/avro/AvroDataSourceV2.scala       |     0
 .../sql/v2/avro/AvroPartitionReaderFactory.scala   |   122 +
 .../org/apache/spark/sql/v2/avro/AvroScan.scala    |    76 +
 .../apache/spark/sql/v2/avro/AvroScanBuilder.scala |    55 +
 .../org/apache/spark/sql/v2/avro/AvroTable.scala   |    55 +
 .../org/apache/spark/sql/v2/avro/AvroWrite.scala   |     0
 .../spark/sql/avro/JavaAvroFunctionsSuite.java     |    66 +
 .../test/resources/before_1582_date_v2_4_5.avro    |   Bin
 .../test/resources/before_1582_date_v2_4_6.avro    |   Bin
 .../test/resources/before_1582_date_v3_2_0.avro    |   Bin 0 -> 264 bytes
 .../before_1582_timestamp_micros_v2_4_5.avro       |   Bin
 .../before_1582_timestamp_micros_v2_4_6.avro       |   Bin
 .../before_1582_timestamp_micros_v3_2_0.avro       |   Bin 0 -> 282 bytes
 .../before_1582_timestamp_millis_v2_4_5.avro       |   Bin
 .../before_1582_timestamp_millis_v2_4_6.avro       |   Bin
 .../before_1582_timestamp_millis_v3_2_0.avro       |   Bin 0 -> 308 bytes
 .../avro/src/test/resources/empty_blocks.avro      |     5 +
 connector/avro/src/test/resources/empty_file.avro  |   Bin 0 -> 508 bytes
 .../avro/src/test/resources/episodes.avro          |   Bin
 .../avro/src/test/resources/log4j2.properties      |    31 +
 .../test-random-partitioned/part-r-00000.avro      |   Bin
 .../test-random-partitioned/part-r-00001.avro      |   Bin
 .../test-random-partitioned/part-r-00002.avro      |   Bin
 .../test-random-partitioned/part-r-00003.avro      |   Bin
 .../test-random-partitioned/part-r-00004.avro      |   Bin
 .../test-random-partitioned/part-r-00005.avro      |   Bin
 .../test-random-partitioned/part-r-00006.avro      |   Bin
 .../test-random-partitioned/part-r-00007.avro      |   Bin
 .../test-random-partitioned/part-r-00008.avro      |   Bin
 .../test-random-partitioned/part-r-00009.avro      |   Bin
 .../test-random-partitioned/part-r-00010.avro      |   Bin
 .../avro/src/test/resources/test.avro              |   Bin
 .../avro/src/test/resources/test.avsc              |     0
 .../avro/src/test/resources/test.json              |     0
 .../avro/src/test/resources/test_sub.avsc          |     0
 .../sql/avro/AvroCatalystDataConversionSuite.scala |   438 +
 .../org/apache/spark/sql/avro/AvroCodecSuite.scala |    78 +
 .../apache/spark/sql/avro/AvroFunctionsSuite.scala |   779 +
 .../spark/sql/avro/AvroLogicalTypeInitSuite.scala  |    73 +
 .../spark/sql/avro/AvroLogicalTypeSuite.scala      |   557 +
 .../apache/spark/sql/avro/AvroRowReaderSuite.scala |    99 +
 .../org/apache/spark/sql/avro/AvroScanSuite.scala  |     0
 .../spark/sql/avro/AvroSchemaHelperSuite.scala     |   159 +
 .../org/apache/spark/sql/avro/AvroSerdeSuite.scala |   256 +
 .../org/apache/spark/sql/avro/AvroSuite.scala      |  3521 +++
 .../sql/avro/DeprecatedAvroFunctionsSuite.scala    |   123 +
 .../execution/benchmark/AvroReadBenchmark.scala    |   308 +
 .../execution/benchmark/AvroWriteBenchmark.scala   |   107 +
 .../datasources/AvroReadSchemaSuite.scala          |     0
 connector/docker-integration-tests/README.md       |   133 +
 connector/docker-integration-tests/pom.xml         |   130 +
 .../src/test/resources/db2-krb-setup.sh            |     0
 .../src/test/resources/log4j2.properties           |    41 +
 .../test/resources/mariadb-docker-entrypoint.sh    |    26 +
 .../src/test/resources/mariadb-krb-setup.sh        |    20 +
 .../src/test/resources/postgres-krb-setup.sh       |    21 +
 .../spark/sql/jdbc/DB2DatabaseOnDocker.scala       |    35 +
 .../spark/sql/jdbc/DB2IntegrationSuite.scala       |   284 +
 .../spark/sql/jdbc/DB2KrbIntegrationSuite.scala    |    91 +
 .../spark/sql/jdbc/DockerIntegrationFunSuite.scala |    54 +
 .../sql/jdbc/DockerJDBCIntegrationSuite.scala      |   295 +
 .../sql/jdbc/DockerKrbJDBCIntegrationSuite.scala   |   201 +
 .../spark/sql/jdbc/MariaDBDatabaseOnDocker.scala   |    33 +
 .../sql/jdbc/MariaDBKrbIntegrationSuite.scala      |    65 +
 .../sql/jdbc/MsSQLServerDatabaseOnDocker.scala     |    33 +
 .../sql/jdbc/MsSqlServerIntegrationSuite.scala     |   496 +
 .../spark/sql/jdbc/MySQLDatabaseOnDocker.scala     |    31 +
 .../spark/sql/jdbc/MySQLIntegrationSuite.scala     |   388 +
 .../spark/sql/jdbc/OracleDatabaseOnDocker.scala    |    39 +
 .../spark/sql/jdbc/OracleIntegrationSuite.scala    |   617 +
 .../spark/sql/jdbc/PostgresDatabaseOnDocker.scala  |    36 +
 .../spark/sql/jdbc/PostgresIntegrationSuite.scala  |   622 +
 .../sql/jdbc/PostgresKrbIntegrationSuite.scala     |    61 +
 .../sql/jdbc/SharedJDBCIntegrationSuite.scala      |    89 +
 .../spark/sql/jdbc/StarRocksDatabaseOnDocker.scala |    30 +
 .../spark/sql/jdbc/StarRocksIntegrationSuite.scala |   133 +
 .../jdbc/querytest/CrossDbmsQueryTestSuite.scala   |   184 +
 .../jdbc/querytest/GeneratedSubquerySuite.scala    |   487 +
 .../jdbc/querytest/PostgresSQLQueryTestSuite.scala |    64 +
 .../spark/sql/jdbc/v2/DB2IntegrationSuite.scala    |   126 +
 .../spark/sql/jdbc/v2/DB2NamespaceSuite.scala      |    60 +
 .../sql/jdbc/v2/DockerJDBCIntegrationV2Suite.scala |    70 +
 .../sql/jdbc/v2/MsSqlServerIntegrationSuite.scala  |   294 +
 .../sql/jdbc/v2/MsSqlServerNamespaceSuite.scala    |    63 +
 .../spark/sql/jdbc/v2/MySQLIntegrationSuite.scala  |   346 +
 .../spark/sql/jdbc/v2/MySQLNamespaceSuite.scala    |   102 +
 .../spark/sql/jdbc/v2/OracleIntegrationSuite.scala |   344 +
 .../spark/sql/jdbc/v2/OracleNamespaceSuite.scala   |    82 +
 .../sql/jdbc/v2/PostgresIntegrationSuite.scala     |   416 +
 .../spark/sql/jdbc/v2/PostgresNamespaceSuite.scala |    52 +
 .../spark/sql/jdbc/v2/V2JDBCNamespaceTest.scala    |   133 +
 .../org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala  |  1285 +
 .../MsSqlServerJoinPushdownIntegrationSuite.scala  |    51 +
 .../join/MySQLJoinPushdownIntegrationSuite.scala   |    52 +
 .../join/OracleJoinPushdownIntegrationSuite.scala  |   123 +
 .../PostgresJoinPushdownIntegrationSuite.scala     |    47 +
 .../scala/org/apache/spark/util/DockerUtils.scala  |    68 +
 connector/kafka-0-10-assembly/pom.xml              |   186 +
 connector/kafka-0-10-sql/pom.xml                   |   211 +
 ...org.apache.spark.sql.sources.DataSourceRegister |    18 +
 .../resources/error/kafka-error-conditions.json    |   132 +
 .../spark/sql/kafka010/ConsumerStrategy.scala      |   149 +
 .../org/apache/spark/sql/kafka010/JsonUtils.scala  |   116 +
 .../org/apache/spark/sql/kafka010/KafkaBatch.scala |     0
 .../sql/kafka010/KafkaBatchPartitionReader.scala   |   184 +
 .../spark/sql/kafka010/KafkaBatchWrite.scala       |    68 +
 .../spark/sql/kafka010/KafkaContinuousStream.scala |   254 +
 .../spark/sql/kafka010/KafkaDataWriter.scala       |     0
 .../spark/sql/kafka010/KafkaExceptions.scala       |   298 +
 .../spark/sql/kafka010/KafkaMicroBatchStream.scala |   553 +
 .../sql/kafka010/KafkaOffsetRangeCalculator.scala  |   173 +
 .../spark/sql/kafka010/KafkaOffsetRangeLimit.scala |     0
 .../spark/sql/kafka010/KafkaOffsetReader.scala     |   256 +
 .../sql/kafka010/KafkaOffsetReaderAdmin.scala      |   497 +
 .../sql/kafka010/KafkaOffsetReaderConsumer.scala   |   580 +
 .../sql/kafka010/KafkaRecordToRowConverter.scala   |   108 +
 .../apache/spark/sql/kafka010/KafkaRelation.scala  |    93 +
 .../org/apache/spark/sql/kafka010/KafkaSink.scala  |    43 +
 .../apache/spark/sql/kafka010/KafkaSource.scala    |   432 +
 .../kafka010/KafkaSourceInitialOffsetWriter.scala  |    63 +
 .../spark/sql/kafka010/KafkaSourceOffset.scala     |    80 +
 .../spark/sql/kafka010/KafkaSourceProvider.scala   |   800 +
 .../apache/spark/sql/kafka010/KafkaSourceRDD.scala |   154 +
 .../spark/sql/kafka010/KafkaStreamingWrite.scala   |    75 +
 .../org/apache/spark/sql/kafka010/KafkaWrite.scala |     0
 .../apache/spark/sql/kafka010/KafkaWriteTask.scala |   137 +
 .../apache/spark/sql/kafka010/KafkaWriter.scala    |   127 +
 .../sql/kafka010/consumer/FetchedDataPool.scala    |   186 +
 .../consumer/InternalKafkaConsumerPool.scala       |   229 +
 .../sql/kafka010/consumer/KafkaDataConsumer.scala  |   855 +
 .../apache/spark/sql/kafka010/package-info.java    |     0
 .../org/apache/spark/sql/kafka010/package.scala    |     0
 .../kafka010/producer/CachedKafkaProducer.scala    |    42 +
 .../producer/InternalKafkaProducerPool.scala       |   208 +
 .../kafka-source-initial-offset-future-version.bin |     0
 .../kafka-source-initial-offset-version-2.1.0.bin  |     0
 .../kafka-source-offset-version-2.1.0.txt          |     0
 .../src/test/resources/log4j2.properties           |    37 +
 .../commits/0                                      |     0
 .../metadata                                       |     0
 .../offsets/0                                      |     0
 .../sources/0/0                                    |   Bin
 .../state/0/0/1.delta                              |   Bin
 .../state/0/1/1.delta                              |   Bin
 .../state/0/2/1.delta                              |   Bin
 .../state/0/3/1.delta                              |   Bin
 .../state/0/4/1.delta                              |   Bin
 .../spark/sql/kafka010/ConsumerStrategySuite.scala |   148 +
 .../apache/spark/sql/kafka010/JsonUtilsSuite.scala |     0
 .../sql/kafka010/KafkaContinuousSourceSuite.scala  |   244 +
 .../spark/sql/kafka010/KafkaContinuousTest.scala   |    98 +
 .../sql/kafka010/KafkaDelegationTokenSuite.scala   |   118 +
 .../kafka010/KafkaDontFailOnDataLossSuite.scala    |   290 +
 .../sql/kafka010/KafkaMicroBatchSourceSuite.scala  |  2840 ++
 .../kafka010/KafkaOffsetRangeCalculatorSuite.scala |   336 +
 .../sql/kafka010/KafkaOffsetReaderSuite.scala      |   326 +
 .../kafka010/KafkaRealTimeIntegrationSuite.scala   |   293 +
 .../sql/kafka010/KafkaRealTimeModeSuite.scala      |   776 +
 .../spark/sql/kafka010/KafkaRelationSuite.scala    |   772 +
 .../apache/spark/sql/kafka010/KafkaSinkSuite.scala |   605 +
 .../sql/kafka010/KafkaSourceOffsetSuite.scala      |   106 +
 .../sql/kafka010/KafkaSourceProviderSuite.scala    |   136 +
 .../spark/sql/kafka010/KafkaSparkConfSuite.scala   |     0
 .../org/apache/spark/sql/kafka010/KafkaTest.scala  |    34 +
 .../apache/spark/sql/kafka010/KafkaTestUtils.scala |   668 +
 .../apache/spark/sql/kafka010/RecordBuilder.scala  |    74 +
 .../kafka010/consumer/FetchedDataPoolSuite.scala   |   305 +
 .../consumer/InternalKafkaConsumerPoolSuite.scala  |   286 +
 .../kafka010/consumer/KafkaDataConsumerSuite.scala |   394 +
 .../producer/InternalKafkaProducerPoolSuite.scala  |     0
 connector/kafka-0-10-token-provider/pom.xml        |   109 +
 ...he.spark.security.HadoopDelegationTokenProvider |    18 +
 .../kafka-token-provider-error-conditions.json     |     8 +
 .../apache/spark/kafka010/KafkaConfigUpdater.scala |    95 +
 .../kafka010/KafkaDelegationTokenProvider.scala    |    87 +
 .../apache/spark/kafka010/KafkaRedactionUtil.scala |    52 +
 .../kafka010/KafkaTokenProviderException.scala     |    45 +
 .../spark/kafka010/KafkaTokenSparkConf.scala       |   110 +
 .../org/apache/spark/kafka010/KafkaTokenUtil.scala |   292 +
 .../src/test/resources/log4j2.properties           |    31 +
 .../spark/kafka010/KafkaConfigUpdaterSuite.scala   |   176 +
 .../spark/kafka010/KafkaDelegationTokenTest.scala  |     0
 .../KafkaHadoopDelegationTokenManagerSuite.scala   |     0
 .../spark/kafka010/KafkaRedactionUtilSuite.scala   |     0
 .../spark/kafka010/KafkaTokenSparkConfSuite.scala  |   147 +
 .../spark/kafka010/KafkaTokenUtilSuite.scala       |   292 +
 connector/kafka-0-10/pom.xml                       |   158 +
 .../streaming/kafka010/ConsumerStrategy.scala      |   471 +
 .../kafka010/DirectKafkaInputDStream.scala         |   352 +
 .../streaming/kafka010/KafkaDataConsumer.scala     |   371 +
 .../apache/spark/streaming/kafka010/KafkaRDD.scala |   315 +
 .../streaming/kafka010/KafkaRDDPartition.scala     |     0
 .../spark/streaming/kafka010/KafkaUtils.scala      |   214 +
 .../streaming/kafka010/LocationStrategy.scala      |    71 +
 .../spark/streaming/kafka010/OffsetRange.scala     |     0
 .../streaming/kafka010/PerPartitionConfig.scala    |    47 +
 .../spark/streaming/kafka010/package-info.java     |     0
 .../apache/spark/streaming/kafka010/package.scala  |     0
 .../kafka010/JavaConsumerStrategySuite.java        |    94 +
 .../kafka010/JavaDirectKafkaStreamSuite.java       |   155 +
 .../streaming/kafka010/JavaKafkaRDDSuite.java      |   121 +
 .../kafka010/JavaLocationStrategySuite.java        |    58 +
 .../src/test/resources/log4j2.properties           |    31 +
 .../kafka010/DirectKafkaStreamSuite.scala          |   815 +
 .../kafka010/KafkaDataConsumerSuite.scala          |   162 +
 .../spark/streaming/kafka010/KafkaRDDSuite.scala   |   308 +
 .../spark/streaming/kafka010/KafkaTestUtils.scala  |   348 +
 .../streaming/kafka010/mocks/MockScheduler.scala   |    71 +
 .../spark/streaming/kafka010/mocks/MockTime.scala  |     0
 connector/kinesis-asl-assembly/pom.xml             |   202 +
 connector/kinesis-asl/pom.xml                      |   180 +
 .../streaming/JavaKinesisWordCountASL.java         |   207 +
 .../streaming/kinesis/KinesisInitialPositions.java |    93 +
 .../examples/streaming/kinesis_wordcount_asl.py    |    82 +
 .../src/main/resources/log4j2.properties           |    43 +
 .../examples/streaming/KinesisExampleUtils.scala   |    35 +
 .../examples/streaming/KinesisWordCountASL.scala   |   300 +
 .../streaming/kinesis/KinesisBackedBlockRDD.scala  |   314 +
 .../streaming/kinesis/KinesisCheckpointer.scala    |   146 +
 .../streaming/kinesis/KinesisInputDStream.scala    |   371 +
 .../kinesis/KinesisReadConfigurations.scala        |    78 +
 .../spark/streaming/kinesis/KinesisReceiver.scala  |   393 +
 .../streaming/kinesis/KinesisRecordProcessor.scala |   201 +
 .../kinesis/KinesisUtilsPythonHelper.scala         |   103 +
 .../streaming/kinesis/SparkAWSCredentials.scala    |   189 +
 .../JavaKinesisInputDStreamBuilderSuite.java       |    99 +
 .../src/test/resources/log4j2.properties           |    31 +
 .../kinesis/KPLBasedKinesisTestUtils.scala         |    79 +
 .../kinesis/KinesisBackedBlockRDDSuite.scala       |   260 +
 .../kinesis/KinesisCheckpointerSuite.scala         |   154 +
 .../spark/streaming/kinesis/KinesisFunSuite.scala  |    46 +
 .../kinesis/KinesisInputDStreamBuilderSuite.scala  |   190 +
 .../streaming/kinesis/KinesisReceiverSuite.scala   |   259 +
 .../streaming/kinesis/KinesisStreamSuite.scala     |   434 +
 .../spark/streaming/kinesis/KinesisTestUtils.scala |   312 +
 .../kinesis/SparkAWSCredentialsBuilderSuite.scala  |     0
 connector/profiler/README.md                       |   134 +
 connector/profiler/pom.xml                         |    54 +
 .../org/apache/spark/profiler/ProfilerPlugin.scala |    97 +
 .../apache/spark/profiler/SparkAsyncProfiler.scala |   217 +
 .../scala/org/apache/spark/profiler/package.scala  |    78 +
 connector/protobuf/README.md                       |    36 +
 connector/protobuf/pom.xml                         |   211 +
 .../sql/protobuf/CatalystDataToProtobuf.scala      |    94 +
 .../sql/protobuf/ProtobufDataToCatalyst.scala      |   187 +
 .../spark/sql/protobuf/ProtobufDeserializer.scala  |   574 +
 .../spark/sql/protobuf/ProtobufSerializer.scala    |   375 +
 .../org/apache/spark/sql/protobuf/package.scala    |    20 +
 .../spark/sql/protobuf/utils/ProtobufOptions.scala |   270 +
 .../spark/sql/protobuf/utils/ProtobufUtils.scala   |   405 +
 .../sql/protobuf/utils/SchemaConverters.scala      |   254 +
 .../protobuf/src/test/resources/log4j2.properties  |    39 +
 .../src/test/resources/protobuf/basicmessage.proto |    40 +
 .../test/resources/protobuf/catalyst_types.proto   |    82 +
 .../src/test/resources/protobuf/duration.proto     |    26 +
 .../test/resources/protobuf/functions_suite.proto  |   352 +
 .../src/test/resources/protobuf/nestedenum.proto   |    28 +
 .../test/resources/protobuf/proto2_messages.proto  |   126 +
 .../resources/protobuf/proto2_messages_ext.proto   |    32 +
 .../src/test/resources/protobuf/pyspark_test.proto |    32 +
 .../src/test/resources/protobuf/serde_suite.proto  |    65 +
 .../src/test/resources/protobuf/timestamp.proto    |    26 +
 .../ProtobufCatalystDataConversionSuite.scala      |   443 +
 .../sql/protobuf/ProtobufExtensionsSuite.scala     |   537 +
 .../sql/protobuf/ProtobufFunctionsSuite.scala      |  2308 ++
 .../spark/sql/protobuf/ProtobufSerdeSuite.scala    |   334 +
 .../spark/sql/protobuf/ProtobufTestBase.scala      |    76 +
 connector/spark-ganglia-lgpl/pom.xml               |    48 +
 .../codahale/metrics/ganglia/GangliaReporter.java  |   433 +
 .../apache/spark/metrics/sink/GangliaSink.scala    |     0
 .../benchmarks/ChecksumBenchmark-jdk21-results.txt |    13 +
 .../benchmarks/ChecksumBenchmark-jdk25-results.txt |    13 +
 core/benchmarks/ChecksumBenchmark-results.txt      |    13 +
 .../CoalescedRDDBenchmark-jdk11-results.txt        |    40 -
 .../CoalescedRDDBenchmark-jdk21-results.txt        |    40 +
 .../CoalescedRDDBenchmark-jdk25-results.txt        |    40 +
 core/benchmarks/CoalescedRDDBenchmark-results.txt  |    64 +-
 core/benchmarks/KryoBenchmark-jdk11-results.txt    |    28 -
 core/benchmarks/KryoBenchmark-jdk21-results.txt    |    28 +
 core/benchmarks/KryoBenchmark-jdk25-results.txt    |    28 +
 core/benchmarks/KryoBenchmark-results.txt          |    40 +-
 .../KryoIteratorBenchmark-jdk21-results.txt        |    28 +
 .../KryoIteratorBenchmark-jdk25-results.txt        |    28 +
 core/benchmarks/KryoIteratorBenchmark-results.txt  |    28 +
 .../KryoSerializerBenchmark-jdk11-results.txt      |    12 -
 .../KryoSerializerBenchmark-jdk21-results.txt      |    12 +
 .../KryoSerializerBenchmark-jdk25-results.txt      |    12 +
 .../benchmarks/KryoSerializerBenchmark-results.txt |     8 +-
 .../LZ4TPCDSDataBenchmark-jdk21-results.txt        |    17 +
 .../LZ4TPCDSDataBenchmark-jdk25-results.txt        |    17 +
 core/benchmarks/LZ4TPCDSDataBenchmark-results.txt  |    17 +
 core/benchmarks/LZFBenchmark-jdk21-results.txt     |    19 +
 core/benchmarks/LZFBenchmark-jdk25-results.txt     |    19 +
 core/benchmarks/LZFBenchmark-results.txt           |    19 +
 .../MapStatusesConvertBenchmark-jdk21-results.txt  |    13 +
 .../MapStatusesConvertBenchmark-jdk25-results.txt  |    13 +
 .../MapStatusesConvertBenchmark-results.txt        |    13 +
 .../MapStatusesSerDeserBenchmark-jdk11-results.txt |    66 -
 .../MapStatusesSerDeserBenchmark-jdk21-results.txt |    66 +
 .../MapStatusesSerDeserBenchmark-jdk25-results.txt |    66 +
 .../MapStatusesSerDeserBenchmark-results.txt       |    72 +-
 .../NettyTransportBenchmark-jdk21-results.txt      |   137 +
 .../NettyTransportBenchmark-jdk25-results.txt      |   137 +
 .../benchmarks/NettyTransportBenchmark-results.txt |   137 +
 .../PercentileHeapBenchmark-jdk21-results.txt      |    41 +
 .../PercentileHeapBenchmark-jdk25-results.txt      |    41 +
 .../benchmarks/PercentileHeapBenchmark-results.txt |    41 +
 .../PersistenceEngineBenchmark-jdk21-results.txt   |    18 +
 .../PersistenceEngineBenchmark-jdk25-results.txt   |    18 +
 .../PersistenceEngineBenchmark-results.txt         |    18 +
 .../benchmarks/PlatformBenchmark-jdk21-results.txt |   290 +
 .../benchmarks/PlatformBenchmark-jdk25-results.txt |   290 +
 core/benchmarks/PlatformBenchmark-results.txt      |   290 +
 .../PropertiesCloneBenchmark-jdk11-results.txt     |    40 -
 .../PropertiesCloneBenchmark-jdk21-results.txt     |    40 +
 .../PropertiesCloneBenchmark-jdk25-results.txt     |    40 +
 .../PropertiesCloneBenchmark-results.txt           |    40 +-
 core/benchmarks/RocksDBBenchmark-jdk21-results.txt |   102 +
 core/benchmarks/RocksDBBenchmark-jdk25-results.txt |   102 +
 core/benchmarks/RocksDBBenchmark-results.txt       |   102 +
 core/benchmarks/SorterBenchmark-jdk21-results.txt  |    29 +
 core/benchmarks/SorterBenchmark-jdk25-results.txt  |    29 +
 core/benchmarks/SorterBenchmark-results.txt        |    29 +
 .../XORShiftRandomBenchmark-jdk11-results.txt      |    44 -
 .../XORShiftRandomBenchmark-jdk21-results.txt      |    44 +
 .../XORShiftRandomBenchmark-jdk25-results.txt      |    44 +
 .../benchmarks/XORShiftRandomBenchmark-results.txt |    38 +-
 .../ZStandardBenchmark-jdk11-results.txt           |    27 -
 .../ZStandardBenchmark-jdk21-results.txt           |    82 +
 .../ZStandardBenchmark-jdk25-results.txt           |    82 +
 core/benchmarks/ZStandardBenchmark-results.txt     |    89 +-
 .../ZStandardTPCDSDataBenchmark-jdk21-results.txt  |    49 +
 .../ZStandardTPCDSDataBenchmark-jdk25-results.txt  |    49 +
 .../ZStandardTPCDSDataBenchmark-results.txt        |    49 +
 core/pom.xml                                       |   398 +-
 .../org/apache/spark/SparkFirehoseListener.java    |     2 +
 .../main/java/org/apache/spark/SparkThrowable.java |    42 -
 .../java/org/apache/spark/api/java/Optional.java   |    11 +-
 .../spark/io/NioBufferedFileInputStream.java       |    50 +-
 .../org/apache/spark/io/ReadAheadInputStream.java  |    48 +-
 .../org/apache/spark/memory/MemoryConsumer.java    |    21 +-
 .../apache/spark/memory/SparkOutOfMemoryError.java |    28 +-
 .../org/apache/spark/memory/TaskMemoryManager.java |   289 +-
 .../spark/shuffle/api/ShuffleDriverComponents.java |     9 +
 .../spark/shuffle/checksum/RowBasedChecksum.scala  |    80 +
 .../shuffle/sort/BypassMergeSortShuffleWriter.java |    50 +-
 .../spark/shuffle/sort/ShuffleExternalSorter.java  |    82 +-
 .../spark/shuffle/sort/ShuffleInMemorySorter.java  |     3 +
 .../spark/shuffle/sort/UnsafeShuffleWriter.java    |    70 +-
 .../io/LocalDiskShuffleExecutorComponents.java     |     6 +-
 .../sort/io/LocalDiskShuffleMapOutputWriter.java   |    19 +-
 .../apache/spark/unsafe/map/BytesToBytesMap.java   |    21 +-
 .../org/apache/spark/util/BestEffortLazyVal.java   |    77 +
 .../spark/util/ChildFirstURLClassLoader.java       |     9 +
 .../main/java/org/apache/spark/util/EnumUtil.java  |    13 +-
 .../util/ExposedBufferByteArrayOutputStream.java   |    26 +
 .../spark/util/TransientBestEffortLazyVal.java     |    84 +
 .../org/apache/spark/util/collection/TimSort.java  |    17 -
 .../collection/unsafe/sort/PrefixComparators.java  |     8 +
 .../util/collection/unsafe/sort/RadixSort.java     |     7 +-
 .../unsafe/sort/UnsafeExternalSorter.java          |   299 +-
 .../unsafe/sort/UnsafeInMemorySorter.java          |     9 +-
 .../sort/UnsafeSorterBoundedSpillMerger.java       |   272 +
 .../unsafe/sort/UnsafeSorterSpillReader.java       |    17 +-
 core/src/main/protobuf/buf.yaml                    |    23 +
 .../apache/spark/status/protobuf/store_types.proto |   820 +
 core/src/main/resources/META-INF/LICENSE           |   240 +
 core/src/main/resources/META-INF/NOTICE            |    29 +
 ....apache.spark.deploy.history.EventFilterBuilder |    17 +
 ...he.spark.security.HadoopDelegationTokenProvider |    17 +
 .../org.apache.spark.status.protobuf.ProtobufSerDe |    33 +
 core/src/main/resources/error/README.md            |   255 -
 core/src/main/resources/error/error-classes.json   |   111 -
 .../org/apache/spark/log4j-defaults.properties     |    43 -
 .../apache/spark/ui/static/bootstrap.bundle.min.js |     8 +-
 .../spark/ui/static/bootstrap.bundle.min.js.map    |     1 -
 .../org/apache/spark/ui/static/bootstrap.min.css   |    11 +-
 .../apache/spark/ui/static/bootstrap.min.css.map   |     1 -
 .../org/apache/spark/ui/static/d3-flamegraph.css   |    47 +
 .../apache/spark/ui/static/d3-flamegraph.min.js    |     2 +
 .../resources/org/apache/spark/ui/static/d3.min.js |     7 +-
 .../org/apache/spark/ui/static/dagre-d3.min.js     |  4803 ++-
 .../static/dataTables.bootstrap4.1.10.20.min.css   |     1 -
 .../ui/static/dataTables.bootstrap4.1.10.20.min.js |    11 -
 .../spark/ui/static/dataTables.bootstrap5.min.css  |     5 +
 .../spark/ui/static/dataTables.bootstrap5.min.js   |     4 +
 .../org/apache/spark/ui/static/environmentpage.js  |   264 +
 .../spark/ui/static/executorspage-template.html    |    51 +-
 .../org/apache/spark/ui/static/executorspage.js    |   253 +-
 .../org/apache/spark/ui/static/flamegraph.js       |    51 +
 .../org/apache/spark/ui/static/graphlib-dot.min.js |   351 +-
 .../apache/spark/ui/static/historypage-common.js   |     4 +-
 .../spark/ui/static/historypage-template.html      |    39 +-
 .../org/apache/spark/ui/static/historypage.js      |   117 +-
 .../org/apache/spark/ui/static/images/sort_asc.png |   Bin 160 -> 0 bytes
 .../spark/ui/static/images/sort_asc_disabled.png   |   Bin 148 -> 0 bytes
 .../apache/spark/ui/static/images/sort_both.png    |   Bin 201 -> 0 bytes
 .../apache/spark/ui/static/images/sort_desc.png    |   Bin 158 -> 0 bytes
 .../spark/ui/static/images/sort_desc_disabled.png  |   Bin 146 -> 0 bytes
 .../apache/spark/ui/static/initialize-tooltips.js  |    14 +-
 .../org/apache/spark/ui/static/jquery-3.5.1.min.js |     2 -
 .../apache/spark/ui/static/jquery.blockUI.min.js   |     6 -
 .../spark/ui/static/jquery.cookies.2.2.0.min.js    |    18 -
 .../ui/static/jquery.dataTables.1.10.20.min.css    |     1 -
 .../ui/static/jquery.dataTables.1.10.20.min.js     |   180 -
 .../spark/ui/static/jquery.dataTables.min.css      |     1 +
 .../spark/ui/static/jquery.dataTables.min.js       |     4 +
 .../org/apache/spark/ui/static/jquery.min.js       |     2 +
 .../org/apache/spark/ui/static/jquery.mustache.js  |   682 -
 .../apache/spark/ui/static/jsonFormatter.min.css   |     1 -
 .../apache/spark/ui/static/jsonFormatter.min.js    |     1 -
 .../org/apache/spark/ui/static/log-view.js         |    10 +-
 .../org/apache/spark/ui/static/package.json        |     5 +
 .../org/apache/spark/ui/static/scroll-button.js    |    48 +
 .../org/apache/spark/ui/static/spark-dag-viz.css   |   111 +-
 .../org/apache/spark/ui/static/spark-dag-viz.js    |   219 +-
 .../spark/ui/static/spark-logo-77x50px-hd.png      |   Bin 3077 -> 0 bytes
 .../org/apache/spark/ui/static/spark-logo-rev.svg  |     7 +
 .../org/apache/spark/ui/static/spark-logo.svg      |     7 +
 .../org/apache/spark/ui/static/stagepage.js        |   225 +-
 .../spark/ui/static/stagespage-template.html       |    21 +-
 .../org/apache/spark/ui/static/streaming-page.js   |    96 +-
 .../spark/ui/static/structured-streaming-page.js   |    99 +-
 .../resources/org/apache/spark/ui/static/table.js  |    63 +-
 .../org/apache/spark/ui/static/timeline-view.css   |   222 +-
 .../org/apache/spark/ui/static/timeline-view.js    |    88 +-
 .../resources/org/apache/spark/ui/static/utils.js  |    83 +-
 .../spark/ui/static/vis-timeline-graph2d.min.css   |     2 +-
 .../spark/ui/static/vis-timeline-graph2d.min.js    |    58 +-
 .../apache/spark/ui/static/webui-dataTables.css    |    12 +-
 .../resources/org/apache/spark/ui/static/webui.css |   303 +-
 .../resources/org/apache/spark/ui/static/webui.js  |   184 +-
 .../apache/spark/util/BoundedPriorityQueue.scala   |    75 -
 .../org/apache/spark/util/TimeStampedHashMap.scala |   150 -
 .../apache/spark/util/BoundedPriorityQueue.scala   |    73 -
 .../org/apache/spark/util/TimeStampedHashMap.scala |   143 -
 .../org/apache/spark/BarrierCoordinator.scala      |    42 +-
 .../org/apache/spark/BarrierTaskContext.scala      |    84 +-
 .../org/apache/spark/ContextAwareIterator.scala    |     2 +
 .../scala/org/apache/spark/ContextCleaner.scala    |    23 +-
 .../main/scala/org/apache/spark/Dependency.scala   |   127 +-
 .../main/scala/org/apache/spark/ErrorInfo.scala    |    67 -
 .../apache/spark/ExecutorAllocationClient.scala    |     5 +-
 .../apache/spark/ExecutorAllocationManager.scala   |   128 +-
 .../main/scala/org/apache/spark/FutureAction.scala |    22 +-
 .../scala/org/apache/spark/HeartbeatReceiver.scala |    30 +-
 .../main/scala/org/apache/spark/Heartbeater.scala  |     9 +-
 .../org/apache/spark/InternalAccumulator.scala     |    15 +
 .../scala/org/apache/spark/JobArtifactSet.scala    |   122 +
 .../scala/org/apache/spark/MapOutputTracker.scala  |   507 +-
 .../org/apache/spark/PartitionEvaluator.scala      |    36 +
 .../apache/spark/PartitionEvaluatorFactory.scala   |    38 +
 .../main/scala/org/apache/spark/Partitioner.scala  |    38 +-
 .../main/scala/org/apache/spark/SSLOptions.scala   |   131 +-
 .../scala/org/apache/spark/SecurityManager.scala   |    94 +-
 .../main/scala/org/apache/spark/SparkConf.scala    |   592 +-
 .../main/scala/org/apache/spark/SparkContext.scala |   808 +-
 .../src/main/scala/org/apache/spark/SparkEnv.scala |   263 +-
 .../scala/org/apache/spark/SparkException.scala    |   242 -
 .../spark/SparkFileAlreadyExistsException.scala    |    40 +
 .../main/scala/org/apache/spark/SparkFiles.scala   |     8 +-
 .../org/apache/spark/SparkStatusTracker.scala      |    17 +-
 .../main/scala/org/apache/spark/TaskContext.scala  |   125 +-
 .../scala/org/apache/spark/TaskContextImpl.scala   |   318 +-
 .../scala/org/apache/spark/TaskEndReason.scala     |     2 +-
 .../org/apache/spark/TaskKilledException.scala     |     2 +-
 .../main/scala/org/apache/spark/TestUtils.scala    |   241 +-
 .../org/apache/spark/api/java/JavaHadoopRDD.scala  |     2 +-
 .../apache/spark/api/java/JavaNewHadoopRDD.scala   |     2 +-
 .../org/apache/spark/api/java/JavaPairRDD.scala    |    12 +-
 .../org/apache/spark/api/java/JavaRDDLike.scala    |    25 +-
 .../apache/spark/api/java/JavaSparkContext.scala   |   116 +-
 .../spark/api/java/JavaSparkStatusTracker.scala    |     9 +-
 .../org/apache/spark/api/python/Py4JServer.scala   |     7 +-
 .../apache/spark/api/python/PythonErrorUtils.scala |    41 +
 .../spark/api/python/PythonGatewayServer.scala     |     9 +-
 .../apache/spark/api/python/PythonHadoopUtil.scala |    11 +-
 .../org/apache/spark/api/python/PythonRDD.scala    |   187 +-
 .../org/apache/spark/api/python/PythonRunner.scala |   896 +-
 .../org/apache/spark/api/python/PythonUtils.scala  |   110 +-
 .../spark/api/python/PythonWorkerFactory.scala     |   346 +-
 .../spark/api/python/PythonWorkerLogCapture.scala  |   198 +
 .../spark/api/python/PythonWorkerUtils.scala       |   254 +
 .../org/apache/spark/api/python/SerDeUtil.scala    |    36 +-
 .../spark/api/python/StreamingPythonRunner.scala   |   184 +
 .../python/WriteInputFormatTestDataGenerator.scala |   189 -
 .../scala/org/apache/spark/api/r/BaseRRunner.scala |    14 +-
 .../scala/org/apache/spark/api/r/RAuthHelper.scala |     1 +
 .../scala/org/apache/spark/api/r/RBackend.scala    |     8 +-
 .../org/apache/spark/api/r/RBackendHandler.scala   |    24 +-
 .../main/scala/org/apache/spark/api/r/RRDD.scala   |    11 +-
 .../main/scala/org/apache/spark/api/r/RUtils.scala |     4 +
 .../main/scala/org/apache/spark/api/r/SerDe.scala  |    57 +-
 .../org/apache/spark/broadcast/Broadcast.scala     |    10 +-
 .../apache/spark/broadcast/BroadcastFactory.scala  |     8 +-
 .../apache/spark/broadcast/BroadcastManager.scala  |    12 +-
 .../apache/spark/broadcast/TorrentBroadcast.scala  |   110 +-
 .../spark/broadcast/TorrentBroadcastFactory.scala  |     8 +-
 .../spark/deploy/ApplicationDescription.scala      |    14 +-
 .../scala/org/apache/spark/deploy/Client.scala     |    56 +-
 .../org/apache/spark/deploy/ClientArguments.scala  |     4 +-
 .../org/apache/spark/deploy/DeployMessage.scala    |    16 +-
 .../apache/spark/deploy/DriverTimeoutPlugin.scala  |    64 +
 .../apache/spark/deploy/ExecutorDescription.scala  |     5 +-
 .../spark/deploy/ExecutorFailureTracker.scala      |   104 +
 .../spark/deploy/ExternalShuffleService.scala      |    29 +-
 .../apache/spark/deploy/FaultToleranceTest.scala   |    11 +-
 .../org/apache/spark/deploy/JsonProtocol.scala     |    93 +-
 .../apache/spark/deploy/LocalSparkCluster.scala    |    47 +-
 .../org/apache/spark/deploy/PythonRunner.scala     |    60 +-
 .../org/apache/spark/deploy/RPackageUtils.scala    |    49 +-
 .../scala/org/apache/spark/deploy/RRunner.scala    |     2 +-
 .../spark/deploy/RedirectConsolePlugin.scala       |   137 +
 .../org/apache/spark/deploy/SparkCuratorUtil.scala |     2 +-
 .../org/apache/spark/deploy/SparkHadoopUtil.scala  |   304 +-
 .../org/apache/spark/deploy/SparkPipelines.scala   |   111 +
 .../org/apache/spark/deploy/SparkSubmit.scala      |   946 +-
 .../apache/spark/deploy/SparkSubmitArguments.scala |   178 +-
 .../spark/deploy/StandaloneResourceUtils.scala     |    16 +-
 .../main/scala/org/apache/spark/deploy/Utils.scala |   104 +
 .../spark/deploy/client/StandaloneAppClient.scala  |    60 +-
 .../spark/deploy/history/ApplicationCache.scala    |    61 +-
 .../history/ApplicationHistoryProvider.scala       |     9 +
 .../apache/spark/deploy/history/EventFilter.scala  |     8 +-
 .../deploy/history/EventLogFileCompactor.scala     |     9 +-
 .../spark/deploy/history/EventLogFileReaders.scala |    10 +-
 .../spark/deploy/history/EventLogFileWriters.scala |    46 +-
 .../spark/deploy/history/FsHistoryProvider.scala   |   870 +-
 .../deploy/history/HistoryAppStatusStore.scala     |    16 +-
 .../apache/spark/deploy/history/HistoryPage.scala  |   142 +-
 .../spark/deploy/history/HistoryServer.scala       |    37 +-
 .../deploy/history/HistoryServerArguments.scala    |    91 +-
 .../deploy/history/HistoryServerDiskManager.scala  |    88 +-
 .../history/HistoryServerMemoryManager.scala       |    18 +-
 .../apache/spark/deploy/history/HybridStore.scala  |    50 +-
 .../org/apache/spark/deploy/history/LogPage.scala  |    79 +
 .../spark/deploy/master/ApplicationInfo.scala      |    89 +-
 .../apache/spark/deploy/master/ExecutorDesc.scala  |     3 +-
 .../master/ExecutorResourceDescription.scala       |    32 +
 .../master/FileSystemPersistenceEngine.scala       |    29 +-
 .../org/apache/spark/deploy/master/Master.scala    |   565 +-
 .../spark/deploy/master/MasterArguments.scala      |    15 +-
 .../spark/deploy/master/RecoveryModeFactory.scala  |    29 +-
 .../deploy/master/RocksDBPersistenceEngine.scala   |   108 +
 .../apache/spark/deploy/master/WorkerInfo.scala    |    23 +-
 .../deploy/master/ZooKeeperPersistenceEngine.scala |     2 +-
 .../spark/deploy/master/ui/ApplicationPage.scala   |    56 +-
 .../spark/deploy/master/ui/EnvironmentPage.scala   |   167 +
 .../apache/spark/deploy/master/ui/LogPage.scala    |    78 +
 .../apache/spark/deploy/master/ui/MasterPage.scala |   144 +-
 .../spark/deploy/master/ui/MasterWebUI.scala       |    64 +-
 .../spark/deploy/rest/RestSubmissionClient.scala   |   169 +-
 .../spark/deploy/rest/RestSubmissionServer.scala   |   120 +-
 .../spark/deploy/rest/StandaloneRestServer.scala   |    98 +-
 .../deploy/rest/SubmitRestProtocolRequest.scala    |     6 -
 .../deploy/rest/SubmitRestProtocolResponse.scala   |    30 +
 .../security/HBaseDelegationTokenProvider.scala    |    11 +-
 .../security/HadoopDelegationTokenManager.scala    |    34 +-
 .../security/HadoopFSDelegationTokenProvider.scala |    54 +-
 .../org/apache/spark/deploy/security/README.md     |     2 +-
 .../apache/spark/deploy/worker/CommandUtils.scala  |    27 +-
 .../apache/spark/deploy/worker/DriverRunner.scala  |    28 +-
 .../apache/spark/deploy/worker/DriverWrapper.scala |     3 +-
 .../spark/deploy/worker/ExecutorRunner.scala       |    33 +-
 .../org/apache/spark/deploy/worker/Worker.scala    |   246 +-
 .../spark/deploy/worker/WorkerArguments.scala      |    23 +-
 .../apache/spark/deploy/worker/WorkerWatcher.scala |    26 +-
 .../apache/spark/deploy/worker/ui/LogPage.scala    |    49 +-
 .../apache/spark/deploy/worker/ui/WorkerPage.scala |    53 +-
 .../spark/deploy/worker/ui/WorkerWebUI.scala       |     6 +-
 .../org/apache/spark/errors/SparkCoreErrors.scala  |   357 +-
 .../executor/CoarseGrainedExecutorBackend.scala    |   147 +-
 .../scala/org/apache/spark/executor/Executor.scala |   929 +-
 .../spark/executor/ExecutorClassLoader.scala       |   270 +
 .../apache/spark/executor/ExecutorExitCode.scala   |    11 +
 .../spark/executor/ExecutorLogUrlHandler.scala     |     7 +-
 .../apache/spark/executor/ExecutorMetrics.scala    |     2 +-
 .../spark/executor/ExecutorMetricsPoller.scala     |    21 +-
 .../org/apache/spark/executor/ExecutorSource.scala |    22 +-
 .../org/apache/spark/executor/InputMetrics.scala   |     2 +
 .../spark/executor/ProcfsMetricsGetter.scala       |   119 +-
 .../apache/spark/executor/ShuffleReadMetrics.scala |   143 +
 .../org/apache/spark/executor/TaskMetrics.scala    |    84 +-
 .../spark/input/FixedLengthBinaryInputFormat.scala |     2 +-
 .../apache/spark/input/PortableDataStream.scala    |    12 +-
 .../spark/input/WholeTextFileInputFormat.scala     |     2 +-
 .../spark/input/WholeTextFileRecordReader.scala    |    17 +-
 .../scala/org/apache/spark/internal/Logging.scala  |   252 -
 .../spark/internal/config/ConfigBuilder.scala      |   281 -
 .../apache/spark/internal/config/ConfigEntry.scala |   288 -
 .../spark/internal/config/ConfigProvider.scala     |    64 -
 .../spark/internal/config/ConfigReader.scala       |   121 -
 .../org/apache/spark/internal/config/Deploy.scala  |    78 +-
 .../org/apache/spark/internal/config/History.scala |   142 +-
 .../org/apache/spark/internal/config/Kryo.scala    |     2 +-
 .../org/apache/spark/internal/config/Network.scala |    12 -
 .../org/apache/spark/internal/config/Python.scala  |    86 +
 .../internal/config/SparkConfigProvider.scala      |    35 +
 .../org/apache/spark/internal/config/Status.scala  |     9 +-
 .../org/apache/spark/internal/config/Tests.scala   |    18 +
 .../org/apache/spark/internal/config/UI.scala      |    72 +-
 .../org/apache/spark/internal/config/Worker.scala  |    30 +-
 .../org/apache/spark/internal/config/package.scala |   865 +-
 .../spark/internal/io/FileCommitProtocol.scala     |    11 +-
 .../internal/io/HadoopMapRedCommitProtocol.scala   |     5 +-
 .../io/HadoopMapReduceCommitProtocol.scala         |    28 +-
 .../spark/internal/io/SparkHadoopWriter.scala      |    18 +-
 .../spark/internal/io/SparkHadoopWriterUtils.scala |    26 +-
 .../spark/internal/plugin/PluginContainer.scala    |    21 +-
 .../spark/internal/plugin/PluginEndpoint.scala     |    17 +-
 .../org/apache/spark/io/CompressionCodec.scala     |    67 +-
 .../org/apache/spark/io/HadoopCodecStreams.scala   |    94 +
 .../apache/spark/launcher/LauncherBackend.scala    |     4 +-
 .../spark/launcher/WorkerCommandBuilder.scala      |     2 +-
 .../spark/mapred/SparkHadoopMapRedUtil.scala       |    18 +-
 .../apache/spark/memory/ExecutionMemoryPool.scala  |     9 +-
 .../org/apache/spark/memory/MemoryManager.scala    |    18 +-
 .../apache/spark/memory/StorageMemoryPool.scala    |    13 +-
 .../apache/spark/memory/UnifiedMemoryManager.scala |   285 +-
 .../spark/memory/UnmanagedMemoryConsumer.scala     |    72 +
 .../apache/spark/metrics/ExecutorMetricType.scala  |    34 +-
 .../org/apache/spark/metrics/MetricsConfig.scala   |    11 +-
 .../org/apache/spark/metrics/MetricsSystem.scala   |    45 +-
 .../apache/spark/metrics/sink/GraphiteSink.scala   |    11 +-
 .../apache/spark/metrics/sink/MetricsServlet.scala |     4 +-
 .../spark/metrics/sink/PrometheusServlet.scala     |    17 +-
 .../apache/spark/metrics/sink/StatsdReporter.scala |     6 +-
 .../org/apache/spark/metrics/sink/StatsdSink.scala |    17 +-
 .../spark/metrics/source/AccumulatorSource.scala   |     7 -
 .../apache/spark/metrics/source/JVMCPUSource.scala |     5 +-
 .../spark/network/netty/NettyBlockRpcServer.scala  |    44 +-
 .../network/netty/NettyBlockTransferService.scala  |    33 +-
 .../spark/network/netty/SparkTransportConf.scala   |    24 +-
 core/src/main/scala/org/apache/spark/package.scala |    63 +-
 .../org/apache/spark/partial/CountEvaluator.scala  |     5 +-
 .../spark/partial/GroupedCountEvaluator.scala      |     4 +-
 .../scala/org/apache/spark/paths/SparkPath.scala   |    55 +
 .../org/apache/spark/rdd/AsyncRDDActions.scala     |    28 +-
 .../main/scala/org/apache/spark/rdd/BlockRDD.scala |     2 +-
 .../scala/org/apache/spark/rdd/CartesianRDD.scala  |     7 +-
 .../scala/org/apache/spark/rdd/CheckpointRDD.scala |    10 +-
 .../scala/org/apache/spark/rdd/CoGroupedRDD.scala  |     2 +-
 .../scala/org/apache/spark/rdd/CoalescedRDD.scala  |     9 +-
 .../scala/org/apache/spark/rdd/HadoopRDD.scala     |   166 +-
 .../apache/spark/rdd/InputFileBlockHolder.scala    |    11 +
 .../main/scala/org/apache/spark/rdd/JdbcRDD.scala  |     6 +-
 .../apache/spark/rdd/LocalRDDCheckpointData.scala  |     3 +-
 .../org/apache/spark/rdd/MapPartitionsRDD.scala    |     7 +-
 .../spark/rdd/MapPartitionsWithEvaluatorRDD.scala  |    42 +
 .../scala/org/apache/spark/rdd/NewHadoopRDD.scala  |    98 +-
 .../org/apache/spark/rdd/OrderedRDDFunctions.scala |     2 +-
 .../org/apache/spark/rdd/PairRDDFunctions.scala    |    27 +-
 .../apache/spark/rdd/ParallelCollectionRDD.scala   |     3 +-
 .../spark/rdd/PartitionerAwareUnionRDD.scala       |    61 +-
 .../main/scala/org/apache/spark/rdd/PipedRDD.scala |    33 +-
 core/src/main/scala/org/apache/spark/rdd/RDD.scala |   197 +-
 .../scala/org/apache/spark/rdd/RDDBarrier.scala    |    16 +-
 .../org/apache/spark/rdd/RDDOperationScope.scala   |    26 +-
 .../apache/spark/rdd/ReliableCheckpointRDD.scala   |    30 +-
 .../spark/rdd/ReliableRDDCheckpointData.scala      |     6 +-
 .../spark/rdd/SequenceFileRDDFunctions.scala       |     7 +-
 .../apache/spark/rdd/SortedMergeCoalescedRDD.scala |   180 +
 .../scala/org/apache/spark/rdd/SubtractedRDD.scala |     4 +-
 .../main/scala/org/apache/spark/rdd/UnionRDD.scala |     4 +-
 .../rdd/ZippedPartitionsWithEvaluatorRDD.scala     |    38 +
 .../org/apache/spark/rdd/ZippedWithIndexRDD.scala  |    17 +-
 .../spark/resource/ExecutorResourceRequests.scala  |     4 +-
 .../apache/spark/resource/ResourceAllocator.scala  |   126 +-
 .../resource/ResourceDiscoveryScriptPlugin.scala   |     4 +-
 .../spark/resource/ResourceInformation.scala       |    16 +-
 .../apache/spark/resource/ResourceProfile.scala    |   175 +-
 .../spark/resource/ResourceProfileBuilder.scala    |     8 +-
 .../spark/resource/ResourceProfileManager.scala    |    96 +-
 .../org/apache/spark/resource/ResourceUtils.scala  |    84 +-
 .../spark/resource/TaskResourceRequest.scala       |    10 +-
 .../spark/resource/TaskResourceRequests.scala      |     2 +-
 .../scala/org/apache/spark/rpc/RpcAddress.scala    |    13 +-
 .../scala/org/apache/spark/rpc/RpcEndpoint.scala   |    21 +-
 .../org/apache/spark/rpc/RpcEndpointRef.scala      |     7 -
 .../main/scala/org/apache/spark/rpc/RpcEnv.scala   |    27 +-
 .../scala/org/apache/spark/rpc/RpcTimeout.scala    |     2 +-
 .../org/apache/spark/rpc/netty/Dispatcher.scala    |     9 +-
 .../scala/org/apache/spark/rpc/netty/Inbox.scala   |     7 +-
 .../org/apache/spark/rpc/netty/MessageLoop.scala   |    11 +-
 .../org/apache/spark/rpc/netty/NettyRpcEnv.scala   |    18 +-
 .../spark/rpc/netty/NettyStreamManager.scala       |     9 +
 .../scala/org/apache/spark/rpc/netty/Outbox.scala  |     2 +-
 .../org/apache/spark/scheduler/ActiveJob.scala     |     9 +-
 .../apache/spark/scheduler/AsyncEventQueue.scala   |    21 +-
 .../org/apache/spark/scheduler/DAGScheduler.scala  |  1939 +-
 .../apache/spark/scheduler/DAGSchedulerEvent.scala |    27 +-
 .../spark/scheduler/EventLoggingListener.scala     |    21 +-
 .../spark/scheduler/ExecutorLossReason.scala       |    11 +-
 .../spark/scheduler/ExecutorResourceInfo.scala     |     8 +-
 .../spark/scheduler/ExecutorResourcesAmounts.scala |   217 +
 .../org/apache/spark/scheduler/HealthTracker.scala |    80 +-
 .../apache/spark/scheduler/InputFormatInfo.scala   |     8 +-
 .../org/apache/spark/scheduler/JobResult.scala     |     3 +-
 .../org/apache/spark/scheduler/JobWaiter.scala     |    17 +-
 .../apache/spark/scheduler/LiveListenerBus.scala   |    20 +-
 .../org/apache/spark/scheduler/MapStatus.scala     |    80 +-
 .../org/apache/spark/scheduler/MergeStatus.scala   |    12 +-
 .../spark/scheduler/OutputCommitCoordinator.scala  |    16 +-
 .../scala/org/apache/spark/scheduler/Pool.scala    |     2 +-
 .../apache/spark/scheduler/ReplayListenerBus.scala |    17 +-
 .../org/apache/spark/scheduler/ResultTask.scala    |     7 +-
 .../spark/scheduler/SchedulableBuilder.scala       |   101 +-
 .../apache/spark/scheduler/SchedulerBackend.scala  |     7 +-
 .../apache/spark/scheduler/ShuffleMapStage.scala   |    18 +-
 .../apache/spark/scheduler/ShuffleMapTask.scala    |    22 +-
 .../org/apache/spark/scheduler/SparkListener.scala |    32 +-
 .../org/apache/spark/scheduler/SplitInfo.scala     |     2 +-
 .../scala/org/apache/spark/scheduler/Stage.scala   |    42 +-
 .../org/apache/spark/scheduler/StageInfo.scala     |    16 +-
 .../spark/scheduler/StatsReportListener.scala      |    11 +-
 ...tructuredStreamingIdAwareSchedulerLogging.scala |   278 +
 .../spark/scheduler/SupportsDelegationToken.scala  |    71 +
 .../scala/org/apache/spark/scheduler/Task.scala    |    89 +-
 .../apache/spark/scheduler/TaskDescription.scala   |   107 +-
 .../org/apache/spark/scheduler/TaskInfo.scala      |    45 +-
 .../org/apache/spark/scheduler/TaskResult.scala    |    27 +-
 .../apache/spark/scheduler/TaskResultGetter.scala  |    18 +-
 .../org/apache/spark/scheduler/TaskScheduler.scala |     8 +-
 .../apache/spark/scheduler/TaskSchedulerImpl.scala |   415 +-
 .../scala/org/apache/spark/scheduler/TaskSet.scala |    16 +-
 .../spark/scheduler/TaskSetExcludeList.scala       |    40 +-
 .../apache/spark/scheduler/TaskSetManager.scala    |   514 +-
 .../org/apache/spark/scheduler/WorkerOffer.scala   |     4 +-
 .../cluster/CoarseGrainedClusterMessage.scala      |    28 +-
 .../cluster/CoarseGrainedSchedulerBackend.scala    |   354 +-
 .../spark/scheduler/cluster/ExecutorData.scala     |     6 +-
 .../spark/scheduler/cluster/ExecutorInfo.scala     |    20 +-
 .../cluster/StandaloneSchedulerBackend.scala       |   150 +-
 .../spark/scheduler/dynalloc/ExecutorMonitor.scala |    58 +-
 .../scheduler/local/LocalSchedulerBackend.scala    |    35 +-
 .../apache/spark/security/CryptoStreamUtils.scala  |   147 +-
 .../security/ShellBasedGroupsMappingProvider.scala |     5 +-
 .../apache/spark/security/SocketAuthHelper.scala   |    14 +-
 .../apache/spark/security/SocketAuthServer.scala   |    65 +-
 .../spark/serializer/GenericAvroSerializer.scala   |    13 +-
 .../apache/spark/serializer/JavaSerializer.scala   |    57 +-
 .../apache/spark/serializer/KryoSerializer.scala   |   160 +-
 .../spark/serializer/SerializationDebugger.scala   |    35 +-
 .../apache/spark/serializer/SerializerHelper.scala |    56 +
 .../spark/serializer/SerializerManager.scala       |    15 +-
 .../spark/shuffle/BlockStoreShuffleReader.scala    |    61 +-
 .../spark/shuffle/FetchFailedException.scala       |     2 +-
 .../spark/shuffle/IndexShuffleBlockResolver.scala  |   154 +-
 .../apache/spark/shuffle/MigratableResolver.scala  |     5 +
 .../apache/spark/shuffle/ShuffleBlockPusher.scala  |    87 +-
 .../spark/shuffle/ShuffleBlockResolver.scala       |     8 +
 .../spark/shuffle/ShuffleChecksumUtils.scala       |    93 +
 .../org/apache/spark/shuffle/ShuffleManager.scala  |    26 +-
 .../shuffle/ShufflePartitionPairsWriter.scala      |     5 +-
 .../spark/shuffle/ShuffleWriteProcessor.scala      |    34 +-
 .../scala/org/apache/spark/shuffle/metrics.scala   |    10 +
 .../spark/shuffle/sort/SortShuffleManager.scala    |    35 +-
 .../spark/shuffle/sort/SortShuffleWriter.scala     |    30 +-
 .../apache/spark/status/AppStatusListener.scala    |   164 +-
 .../org/apache/spark/status/AppStatusSource.scala  |     2 +-
 .../org/apache/spark/status/AppStatusStore.scala   |   284 +-
 .../org/apache/spark/status/AppStatusUtils.scala   |    12 +
 .../apache/spark/status/ElementTrackingStore.scala |    12 +-
 .../scala/org/apache/spark/status/KVUtils.scala    |   167 +-
 .../scala/org/apache/spark/status/LiveEntity.scala |   204 +-
 .../spark/status/api/v1/ApiRootResource.scala      |    14 +-
 .../status/api/v1/ApplicationListResource.scala    |    11 +-
 .../spark/status/api/v1/JacksonMessageWriter.scala |     8 +-
 .../status/api/v1/OneApplicationResource.scala     |    44 +-
 .../spark/status/api/v1/PrometheusResource.scala   |    17 +-
 .../spark/status/api/v1/SimpleDateParam.scala      |     7 +-
 .../spark/status/api/v1/StagesResource.scala       |    12 +-
 .../scala/org/apache/spark/status/api/v1/api.scala |   148 +-
 .../protobuf/AccumulableInfoSerializer.scala       |    50 +
 .../status/protobuf/AppSummarySerializer.scala     |    38 +
 ...plicationEnvironmentInfoWrapperSerializer.scala |   156 +
 .../ApplicationInfoWrapperSerializer.scala         |   116 +
 .../status/protobuf/CachedQuantileSerializer.scala |   118 +
 .../protobuf/ExecutorMetricsSerializer.scala       |    38 +
 .../protobuf/ExecutorStageSummarySerializer.scala  |    72 +
 .../ExecutorStageSummaryWrapperSerializer.scala    |    46 +
 .../ExecutorSummaryWrapperSerializer.scala         |   181 +
 .../status/protobuf/JobDataWrapperSerializer.scala |   112 +
 .../protobuf/JobExecutionStatusSerializer.scala    |    43 +
 .../protobuf/KVStoreProtobufSerializer.scala       |    56 +
 .../spark/status/protobuf/PoolDataSerializer.scala |    41 +
 .../protobuf/ProcessSummaryWrapperSerializer.scala |    72 +
 .../spark/status/protobuf/ProtobufSerDe.scala      |    45 +
 .../RDDOperationGraphWrapperSerializer.scala       |   146 +
 .../protobuf/RDDStorageInfoWrapperSerializer.scala |   129 +
 .../ResourceProfileWrapperSerializer.scala         |    39 +
 .../SpeculationStageSummaryWrapperSerializer.scala |    64 +
 .../protobuf/StageDataWrapperSerializer.scala      |   716 +
 .../status/protobuf/StageStatusSerializer.scala    |    45 +
 .../protobuf/StreamBlockDataSerializer.scala       |    54 +
 .../protobuf/TaskDataWrapperSerializer.scala       |   142 +
 .../org/apache/spark/status/protobuf/Utils.scala   |    46 +
 .../scala/org/apache/spark/status/storeTypes.scala |   106 +-
 .../scala/org/apache/spark/storage/BlockId.scala   |    81 +
 .../apache/spark/storage/BlockInfoManager.scala    |   569 +-
 .../org/apache/spark/storage/BlockManager.scala    |   534 +-
 .../spark/storage/BlockManagerDecommissioner.scala |   172 +-
 .../org/apache/spark/storage/BlockManagerId.scala  |     6 +-
 .../spark/storage/BlockManagerManagedBuffer.scala  |     2 +
 .../apache/spark/storage/BlockManagerMaster.scala  |    78 +-
 .../spark/storage/BlockManagerMasterEndpoint.scala |   607 +-
 .../spark/storage/BlockManagerMessages.scala       |    17 +-
 .../storage/BlockManagerStorageEndpoint.scala      |    50 +-
 .../spark/storage/BlockReplicationPolicy.scala     |     6 +-
 ...avedOnDecommissionedBlockManagerException.scala |     2 +-
 .../apache/spark/storage/DiskBlockManager.scala    |   128 +-
 .../spark/storage/DiskBlockObjectWriter.scala      |    95 +-
 .../scala/org/apache/spark/storage/DiskStore.scala |    42 +-
 .../org/apache/spark/storage/FallbackStorage.scala |    70 +-
 .../apache/spark/storage/LogBlockIdGenerator.scala |    60 +
 .../org/apache/spark/storage/LogBlockWriter.scala  |   182 +
 .../scala/org/apache/spark/storage/LogLine.scala   |    53 +
 .../spark/storage/PushBasedFetchHelper.scala       |    59 +-
 .../scala/org/apache/spark/storage/RDDInfo.scala   |     6 +-
 .../apache/spark/storage/RollingLogWriter.scala    |   115 +
 .../storage/ShuffleBlockFetcherIterator.scala      |   412 +-
 .../ShuffleManagerNotInitializedException.scala    |    27 +
 .../org/apache/spark/storage/StorageLevel.scala    |   239 -
 .../org/apache/spark/storage/StorageUtils.scala    |    33 +-
 .../org/apache/spark/storage/TopologyMapper.scala  |     3 +-
 .../apache/spark/storage/memory/MemoryStore.scala  |   116 +-
 .../org/apache/spark/ui/ConsoleProgressBar.scala   |    27 +-
 .../main/scala/org/apache/spark/ui/CspNonce.scala  |    44 +
 .../scala/org/apache/spark/ui/DriverLogPage.scala  |   155 +
 .../scala/org/apache/spark/ui/GraphUIData.scala    |    37 +-
 .../org/apache/spark/ui/HttpSecurityFilter.scala   |    90 +-
 .../main/scala/org/apache/spark/ui/JWSFilter.scala |    80 +
 .../scala/org/apache/spark/ui/JettyUtils.scala     |   241 +-
 .../scala/org/apache/spark/ui/PagedTable.scala     |    91 +-
 .../main/scala/org/apache/spark/ui/SparkUI.scala   |    52 +-
 .../main/scala/org/apache/spark/ui/ToolTips.scala  |    13 +-
 .../main/scala/org/apache/spark/ui/UIUtils.scala   |   392 +-
 .../org/apache/spark/ui/UIWorkloadGenerator.scala  |    22 +-
 .../src/main/scala/org/apache/spark/ui/WebUI.scala |    25 +-
 .../org/apache/spark/ui/env/EnvironmentPage.scala  |   232 +-
 .../spark/ui/exec/ExecutorHeapHistogramPage.scala  |    89 +
 .../spark/ui/exec/ExecutorThreadDumpPage.scala     |   155 +-
 .../org/apache/spark/ui/exec/ExecutorsTab.scala    |    47 +-
 .../spark/ui/flamegraph/FlamegraphNode.scala       |    50 +
 .../org/apache/spark/ui/jobs/AllJobsPage.scala     |    87 +-
 .../org/apache/spark/ui/jobs/AllStagesPage.scala   |    21 +-
 .../scala/org/apache/spark/ui/jobs/JobPage.scala   |    90 +-
 .../scala/org/apache/spark/ui/jobs/JobsTab.scala   |     4 +-
 .../scala/org/apache/spark/ui/jobs/PoolPage.scala  |    13 +-
 .../scala/org/apache/spark/ui/jobs/PoolTable.scala |    11 +-
 .../org/apache/spark/ui/jobs/StageDataUtil.scala   |    39 +
 .../scala/org/apache/spark/ui/jobs/StagePage.scala |   389 +-
 .../org/apache/spark/ui/jobs/StageTable.scala      |    46 +-
 .../scala/org/apache/spark/ui/jobs/StagesTab.scala |     6 +-
 .../spark/ui/jobs/TaskDetailsClassNames.scala      |    35 -
 .../apache/spark/ui/jobs/TaskThreadDumpPage.scala  |   107 +
 .../apache/spark/ui/scope/RDDOperationGraph.scala  |    41 +-
 .../org/apache/spark/ui/storage/RDDPage.scala      |    13 +-
 .../org/apache/spark/ui/storage/StoragePage.scala  |    18 +-
 .../org/apache/spark/util/AccumulatorV2.scala      |    25 +-
 .../util/BlockingThreadPoolExecutorService.scala   |   124 +
 .../apache/spark/util/BoundedPriorityQueue.scala   |    71 +
 .../main/scala/org/apache/spark/util/Clock.scala   |     4 +-
 .../org/apache/spark/util/ClosureCleaner.scala     |   876 -
 .../org/apache/spark/util/CommandLineUtils.scala   |     8 +-
 .../org/apache/spark/util/DependencyUtils.scala    |   105 +-
 .../spark/util/DirectByteBufferOutputStream.scala  |   108 +
 .../scala/org/apache/spark/util/Distribution.scala |     8 +-
 .../scala/org/apache/spark/util/EventLoop.scala    |     5 +-
 .../org/apache/spark/util/HadoopFSUtils.scala      |   176 +-
 .../scala/org/apache/spark/util/JsonProtocol.scala |  1936 +-
 .../apache/spark/util/LastAttemptAccumulator.scala |   984 +
 .../main/scala/org/apache/spark/util/LazyTry.scala |    74 +
 .../scala/org/apache/spark/util/ListenerBus.scala  |    15 +-
 .../scala/org/apache/spark/util/MetricUtils.scala  |   108 +
 .../apache/spark/util/NonFateSharingCache.scala    |   110 +
 .../apache/spark/util/PeriodicCheckpointer.scala   |     7 +-
 .../scala/org/apache/spark/util/RpcUtils.scala     |    25 +-
 .../spark/util/SerializableConfiguration.scala     |    12 +
 .../apache/spark/util/ShutdownHookManager.scala    |    31 +-
 .../scala/org/apache/spark/util/SignalUtils.scala  |    14 +-
 .../org/apache/spark/util/SizeEstimator.scala      |    88 +-
 .../apache/spark/util/SparkClosureCleaner.scala    |    55 +
 .../org/apache/spark/util/SparkExitCode.scala      |    24 +
 .../apache/spark/util/SparkFatalException.scala    |    27 -
 .../spark/util/SparkUncaughtExceptionHandler.scala |    74 +-
 .../scala/org/apache/spark/util/StatCounter.scala  |     8 +-
 .../org/apache/spark/util/StubClassLoader.scala    |    96 +
 .../scala/org/apache/spark/util/ThreadUtils.scala  |   186 +-
 .../apache/spark/util/UninterruptibleThread.scala  |   128 +-
 .../main/scala/org/apache/spark/util/Utils.scala   |  1546 +-
 .../spark/util/collection/AppendOnlyMap.scala      |     2 +-
 .../org/apache/spark/util/collection/BitSet.scala  |    41 +-
 .../spark/util/collection/CompactBuffer.scala      |     4 +-
 .../util/collection/ExternalAppendOnlyMap.scala    |    67 +-
 .../spark/util/collection/ExternalSorter.scala     |   108 +-
 .../spark/util/collection/ImmutableBitSet.scala    |     2 +-
 .../apache/spark/util/collection/MedianHeap.scala  |    93 -
 .../apache/spark/util/collection/OpenHashMap.scala |    27 +-
 .../apache/spark/util/collection/OpenHashSet.scala |    46 +-
 .../util/collection/PartitionedPairBuffer.scala    |     2 +-
 .../spark/util/collection/PercentileHeap.scala     |    81 +
 .../util/collection/PrimitiveKeyOpenHashMap.scala  |   133 -
 .../apache/spark/util/collection/Spillable.scala   |    31 +-
 .../org/apache/spark/util/collection/Utils.scala   |    63 +-
 .../apache/spark/util/io/ChunkedByteBuffer.scala   |    88 +-
 .../util/io/ChunkedByteBufferFileRegion.scala      |     2 +-
 .../apache/spark/util/logging/DriverLogger.scala   |    77 +-
 .../apache/spark/util/logging/FileAppender.scala   |    31 +-
 .../spark/util/logging/RollingFileAppender.scala   |    35 +-
 .../apache/spark/util/logging/RollingPolicy.scala  |    11 +-
 .../apache/spark/util/random/RandomSampler.scala   |     2 +-
 .../util/random/StratifiedSamplingUtils.scala      |     6 +-
 .../org/apache/spark/util/taskListeners.scala      |    10 +
 .../java/org/apache/spark/JavaJdbcRDDSuite.java    |    24 +-
 .../org/apache/spark/api/java/OptionalSuite.java   |    81 +-
 .../spark/io/GenericFileInputStreamSuite.java      |    24 +-
 .../spark/io/NioBufferedInputStreamSuite.java      |     4 +-
 .../apache/spark/io/ReadAheadInputStreamSuite.java |     4 +-
 .../apache/spark/launcher/SparkLauncherSuite.java  |    56 +-
 .../spark/memory/TaskMemoryManagerSuite.java       |   176 +-
 .../apache/spark/memory/TestMemoryConsumer.java    |     6 +-
 .../spark/resource/JavaResourceProfileSuite.java   |    10 +-
 .../spark/serializer/ContainsProxyClass.java       |    50 +
 .../shuffle/sort/PackedRecordPointerSuite.java     |    20 +-
 .../shuffle/sort/ShuffleInMemorySorterSuite.java   |    33 +-
 .../shuffle/sort/UnsafeShuffleWriterSuite.java     |   159 +-
 .../spark/unsafe/PlatformBenchmarkTestUtils.java   |    37 +
 .../unsafe/map/AbstractBytesToBytesMapSuite.java   |   214 +-
 .../spark/util/SerializableConfigurationSuite.java |    12 +-
 .../unsafe/sort/UnsafeExternalSorterSuite.java     |   318 +-
 .../unsafe/sort/UnsafeInMemorySorterSuite.java     |    24 +-
 .../test/org/apache/spark/Java8RDDAPISuite.java    |   104 +-
 .../java/test/org/apache/spark/JavaAPISuite.java   |    83 +-
 .../org/apache/spark/JavaSparkContextSuite.java    |     2 +-
 .../apache/spark/JavaTaskContextCompileCheck.java  |     8 +
 .../app_environment_expectation.json               |   288 +-
 .../application_list_json_expectation.json         |   389 +-
 .../complete_stage_list_json_expectation.json      |   345 +-
 .../completed_app_list_json_expectation.json       |   389 +-
 .../excludeOnFailure_for_stage_expectation.json    |  1153 +-
 ...xcludeOnFailure_node_for_stage_expectation.json |  1598 +-
 .../executor_list_json_expectation.json            |    73 +-
 ...ist_with_executor_metrics_json_expectation.json |   382 +-
 .../executor_memory_usage_expectation.json         |   386 +-
 ...executor_node_excludeOnFailure_expectation.json |   390 +-
 ...e_excludeOnFailure_unexcluding_expectation.json |   342 +-
 .../executor_resource_information_expectation.json |   198 +-
 .../failed_stage_list_json_expectation.json        |   117 +-
 ...from_multi_attempt_app_json_1__expectation.json |    19 +-
 ...from_multi_attempt_app_json_2__expectation.json |    19 +-
 .../job_list_json_expectation.json                 |    57 +-
 .../limit_app_list_json_expectation.json           |    74 +-
 .../maxDate2_app_list_json_expectation.json        |    22 +-
 .../maxDate_app_list_json_expectation.json         |    44 +-
 .../maxEndDate_app_list_json_expectation.json      |   142 +-
 ...e_and_maxEndDate_app_list_json_expectation.json |    76 +-
 .../minDate_app_list_json_expectation.json         |   345 +-
 ...e_and_maxEndDate_app_list_json_expectation.json |    78 +-
 .../minEndDate_app_list_json_expectation.json      |   285 +-
 .../miscellaneous_process_expectation.json         |    12 +-
 .../multiple_resource_profiles_expectation.json    |    81 +-
 .../one_app_json_expectation.json                  |    22 +-
 .../one_app_multi_attempt_json_expectation.json    |    38 +-
 .../one_job_json_expectation.json                  |    19 +-
 .../one_rdd_storage_json_expectation.json          |    10 +-
 ..._json_details_with_failed_task_expectation.json |   174 +-
 .../one_stage_attempt_json_expectation.json        |   712 +-
 .../one_stage_json_expectation.json                |   712 +-
 .../one_stage_json_with_details_expectation.json   |   712 +-
 ...ne_stage_json_with_partitionId_expectation.json |   760 +
 .../stage_list_json_expectation.json               |   462 +-
 ...age_list_with_accumulable_json_expectation.json |   123 +-
 .../stage_list_with_peak_metrics_expectation.json  |   349 +-
 .../stage_task_list_expectation.json               |  1180 +-
 ...from_multi_attempt_app_json_1__expectation.json |   552 +-
 ...from_multi_attempt_app_json_2__expectation.json |   552 +-
 ...e_task_list_w__offset___length_expectation.json |  2956 +-
 .../stage_task_list_w__sortBy_expectation.json     |  1180 +-
 ...__sortBy_short_names___runtime_expectation.json |  1180 +-
 ...w__sortBy_short_names__runtime_expectation.json |  1180 +-
 ...st_w__status___offset___length_expectation.json |   118 +-
 ...___sortBy_short_names__runtime_expectation.json |  1180 +-
 .../stage_task_list_w__status_expectation.json     |   670 +-
 ...age_task_list_with_partitionId_expectation.json |   497 +
 ...sk_summary_w__custom_quantiles_expectation.json |    36 +-
 ...ge_task_summary_w_shuffle_read_expectation.json |    36 +-
 ...e_task_summary_w_shuffle_write_expectation.json |    36 +-
 .../stage_with_accumulable_json_expectation.json   |   794 +-
 .../stage_with_peak_metrics_expectation.json       |  1421 +-
 ...stage_with_speculation_summary_expectation.json |   599 +
 .../stage_with_summaries_expectation.json          |  1602 +-
 ...succeeded_failed_job_list_json_expectation.json |    57 +-
 .../succeeded_job_list_json_expectation.json       |    38 +-
 ...g.apache.spark.scheduler.ExternalClusterManager |    17 +
 ...he.spark.security.HadoopDelegationTokenProvider |    17 +
 core/src/test/resources/ProcfsMetrics/487713/stat  |     1 +
 core/src/test/resources/TestUDTF.jar               |   Bin 1328 -> 0 bytes
 core/src/test/resources/certchain.pem              |    17 +
 core/src/test/resources/key.pem                    |    29 +
 core/src/test/resources/log4j.properties           |    36 -
 core/src/test/resources/log4j2.properties          |    39 +
 .../application_1656321732247_0006_1               |    10 +
 .../application_1656321732247_0006_2               |     9 +
 .../application_1656321732247_0006_1               |     9 +
 .../application_1656321732247_0006_2               |    10 +
 .../resources/spark-events/app-20161116163331-0000 |     2 +-
 .../resources/spark-events/app-20200706201101-0003 |    96 +-
 .../spark-events/application_1628109047826_1317105 |    52 +
 .../.appstatus_local-1766844910796.crc             |   Bin 0 -> 8 bytes
 .../appstatus_local-1766844910796                  |     0
 .../events_1_local-1766844910796                   |    15 +
 .../resources/spark-events/local-1642039451826     |   115 +
 core/src/test/resources/unencrypted-certchain.pem  |    21 +
 core/src/test/resources/unencrypted-key.pem        |    28 +
 .../scala/org/apache/spark/AccumulatorSuite.scala  |    38 +-
 .../spark/BarrierStageOnSubmittedSuite.scala       |     2 +-
 .../scala/org/apache/spark/CheckpointSuite.scala   |    42 +-
 .../org/apache/spark/ContextCleanerSuite.scala     |     8 +-
 .../scala/org/apache/spark/DistributedSuite.scala  |    32 +-
 .../test/scala/org/apache/spark/DriverSuite.scala  |     2 +-
 .../spark/ExecutorAllocationManagerSuite.scala     |   155 +-
 .../apache/spark/ExternalShuffleServiceSuite.scala |   223 +-
 .../apache/spark/FeatureSpecSparkTestSuite.scala   |    51 +
 .../test/scala/org/apache/spark/FileSuite.scala    |   114 +-
 .../org/apache/spark/FlatSpecSparkTestSuite.scala  |    47 +
 .../org/apache/spark/FreeSpecSparkTestSuite.scala  |    51 +
 .../org/apache/spark/FunSpecSparkTestSuite.scala   |    49 +
 .../org/apache/spark/HeartbeatReceiverSuite.scala  |    38 +-
 .../org/apache/spark/JobArtifactSetSuite.scala     |   124 +
 .../org/apache/spark/JobCancellationSuite.scala    |   484 +-
 .../scala/org/apache/spark/LocalRootDirsTest.scala |    10 +-
 .../org/apache/spark/MapOutputTrackerSuite.scala   |   450 +-
 .../apache/spark/MapStatusesConvertBenchmark.scala |    92 +
 .../spark/MapStatusesSerDeserBenchmark.scala       |    15 +-
 .../scala/org/apache/spark/PartitioningSuite.scala |    33 +-
 .../scala/org/apache/spark/ProcessTestUtils.scala  |    37 +
 .../org/apache/spark/PropSpecSparkTestSuite.scala  |    47 +
 .../org/apache/spark/RefSpecSparkTestSuite.scala   |    51 +
 .../scala/org/apache/spark/SSLOptionsSuite.scala   |   150 +-
 .../org/apache/spark/SecurityManagerSuite.scala    |    40 +-
 .../org/apache/spark/SerializerTestUtils.scala     |    44 +
 .../org/apache/spark/SharedSparkContext.scala      |     5 +-
 .../scala/org/apache/spark/ShuffleNettySuite.scala |    37 +-
 .../spark/ShuffleOldFetchProtocolSuite.scala       |     4 +-
 .../test/scala/org/apache/spark/ShuffleSuite.scala |    45 +-
 .../scala/org/apache/spark/SortShuffleSuite.scala  |    20 +-
 .../scala/org/apache/spark/SparkConfSuite.scala    |    80 +-
 .../org/apache/spark/SparkContextInfoSuite.scala   |    11 +-
 .../spark/SparkContextSchedulerCreationSuite.scala |    18 +-
 .../scala/org/apache/spark/SparkContextSuite.scala |   294 +-
 .../scala/org/apache/spark/SparkFunSuite.scala     |   207 +-
 .../scala/org/apache/spark/SparkTestSuite.scala    |   591 +
 .../org/apache/spark/SparkThrowableSuite.scala     |   782 +-
 .../spark/SslExternalShuffleServiceSuite.scala     |    53 +
 .../org/apache/spark/SslShuffleNettySuite.scala    |    28 +
 .../org/apache/spark/StatusTrackerSuite.scala      |    44 +
 .../org/apache/spark/TempLocalSparkContext.scala   |     2 +-
 .../TestShuffleDataIOWithMockedComponents.scala    |    53 +
 .../test/scala/org/apache/spark/ThreadAudit.scala  |     4 +-
 .../scala/org/apache/spark/ThreadingSuite.scala    |     4 +-
 .../scala/org/apache/spark/UnpersistSuite.scala    |     6 +-
 .../org/apache/spark/WordSpecSparkTestSuite.scala  |    51 +
 .../org/apache/spark/api/java/JavaUtilsSuite.scala |     4 +-
 .../spark/api/python/PythonHadoopUtilSuite.scala   |    79 +
 .../apache/spark/api/python/PythonRDDSuite.scala   |    18 +-
 .../api/python/PythonWorkerFactorySuite.scala      |   112 +
 .../python/WriteInputFormatTestDataGenerator.scala |   189 +
 .../org/apache/spark/api/r/BaseRRunnerSuite.scala  |    36 +
 .../org/apache/spark/benchmark/Benchmark.scala     |    29 +-
 .../org/apache/spark/benchmark/BenchmarkBase.scala |    11 +-
 .../org/apache/spark/benchmark/Benchmarks.scala    |     4 +-
 .../apache/spark/broadcast/BroadcastSuite.scala    |    33 +-
 .../spark/deploy/DecommissionWorkerSuite.scala     |    17 +-
 .../org/apache/spark/deploy/DeployTestUtils.scala  |    46 +-
 .../spark/deploy/ExecutorFailureTrackerSuite.scala |   103 +
 .../deploy/ExternalShuffleServiceDbSuite.scala     |    22 +-
 .../ExternalShuffleServiceMetricsSuite.scala       |     2 +-
 .../org/apache/spark/deploy/IvyTestUtils.scala     |   406 -
 .../apache/spark/deploy/JsonProtocolSuite.scala    |    69 +-
 .../spark/deploy/LogUrlsStandaloneSuite.scala      |     6 +-
 .../apache/spark/deploy/PythonRunnerSuite.scala    |    30 +-
 .../apache/spark/deploy/RPackageUtilsSuite.scala   |    23 +-
 .../apache/spark/deploy/SparkHadoopUtilSuite.scala |   165 +-
 .../apache/spark/deploy/SparkPipelinesSuite.scala  |   212 +
 .../org/apache/spark/deploy/SparkSubmitSuite.scala |   748 +-
 .../apache/spark/deploy/SparkSubmitTestUtils.scala |   107 +
 .../spark/deploy/SparkSubmitUtilsSuite.scala       |   307 -
 .../deploy/StandaloneDynamicAllocationSuite.scala  |    32 +-
 .../spark/deploy/client/AppClientSuite.scala       |    75 +-
 .../deploy/history/ApplicationCacheSuite.scala     |    98 +-
 .../history/BasicEventFilterBuilderSuite.scala     |    16 +-
 .../history/ChromeUIHistoryServerSuite.scala       |    20 +-
 .../history/EventLogFileCompactorSuite.scala       |     3 +-
 .../deploy/history/EventLogFileReadersSuite.scala  |    45 +-
 .../deploy/history/EventLogFileWritersSuite.scala  |   187 +-
 .../spark/deploy/history/EventLogTestHelper.scala  |    13 +-
 .../deploy/history/FsHistoryProviderSuite.scala    |  1005 +-
 .../history/HistoryServerArgumentsSuite.scala      |    21 +-
 .../history/HistoryServerDiskManagerSuite.scala    |    64 +-
 .../deploy/history/HistoryServerPageSuite.scala    |   119 +
 .../spark/deploy/history/HistoryServerSuite.scala  |   337 +-
 .../spark/deploy/history/HybridStoreSuite.scala    |    65 +-
 .../history/RealBrowserUIHistoryServerSuite.scala  |    23 +-
 .../deploy/master/MasterDecommissionSuite.scala    |    62 +
 .../apache/spark/deploy/master/MasterSuite.scala   |  1025 +-
 .../spark/deploy/master/MasterSuiteBase.scala      |   598 +
 .../spark/deploy/master/MasterWorkerUISuite.scala  |   161 +
 .../deploy/master/PersistenceEngineBenchmark.scala |   185 +
 .../deploy/master/PersistenceEngineSuite.scala     |    79 +-
 .../apache/spark/deploy/master/RecoverySuite.scala |   588 +
 .../deploy/master/ResourceProfilesSuite.scala      |    66 +
 .../spark/deploy/master/WorkerSelectionSuite.scala |    81 +
 .../deploy/master/ui/ApplicationPageSuite.scala    |    86 +
 .../spark/deploy/master/ui/MasterWebUISuite.scala  |    29 +-
 .../master/ui/ReadOnlyMasterWebUISuite.scala       |   122 +
 .../deploy/rest/StandaloneRestSubmitSuite.scala    |   265 +-
 .../deploy/rest/SubmitRestProtocolSuite.scala      |    34 +-
 .../HadoopDelegationTokenManagerSuite.scala        |    30 +-
 .../spark/deploy/worker/CommandUtilsSuite.scala    |    28 +-
 .../spark/deploy/worker/DriverRunnerTest.scala     |     2 +-
 .../spark/deploy/worker/ExecutorRunnerTest.scala   |    16 +-
 .../apache/spark/deploy/worker/WorkerSuite.scala   |    88 +-
 .../spark/executor/ClassLoaderIsolationSuite.scala |   252 +
 .../CoarseGrainedExecutorBackendSuite.scala        |   354 +-
 .../spark/executor/ExecutorClassLoaderSuite.scala  |   355 +
 .../executor/ExecutorMetricsPollerSuite.scala      |    10 +-
 .../ExecutorSideSessionManagementSuite.scala       |   233 +
 .../org/apache/spark/executor/ExecutorSuite.scala  |   272 +-
 .../spark/executor/ProcfsMetricsGetterSuite.scala  |    67 +-
 .../input/WholeTextFileInputFormatSuite.scala      |    11 +-
 .../input/WholeTextFileRecordReaderSuite.scala     |   107 +-
 .../org/apache/spark/internal/LoggingSuite.scala   |    44 +-
 .../spark/internal/config/ConfigEntrySuite.scala   |   179 +-
 .../spark/internal/config/ConfigReaderSuite.scala  |     2 +-
 .../internal/plugin/PluginContainerSuite.scala     |   170 +-
 .../io/ChunkedByteBufferFileRegionSuite.scala      |     4 +-
 .../apache/spark/io/ChunkedByteBufferSuite.scala   |    58 +-
 .../apache/spark/io/CompressionCodecSuite.scala    |    44 +-
 .../apache/spark/io/LZ4TPCDSDataBenchmark.scala    |    78 +
 .../scala/org/apache/spark/io/LZFBenchmark.scala   |    93 +
 .../org/apache/spark/io/TPCDSDataBenchmark.scala   |    64 +
 .../org/apache/spark/io/ZStandardBenchmark.scala   |    61 +-
 .../spark/io/ZStandardTPCDSDataBenchmark.scala     |   116 +
 .../spark/launcher/LauncherBackendSuite.scala      |     3 +-
 .../apache/spark/memory/MemoryManagerSuite.scala   |    26 +-
 .../apache/spark/memory/MemoryTestingUtils.scala   |     1 +
 .../apache/spark/memory/TestMemoryManager.scala    |     6 +-
 .../spark/memory/TestMemoryManagerSuite.scala      |     2 +-
 .../spark/memory/UnifiedMemoryManagerSuite.scala   |   286 +
 .../spark/metrics/InputOutputMetricsSuite.scala    |     4 +-
 .../apache/spark/metrics/MetricsSystemSuite.scala  |    23 +-
 .../spark/metrics/sink/GraphiteSinkSuite.scala     |    44 +-
 .../metrics/sink/PrometheusServletSuite.scala      |     4 +-
 .../spark/metrics/sink/StatsdSinkSuite.scala       |    30 +
 .../spark/network/BlockTransferServiceSuite.scala  |     2 +
 .../spark/network/NettyTransportBenchmark.scala    |   743 +
 .../network/netty/NettyBlockRpcServerSuite.scala   |    59 +
 .../netty/NettyBlockTransferSecuritySuite.scala    |    61 +-
 .../netty/NettyBlockTransferServiceSuite.scala     |     8 +-
 .../apache/spark/rdd/AsyncRDDActionsSuite.scala    |    13 +-
 .../apache/spark/rdd/CoalescedRDDBenchmark.scala   |     8 +-
 .../org/apache/spark/rdd/DoubleRDDSuite.scala      |     3 +-
 .../org/apache/spark/rdd/HadoopRDDSuite.scala      |    30 +
 .../scala/org/apache/spark/rdd/JdbcRDDSuite.scala  |     4 +-
 .../apache/spark/rdd/LocalCheckpointSuite.scala    |     6 +-
 .../apache/spark/rdd/PairRDDFunctionsSuite.scala   |    75 +-
 .../spark/rdd/ParallelCollectionSplitSuite.scala   |    19 +-
 .../spark/rdd/PartitionwiseSampledRDDSuite.scala   |     5 +-
 .../scala/org/apache/spark/rdd/PipedRDDSuite.scala |    35 +-
 .../org/apache/spark/rdd/RDDCleanerSuite.scala     |     8 +-
 .../apache/spark/rdd/RDDOperationScopeSuite.scala  |     2 +-
 .../test/scala/org/apache/spark/rdd/RDDSuite.scala |   203 +-
 .../spark/rdd/SortedMergeCoalescedRDDSuite.scala   |   219 +
 .../scala/org/apache/spark/rdd/SortingSuite.scala  |    36 +-
 .../apache/spark/rdd/ZippedPartitionsSuite.scala   |     4 +-
 .../resource/ResourceDiscoveryPluginSuite.scala    |     5 +-
 .../resource/ResourceProfileManagerSuite.scala     |    88 +-
 .../spark/resource/ResourceProfileSuite.scala      |   231 +-
 .../apache/spark/resource/ResourceUtilsSuite.scala |   255 +-
 .../org/apache/spark/rpc/RpcAddressSuite.scala     |    28 +
 .../scala/org/apache/spark/rpc/RpcEnvSuite.scala   |    81 +-
 .../apache/spark/rpc/netty/NettyRpcEnvSuite.scala  |    20 +-
 .../apache/spark/scheduler/AQEShuffledRDD.scala    |     4 +-
 .../spark/scheduler/BarrierTaskContextSuite.scala  |    33 +-
 .../CoarseGrainedSchedulerBackendSuite.scala       |   373 +-
 .../apache/spark/scheduler/DAGSchedulerSuite.scala |  3760 ++-
 .../scheduler/EventLoggingListenerSuite.scala      |   121 +-
 .../scheduler/ExecutorResourceInfoSuite.scala      |   125 +-
 .../spark/scheduler/ExecutorResourceUtils.scala    |    28 +
 .../scheduler/ExecutorResourcesAmountsSuite.scala  |   537 +
 .../scheduler/ExternalClusterManagerSuite.scala    |     6 +-
 .../org/apache/spark/scheduler/FakeTask.scala      |    26 +-
 .../scheduler/HealthTrackerIntegrationSuite.scala  |     4 +-
 .../spark/scheduler/HealthTrackerSuite.scala       |    27 +-
 .../apache/spark/scheduler/MapStatusSuite.scala    |   100 +-
 .../spark/scheduler/NotSerializableFakeTask.scala  |     4 +-
 .../OutputCommitCoordinatorIntegrationSuite.scala  |     2 +-
 .../scheduler/OutputCommitCoordinatorSuite.scala   |    26 +-
 .../org/apache/spark/scheduler/PoolSuite.scala     |    44 +-
 .../spark/scheduler/ReplayListenerSuite.scala      |    30 +-
 .../scheduler/SchedulerIntegrationSuite.scala      |   136 +-
 .../spark/scheduler/SparkListenerSuite.scala       |   103 +-
 .../scheduler/SparkListenerWithClusterSuite.scala  |     1 +
 ...uredStreamingIdAwareSchedulerLoggingSuite.scala |   317 +
 .../apache/spark/scheduler/TaskContextSuite.scala  |   525 +-
 .../spark/scheduler/TaskDescriptionSuite.scala     |    38 +-
 .../spark/scheduler/TaskResultGetterSuite.scala    |    23 +-
 .../spark/scheduler/TaskSchedulerImplSuite.scala   |   929 +-
 .../spark/scheduler/TaskSetExcludelistSuite.scala  |     3 +-
 .../spark/scheduler/TaskSetManagerSuite.scala      |   731 +-
 .../WorkerDecommissionExtendedSuite.scala          |     4 +-
 .../scheduler/dynalloc/ExecutorMonitorSuite.scala  |    67 +-
 .../spark/security/CryptoStreamUtilsSuite.scala    |    51 +-
 .../spark/security/SocketAuthHelperSuite.scala     |    12 +-
 .../serializer/GenericAvroSerializerSuite.scala    |    16 +
 .../spark/serializer/JavaSerializerSuite.scala     |    28 +-
 .../apache/spark/serializer/KryoBenchmark.scala    |     4 +-
 .../spark/serializer/KryoIteratorBenchmark.scala   |   120 +
 .../spark/serializer/KryoSerializerBenchmark.scala |    12 +-
 .../KryoSerializerDistributedSuite.scala           |     2 +-
 .../KryoSerializerResizableOutputSuite.scala       |    16 +-
 .../spark/serializer/KryoSerializerSuite.scala     |    75 +-
 .../serializer/SerializationDebuggerSuite.scala    |    24 +-
 .../shuffle/BlockStoreShuffleReaderSuite.scala     |     3 +-
 .../apache/spark/shuffle/ChecksumBenchmark.scala   |    54 +
 .../shuffle/HostLocalShuffleReadingSuite.scala     |     7 +-
 .../spark/shuffle/ShuffleBlockPusherSuite.scala    |   141 +-
 .../spark/shuffle/ShuffleChecksumTestHelper.scala  |    57 +-
 .../shuffle/ShuffleDriverComponentsSuite.scala     |     7 +-
 .../checksum/OutputStreamRowBasedChecksum.scala    |    64 +
 .../sort/BypassMergeSortShuffleWriterSuite.scala   |    74 +-
 .../sort/IndexShuffleBlockResolverSuite.scala      |    24 +-
 .../shuffle/sort/ShuffleExternalSorterSuite.scala  |    99 +-
 .../shuffle/sort/SortShuffleWriterSuite.scala      |    70 +-
 .../io/LocalDiskShuffleMapOutputWriterSuite.scala  |    10 +-
 .../spark/status/AppStatusListenerSuite.scala      |   246 +-
 .../apache/spark/status/AppStatusStoreSuite.scala  |   269 +-
 .../apache/spark/status/AppStatusUtilsSuite.scala  |    13 +
 .../spark/status/AutoCleanupLiveUIDirSuite.scala   |    56 +
 .../spark/status/ListenerEventsTestHelper.scala    |    11 +-
 .../org/apache/spark/status/LiveEntitySuite.scala  |   129 +
 .../spark/status/api/v1/SimpleDateParamSuite.scala |     3 +-
 .../protobuf/KVStoreProtobufSerializerSuite.scala  |  1700 +
 .../spark/storage/BlockInfoManagerSuite.scala      |    88 +-
 .../BlockManagerDecommissionIntegrationSuite.scala |   165 +-
 .../BlockManagerDecommissionUnitSuite.scala        |   173 +-
 .../spark/storage/BlockManagerInfoSuite.scala      |     4 +-
 .../spark/storage/BlockManagerMasterSuite.scala    |    66 +-
 .../storage/BlockManagerReplicationSuite.scala     |    87 +-
 .../apache/spark/storage/BlockManagerSuite.scala   |   717 +-
 .../storage/BlockReplicationPolicySuite.scala      |     2 +-
 .../spark/storage/DiskBlockManagerSuite.scala      |    68 +-
 .../spark/storage/DiskBlockObjectWriterSuite.scala |   107 +-
 .../org/apache/spark/storage/DiskStoreSuite.scala  |    16 +-
 .../spark/storage/FallbackStorageSuite.scala       |   146 +-
 .../spark/storage/FlatmapIteratorSuite.scala       |     6 +-
 .../org/apache/spark/storage/LocalDirsSuite.scala  |     2 +-
 .../apache/spark/storage/LogBlockWriterSuite.scala |   137 +
 .../apache/spark/storage/MemoryStoreSuite.scala    |    16 +-
 .../storage/PartiallySerializedBlockSuite.scala    |    31 +-
 .../storage/ShuffleBlockFetcherIteratorSuite.scala |   382 +-
 .../storage/SslBlockManagerReplicationSuite.scala  |    40 +
 .../org/apache/spark/storage/StorageSuite.scala    |    23 -
 .../apache/spark/ui/ChromeUISeleniumSuite.scala    |     3 +-
 .../scala/org/apache/spark/ui/CspNonceSuite.scala  |    84 +
 .../org/apache/spark/ui/DriverLogPageSuite.scala   |    65 +
 .../apache/spark/ui/HttpSecurityFilterSuite.scala  |    84 +-
 .../scala/org/apache/spark/ui/JWSFilterSuite.scala |   112 +
 .../spark/ui/RealBrowserUISeleniumSuite.scala      |    53 +-
 .../scala/org/apache/spark/ui/StagePageSuite.scala |   160 -
 .../org/apache/spark/ui/UISeleniumSuite.scala      |   183 +-
 .../test/scala/org/apache/spark/ui/UISuite.scala   |   136 +-
 .../scala/org/apache/spark/ui/UIUtilsSuite.scala   |    67 +-
 .../apache/spark/ui/env/EnvironmentPageSuite.scala |    66 +
 .../spark/ui/scope/RDDOperationGraphSuite.scala    |    10 +-
 .../apache/spark/ui/storage/StoragePageSuite.scala |    27 +-
 .../apache/spark/unsafe/PlatformBenchmark.scala    |   606 +
 .../apache/spark/util/ArrayImplicitsSuite.scala    |    50 +
 .../apache/spark/util/BestEffortLazyValSuite.scala |   120 +
 .../apache/spark/util/ClosureCleanerSuite.scala    |    19 +-
 .../apache/spark/util/ClosureCleanerSuite2.scala   |    11 +-
 .../spark/util/CompletionIteratorSuite.scala       |     8 +-
 .../apache/spark/util/DependencyUtilsSuite.scala   |     6 +
 .../util/DirectByteBufferOutputStreamSuite.scala   |    59 +
 .../org/apache/spark/util/EventLoopSuite.scala     |     2 +-
 .../org/apache/spark/util/FileAppenderSuite.scala  |    80 +-
 .../org/apache/spark/util/HadoopFSUtilsSuite.scala |    32 +
 .../org/apache/spark/util/JsonProtocolSuite.scala  |  1202 +-
 .../scala/org/apache/spark/util/KeyLockSuite.scala |     4 +-
 .../scala/org/apache/spark/util/LazyTrySuite.scala |   151 +
 .../org/apache/spark/util/MetricUtilsSuite.scala   |    51 +
 .../spark/util/MutableURLClassLoaderSuite.scala    |     5 +-
 .../org/apache/spark/util/NextIteratorSuite.scala  |    10 +-
 .../spark/util/NonFateSharingCacheSuite.scala      |   161 +
 .../spark/util/PropertiesCloneBenchmark.scala      |     4 +-
 .../org/apache/spark/util/SizeEstimatorSuite.scala |   163 +-
 .../util/SparkUncaughtExceptionHandlerSuite.scala  |    55 +-
 .../scala/org/apache/spark/util/SslTestUtils.scala |    36 +
 .../org/apache/spark/util/ThreadUtilsSuite.scala   |   163 +-
 .../spark/util/TimeStampedHashMapSuite.scala       |   179 -
 .../util/TransientBestEffortLazyValSuite.scala     |   116 +
 .../spark/util/UninterruptibleThreadSuite.scala    |    57 +-
 .../scala/org/apache/spark/util/UtilsSuite.scala   |   557 +-
 .../org/apache/spark/util/VersionUtilsSuite.scala  |   115 -
 .../spark/util/collection/AppendOnlyMapSuite.scala |     6 +-
 .../apache/spark/util/collection/BitSetSuite.scala |    30 +
 .../collection/ExternalAppendOnlyMapSuite.scala    |    15 +-
 .../util/collection/ExternalSorterSpillSuite.scala |     3 +-
 .../util/collection/ExternalSorterSuite.scala      |     2 +-
 .../spark/util/collection/MedianHeapSuite.scala    |    66 -
 .../spark/util/collection/OpenHashMapSuite.scala   |    54 +-
 .../spark/util/collection/OpenHashSetSuite.scala   |    39 +
 .../util/collection/PercentileHeapBenchmark.scala  |    74 +
 .../util/collection/PercentileHeapSuite.scala      |    60 +
 .../collection/PrimitiveKeyOpenHashMapSuite.scala  |   126 -
 .../spark/util/collection/SizeTrackerSuite.scala   |     8 +-
 .../spark/util/collection/SorterBenchmark.scala    |   232 +
 .../apache/spark/util/collection/SorterSuite.scala |   136 +-
 .../unsafe/sort/PrefixComparatorsSuite.scala       |     2 +-
 .../collection/unsafe/sort/RadixSortSuite.scala    |    21 +-
 .../io/ChunkedByteBufferOutputStreamSuite.scala    |    11 +-
 .../spark/util/kvstore/RocksDBBenchmark.scala      |   547 +
 .../spark/util/logging/DriverLoggerSuite.scala     |    55 +-
 .../spark/util/random/RandomSamplerSuite.scala     |     7 +-
 .../util/random/XORShiftRandomBenchmark.scala      |     4 +-
 dev/.gitignore                                     |     2 -
 dev/.rat-excludes                                  |    61 +-
 dev/.scalafmt.conf                                 |     8 +-
 dev/ansible-for-test-node/README.md                |    25 -
 .../deploy-jenkins-worker.yml                      |     8 -
 dev/ansible-for-test-node/roles/common/README.md   |     4 -
 .../roles/common/tasks/main.yml                    |     4 -
 .../roles/common/tasks/setup_local_userspace.yml   |     8 -
 .../roles/common/tasks/system_packages.yml         |    73 -
 .../roles/jenkins-worker/README.md                 |    15 -
 .../roles/jenkins-worker/defaults/main.yml         |    30 -
 .../files/python_environments/base-py3-pip.txt     |     3 -
 .../files/python_environments/base-py3-spec.txt    |    21 -
 .../files/python_environments/py36.txt             |    49 -
 .../files/python_environments/spark-py2-pip.txt    |     8 -
 .../files/python_environments/spark-py36-spec.txt  |   119 -
 .../files/python_environments/spark-py3k-spec.txt  |    42 -
 .../files/scripts/jenkins-gitcache-cron            |     7 -
 .../files/util_scripts/kill_zinc_nailgun.py        |    60 -
 .../files/util_scripts/post_github_pr_comment.py   |    81 -
 .../files/util_scripts/session_lock_resource.py    |   152 -
 .../roles/jenkins-worker/files/worker-limits.conf  |     5 -
 .../roles/jenkins-worker/tasks/cleanup.yml         |    12 -
 .../jenkins-worker/tasks/install_anaconda.yml      |    79 -
 .../tasks/install_build_packages.yml               |    21 -
 .../roles/jenkins-worker/tasks/install_docker.yml  |    33 -
 .../jenkins-worker/tasks/install_minikube.yml      |    16 -
 .../tasks/install_spark_build_packages.yml         |   183 -
 .../jenkins-worker/tasks/jenkins_userspace.yml     |   119 -
 .../roles/jenkins-worker/tasks/main.yml            |    22 -
 .../roles/jenkins-worker/vars/main.yml             |     9 -
 dev/appveyor-guide.md                              |   186 -
 dev/appveyor-install-dependencies.ps1              |   128 -
 dev/change-scala-version.sh                        |    32 +-
 dev/check-license                                  |     5 +-
 dev/check-protos.py                                |    87 +
 dev/check_pyspark_custom_errors.py                 |   189 +
 dev/checkstyle-suppressions.xml                    |    32 +-
 dev/checkstyle.xml                                 |   156 +-
 dev/connect-gen-protos.sh                          |    27 +
 dev/connect-jvm-client-mima-check                  |    70 +
 dev/create-release/do-release-docker.sh            |    36 +-
 dev/create-release/do-release.sh                   |    10 +-
 dev/create-release/generate-contributors.py        |   196 +-
 dev/create-release/generate-llms-txt.py            |   198 +
 dev/create-release/known_translations              |   418 -
 dev/create-release/release-build.sh                |   691 +-
 dev/create-release/release-tag.sh                  |    20 +-
 dev/create-release/release-util.sh                 |    50 +-
 dev/create-release/releaseutils.py                 |   172 +-
 dev/create-release/spark-rm/Dockerfile             |   116 +-
 dev/create-release/spark-rm/Dockerfile.base        |   112 +
 dev/create-release/translate-contributors.py       |   261 -
 dev/create-release/vote.tmpl                       |     2 +-
 dev/create_jira_and_branch.py                      |   118 +
 dev/create_spark_jira.py                           |    75 +
 dev/deps/spark-deps-hadoop-2.7-hive-2.3            |   246 -
 dev/deps/spark-deps-hadoop-3-hive-2.3              |   290 +
 dev/deps/spark-deps-hadoop-3.2-hive-2.3            |   214 -
 dev/eslint.js                                      |    48 +
 dev/eslint.json                                    |    24 -
 dev/free_disk_space                                |    59 +
 dev/free_disk_space_container                      |    40 +
 dev/gen-protos.sh                                  |   127 +
 dev/generate_srs_registry.py                       |   382 +
 dev/github_jira_sync.py                            |   201 -
 dev/infra/Dockerfile                               |   154 +
 dev/is-changed.py                                  |   100 +
 dev/java-file-header                               |    16 +
 dev/lint-java                                      |     2 +-
 dev/lint-js                                        |     3 +-
 dev/lint-python                                    |   332 +-
 dev/lint-scala                                     |    25 +
 dev/make-distribution.sh                           |   116 +-
 dev/merge_spark_pr.py                              |   727 +-
 dev/mima                                           |    10 +-
 dev/package-lock.json                              |  3141 +-
 dev/package.json                                   |     8 +-
 dev/pip-sanity-check.py                            |     7 +-
 dev/protobuf-breaking-changes-check.sh             |    46 +
 dev/py-cleanup                                     |    31 +
 dev/reformat-python                                |    12 +-
 dev/requirements.txt                               |    87 +-
 dev/run-pip-tests                                  |   140 +-
 dev/run-tests                                      |     4 +-
 dev/run-tests-jenkins                              |    37 -
 dev/run-tests-jenkins.py                           |   239 -
 dev/run-tests.py                                   |   474 +-
 dev/sbt-checkstyle                                 |     2 +-
 dev/scalafmt                                       |     3 +-
 dev/scalastyle                                     |    61 +-
 dev/spark-test-image-util/docs/build-docs          |    73 +
 dev/spark-test-image-util/docs/run-in-container    |    37 +
 dev/spark-test-image/connect-gen-protos/Dockerfile |    63 +
 dev/spark-test-image/docs/Dockerfile               |    99 +
 dev/spark-test-image/lint/Dockerfile               |   109 +
 dev/spark-test-image/python-310/Dockerfile         |    74 +
 dev/spark-test-image/python-311/Dockerfile         |    73 +
 .../python-312-classic-only/Dockerfile             |    68 +
 .../python-312-pandas-3/Dockerfile                 |    70 +
 dev/spark-test-image/python-312/Dockerfile         |    69 +
 dev/spark-test-image/python-313/Dockerfile         |    73 +
 dev/spark-test-image/python-314-nogil/Dockerfile   |    68 +
 dev/spark-test-image/python-314/Dockerfile         |    73 +
 dev/spark-test-image/python-minimum/Dockerfile     |    70 +
 dev/spark-test-image/python-ps-minimum/Dockerfile  |    70 +
 dev/spark-test-image/sparkr/Dockerfile             |    82 +
 dev/spark_jira_utils.py                            |   103 +
 dev/sparktestsupport/__init__.py                   |     2 +-
 dev/sparktestsupport/modules.py                    |  1258 +-
 dev/sparktestsupport/shellutils.py                 |     6 +-
 dev/sparktestsupport/toposort.py                   |    27 +-
 dev/sparktestsupport/utils.py                      |   191 +
 dev/streaming-gen-protos.sh                        |    27 +
 dev/structured_logging_style.py                    |    94 +
 dev/test-dependencies.sh                           |    42 +-
 dev/tests/pr_merge_ability.sh                      |    39 -
 dev/tests/pr_public_classes.sh                     |    73 -
 dev/tox.ini                                        |    52 +-
 docs/.bundle/config                                |     2 +
 docs/404.md                                        |    28 +
 docs/Gemfile                                       |    16 +-
 docs/Gemfile.lock                                  |    88 +-
 docs/README.md                                     |    81 +-
 docs/_config.yml                                   |    18 +-
 docs/_data/menu-mllib.yaml                         |     2 +-
 docs/_data/menu-sql.yaml                           |    51 +-
 docs/_data/menu-streaming.yaml                     |    57 +
 docs/_includes/nav-left-wrapper-ml.html            |     1 +
 docs/_includes/nav-left-wrapper-streaming.html     |    22 +
 docs/_includes/nav-left.html                       |     2 +-
 docs/_layouts/404.html                             |   157 -
 docs/_layouts/global.html                          |   268 +-
 docs/_layouts/redirect.html                        |    29 +
 docs/_plugins/build-error-docs.py                  |   152 +
 docs/_plugins/build_api_docs.rb                    |   291 +
 docs/_plugins/conditonal_includes.rb               |    71 +
 docs/_plugins/copy_api_dirs.rb                     |   192 -
 docs/_plugins/include_example.rb                   |     4 +-
 docs/_plugins/production_tag.rb                    |    22 +-
 docs/app-dev-spark-connect.md                      |   270 +
 docs/building-spark.md                             |   131 +-
 docs/cloud-integration.md                          |   129 +-
 docs/cluster-overview.md                           |    16 +-
 docs/configuration.md                              |  1400 +-
 docs/control-flow/case-stmt.md                     |   102 +
 docs/control-flow/close-stmt.md                    |   147 +
 docs/control-flow/compound-stmt.md                 |   213 +
 docs/control-flow/fetch-stmt.md                    |   194 +
 docs/control-flow/for-stmt.md                      |    92 +
 docs/control-flow/if-stmt.md                       |    70 +
 docs/control-flow/iterate-stmt.md                  |    70 +
 docs/control-flow/leave-stmt.md                    |    71 +
 docs/control-flow/loop-stmt.md                     |    83 +
 docs/control-flow/open-stmt.md                     |   162 +
 docs/control-flow/repeat-stmt.md                   |    84 +
 docs/control-flow/while-stmt.md                    |    83 +
 docs/core-migration-guide.md                       |    74 +
 docs/css/custom.css                                |  1028 +
 docs/css/docsearch.min.css                         |     2 +
 docs/css/docsearch.min.css.map                     |     1 +
 docs/css/main.css                                  |   420 -
 docs/declarative-pipelines-programming-guide.md    |   561 +
 docs/graphx-programming-guide.md                   |     6 +-
 docs/hardware-provisioning.md                      |     3 +-
 docs/img/AllJobsPageDetail1.png                    |   Bin 239175 -> 20567 bytes
 docs/img/AllStagesPageDetail6.png                  |   Bin 106909 -> 163423 bytes
 docs/img/async-progress.png                        |   Bin 0 -> 29770 bytes
 docs/img/declarative-pipelines-dataflow-graph.png  |   Bin 0 -> 132100 bytes
 docs/img/extending-spark-connect-labelled.png      |   Bin 0 -> 131995 bytes
 docs/img/extending-spark-connect.png               |   Bin 0 -> 118823 bytes
 docs/img/pyspark-components.png                    |   Bin 32727 -> 0 bytes
 docs/img/pyspark-components.pptx                   |   Bin 354673 -> 353804 bytes
 docs/img/pyspark-machine_learning.png              |   Bin 0 -> 8709 bytes
 docs/img/pyspark-pandas_api_on_spark.png           |   Bin 0 -> 8732 bytes
 docs/img/pyspark-python_spark_connect_client.png   |   Bin 0 -> 10181 bytes
 docs/img/pyspark-spark_core_and_rdds.png           |   Bin 0 -> 11267 bytes
 docs/img/pyspark-spark_sql_and_dataframes.png      |   Bin 0 -> 9729 bytes
 docs/img/pyspark-structured_streaming.png          |   Bin 0 -> 9111 bytes
 docs/img/pyspark-udf-profile.png                   |   Bin 0 -> 281384 bytes
 docs/img/spark-connect-api.png                     |   Bin 0 -> 808246 bytes
 docs/img/spark-connect-communication.png           |   Bin 0 -> 566977 bytes
 docs/img/spark-connect-gotchas-antipattern.png     |   Bin 0 -> 171837 bytes
 docs/img/spark-logo-hd.png                         |   Bin 11306 -> 0 bytes
 docs/img/spark-logo-reverse.png                    |   Bin 21547 -> 0 bytes
 docs/img/type-precedence-list.png                  |   Bin 133793 -> 104106 bytes
 docs/index.md                                      |    95 +-
 docs/job-scheduling.md                             |    52 +-
 docs/js/api-docs.js                                |     4 +-
 docs/js/api-javadocs.js                            |     6 +-
 docs/js/main.js                                    |    57 +-
 docs/js/vendor/bootstrap.bundle.min.js             |     8 +-
 docs/js/vendor/bootstrap.bundle.min.js.map         |     2 +-
 docs/js/vendor/docsearch.min.js                    |     2 +
 docs/js/vendor/jquery-3.5.1.min.js                 |     2 -
 docs/js/vendor/jquery.min.js                       |     2 +
 docs/migration-guide.md                            |     2 +-
 docs/ml-classification-regression.md               |   347 +-
 docs/ml-clustering.md                              |    84 +-
 docs/ml-collaborative-filtering.md                 |    55 +-
 docs/ml-datasource.md                              |    78 +-
 docs/ml-features.md                                |   675 +-
 docs/ml-frequent-pattern-mining.md                 |    42 +-
 docs/ml-guide.md                                   |     7 +-
 docs/ml-linalg-guide.md                            |    17 +-
 docs/ml-migration-guide.md                         |    51 +-
 docs/ml-pipeline.md                                |    40 +-
 docs/ml-security.md                                |    74 +
 docs/ml-statistics.md                              |    43 +-
 docs/ml-tuning.md                                  |    28 +-
 docs/mllib-classification-regression.md            |     8 +-
 docs/mllib-clustering.md                           |   122 +-
 docs/mllib-collaborative-filtering.md              |    36 +-
 docs/mllib-data-types.md                           |   429 +-
 docs/mllib-decision-tree.md                        |    44 +-
 docs/mllib-dimensionality-reduction.md             |    43 +-
 docs/mllib-ensembles.md                            |    70 +-
 docs/mllib-evaluation-metrics.md                   |    66 +-
 docs/mllib-feature-extraction.md                   |    70 +-
 docs/mllib-frequent-pattern-mining.md              |    31 +-
 docs/mllib-guide.md                                |     5 +-
 docs/mllib-isotonic-regression.md                  |    39 +-
 docs/mllib-linear-methods.md                       |    91 +-
 docs/mllib-naive-bayes.md                          |    31 +-
 docs/mllib-pmml-model-export.md                    |     8 +-
 docs/mllib-statistics.md                           |   169 +-
 docs/monitoring.md                                 |   342 +-
 docs/quick-start.md                                |   291 +-
 docs/rdd-programming-guide.md                      |   825 +-
 docs/running-on-kubernetes.md                      |   806 +-
 docs/running-on-mesos.md                           |   901 -
 docs/running-on-yarn.md                            |   291 +-
 docs/security.md                                   |   299 +-
 docs/spark-connect-gotchas.md                      |   432 +
 docs/spark-connect-overview.md                     |   415 +
 docs/spark-standalone.md                           |   410 +-
 docs/sparkr-migration-guide.md                     |     8 +
 docs/sparkr.md                                     |    64 +-
 docs/sql-data-sources-avro.md                      |   271 +-
 docs/sql-data-sources-binaryFile.md                |    16 +-
 docs/sql-data-sources-csv.md                       |    78 +-
 docs/sql-data-sources-generic-options.md           |    49 +-
 docs/sql-data-sources-hive-tables.md               |    30 +-
 docs/sql-data-sources-jdbc.md                      |  1906 +-
 docs/sql-data-sources-json.md                      |    73 +-
 docs/sql-data-sources-load-save-functions.md       |   139 +-
 docs/sql-data-sources-orc.md                       |    61 +-
 docs/sql-data-sources-parquet.md                   |   255 +-
 docs/sql-data-sources-protobuf.md                  |   502 +
 docs/sql-data-sources-text.md                      |    16 +-
 docs/sql-data-sources-v2.md                        |   443 +
 docs/sql-data-sources-xml.md                       |   246 +
 docs/sql-data-sources.md                           |    13 +-
 docs/sql-distributed-sql-engine-spark-sql-cli.md   |   191 +
 docs/sql-distributed-sql-engine.md                 |    12 +-
 docs/sql-error-conditions.md                       |    37 +
 docs/sql-getting-started.md                        |   149 +-
 docs/sql-migration-guide.md                        |   511 +-
 docs/sql-migration-old.md                          |    23 -
 docs/sql-performance-tuning.md                     |   432 +-
 docs/sql-pipe-syntax.md                            |   656 +
 docs/sql-programming-guide.md                      |     5 +-
 docs/sql-pyspark-pandas-with-arrow.md              |     2 +-
 docs/sql-ref-ansi-compliance.md                    |   481 +-
 docs/sql-ref-datatypes.md                          |   139 +-
 docs/sql-ref-datetime-pattern.md                   |     8 +-
 docs/sql-ref-function-invocation.md                |   112 +
 docs/sql-ref-functions-builtin.md                  |   133 +-
 docs/sql-ref-functions-udf-hive.md                 |    14 +-
 docs/sql-ref-functions.md                          |    19 +-
 docs/sql-ref-geospatial-types.md                   |   207 +
 docs/sql-ref-identifier-clause.md                  |   194 +
 docs/sql-ref-identifier.md                         |    18 +-
 docs/sql-ref-literals.md                           |    97 +-
 docs/sql-ref-name-resolution.md                    |   423 +
 docs/sql-ref-null-semantics.md                     |    87 +-
 docs/sql-ref-number-pattern.md                     |   186 +
 docs/sql-ref-operators.md                          |   124 +
 docs/sql-ref-parameter-markers.md                  |   228 +
 docs/sql-ref-scripting.md                          |   101 +
 docs/sql-ref-sketch-aggregates.md                  |  1582 +
 docs/sql-ref-syntax-aux-analyze-table.md           |     4 +-
 docs/sql-ref-syntax-aux-cache-cache-table.md       |     2 +
 docs/sql-ref-syntax-aux-cache-refresh.md           |     5 +-
 docs/sql-ref-syntax-aux-cache-uncache-table.md     |     2 +
 docs/sql-ref-syntax-aux-conf-mgmt-reset.md         |     2 +-
 docs/sql-ref-syntax-aux-conf-mgmt-set.md           |     3 +
 docs/sql-ref-syntax-aux-describe-function.md       |     2 +-
 docs/sql-ref-syntax-aux-describe-table.md          |   109 +-
 docs/sql-ref-syntax-aux-exec-imm.md                |   113 +
 docs/sql-ref-syntax-aux-resource-mgmt-add-jar.md   |     6 +-
 docs/sql-ref-syntax-aux-resource-mgmt-list-file.md |     2 +-
 docs/sql-ref-syntax-aux-set-var.md                 |    98 +
 docs/sql-ref-syntax-aux-show-collations.md         |   111 +
 docs/sql-ref-syntax-aux-show-create-table.md       |    25 +-
 docs/sql-ref-syntax-aux-show-functions.md          |    13 +-
 docs/sql-ref-syntax-aux-show-partitions.md         |     2 +-
 docs/sql-ref-syntax-aux-show-views.md              |     2 +-
 docs/sql-ref-syntax-ddl-alter-database.md          |    90 +-
 docs/sql-ref-syntax-ddl-alter-table.md             |   343 +-
 docs/sql-ref-syntax-ddl-alter-view.md              |    44 +
 docs/sql-ref-syntax-ddl-create-function.md         |     7 +-
 docs/sql-ref-syntax-ddl-create-sql-function.md     |   303 +
 docs/sql-ref-syntax-ddl-create-table-datasource.md |    34 +-
 docs/sql-ref-syntax-ddl-create-table-hiveformat.md |     4 +-
 docs/sql-ref-syntax-ddl-create-view.md             |    20 +-
 docs/sql-ref-syntax-ddl-declare-variable.md        |    93 +
 docs/sql-ref-syntax-ddl-drop-table.md              |     9 +-
 docs/sql-ref-syntax-ddl-drop-variable.md           |    67 +
 docs/sql-ref-syntax-ddl-repair-table.md            |     8 +-
 docs/sql-ref-syntax-dml-insert-into.md             |   254 -
 ...f-syntax-dml-insert-overwrite-directory-hive.md |    77 -
 ...ql-ref-syntax-dml-insert-overwrite-directory.md |    52 +-
 docs/sql-ref-syntax-dml-insert-overwrite-table.md  |   237 -
 docs/sql-ref-syntax-dml-insert-table.md            |   503 +
 docs/sql-ref-syntax-qry-select-aggregate.md        |   141 +
 docs/sql-ref-syntax-qry-select-case.md             |     2 +
 docs/sql-ref-syntax-qry-select-clusterby.md        |     2 +
 docs/sql-ref-syntax-qry-select-cte.md              |     4 -
 docs/sql-ref-syntax-qry-select-distribute-by.md    |     2 +
 docs/sql-ref-syntax-qry-select-groupby.md          |     2 +
 docs/sql-ref-syntax-qry-select-having.md           |     2 +
 docs/sql-ref-syntax-qry-select-hints.md            |    11 +-
 docs/sql-ref-syntax-qry-select-join.md             |    27 +-
 docs/sql-ref-syntax-qry-select-lateral-subquery.md |    87 +
 docs/sql-ref-syntax-qry-select-lateral-view.md     |     2 +
 docs/sql-ref-syntax-qry-select-limit.md            |     2 +
 docs/sql-ref-syntax-qry-select-offset.md           |   101 +
 docs/sql-ref-syntax-qry-select-orderby.md          |     2 +
 docs/sql-ref-syntax-qry-select-pivot.md            |    25 +-
 docs/sql-ref-syntax-qry-select-qualify.md          |    99 +
 docs/sql-ref-syntax-qry-select-sortby.md           |     2 +
 docs/sql-ref-syntax-qry-select-transform.md        |    30 +-
 docs/sql-ref-syntax-qry-select-tvf.md              |    24 +-
 docs/sql-ref-syntax-qry-select-unpivot.md          |   144 +
 docs/sql-ref-syntax-qry-select-where.md            |     2 +
 docs/sql-ref-syntax-qry-select-window.md           |    41 +-
 docs/sql-ref-syntax-qry-select.md                  |    31 +-
 docs/sql-ref-syntax-qry-star.md                    |    97 +
 docs/sql-ref-syntax.md                             |    28 +-
 docs/sql-ref.md                                    |    19 +-
 docs/ss-migration-guide.md                         |    25 +-
 docs/storage-openstack-swift.md                    |    16 +-
 docs/streaming-custom-receivers.md                 |     8 +-
 docs/streaming-kafka-0-10-integration.md           |     6 +-
 docs/streaming-kinesis-integration.md              |   388 +-
 docs/streaming-programming-guide.md                |   885 +-
 docs/streaming/additional-information.md           |    58 +
 docs/streaming/apis-on-dataframes-and-datasets.md  |  3596 +++
 docs/streaming/getting-started.md                  |   509 +
 docs/streaming/index.md                            |    28 +
 docs/streaming/performance-tips.md                 |   174 +
 docs/streaming/ss-migration-guide.md               |    61 +
 .../structured-streaming-kafka-integration.md      |  1227 +
 .../structured-streaming-state-data-source.md      |   421 +
 .../structured-streaming-transform-with-state.md   |   743 +
 docs/structured-streaming-kafka-integration.md     |  1142 +-
 docs/structured-streaming-programming-guide.md     |  3594 +--
 docs/submitting-applications.md                    |    39 +-
 docs/tuning.md                                     |    38 +-
 docs/web-ui.md                                     |    59 +-
 examples/pom.xml                                   |     8 +-
 .../org/apache/spark/examples/JavaLogQuery.java    |     1 +
 .../examples/ml/JavaTargetEncoderExample.java      |    90 +
 .../mllib/JavaPowerIterationClusteringExample.java |     1 -
 .../mllib/JavaStratifiedSamplingExample.java       |     4 +-
 .../examples/sql/JavaSQLDataSourceExample.java     |    71 +-
 .../apache/spark/examples/sql/JavaSparkSQLCli.java |    41 +
 .../sql/JavaUserDefinedTypedAggregation.java       |     6 +
 .../sql/JavaUserDefinedUntypedAggregation.java     |     6 +
 .../streaming/JavaRecoverableNetworkWordCount.java |     7 +-
 .../streaming/JavaStatefulNetworkWordCount.java    |     1 -
 examples/src/main/python/__init__.py               |    16 +
 examples/src/main/python/als.py                    |    24 +-
 examples/src/main/python/avro_inputformat.py       |     4 +-
 examples/src/main/python/kmeans.py                 |     5 +-
 examples/src/main/python/logistic_regression.py    |     9 +-
 examples/src/main/python/ml/__init__,py            |    16 +
 examples/src/main/python/ml/bucketizer_example.py  |     2 +-
 .../src/main/python/ml/chi_square_test_example.py  |     5 +
 examples/src/main/python/ml/correlation_example.py |     8 +
 .../ml/estimator_transformer_param_example.py      |     2 +-
 examples/src/main/python/ml/pipeline_example.py    |     4 +-
 .../src/main/python/ml/target_encoder_example.py   |    65 +
 examples/src/main/python/mllib/__init__.py         |    16 +
 examples/src/main/python/pagerank.py               |    11 +-
 examples/src/main/python/parquet_inputformat.py    |     6 +-
 examples/src/main/python/pi.py                     |     2 +-
 examples/src/main/python/sort.py                   |     4 +-
 examples/src/main/python/sql/__init__.py           |    16 +
 examples/src/main/python/sql/arrow.py              |   104 +-
 examples/src/main/python/sql/basic.py              |     7 +-
 examples/src/main/python/sql/datasource.py         |   124 +-
 examples/src/main/python/sql/jdbc.py               |    48 +
 examples/src/main/python/sql/streaming/__init__,py |    16 +
 .../structured_blockchain_admission_control.py     |   202 +
 .../structured_network_wordcount_session_window.py |   142 +
 examples/src/main/python/sql/udtf.py               |   320 +
 examples/src/main/python/status_api_demo.py        |    16 +-
 examples/src/main/python/streaming/__init__.py     |    16 +
 .../src/main/python/streaming/hdfs_wordcount.py    |     2 +-
 .../src/main/python/streaming/network_wordcount.py |     2 +-
 .../python/streaming/network_wordjoinsentiments.py |    19 +-
 .../streaming/recoverable_network_wordcount.py     |    15 +-
 .../main/python/streaming/sql_network_wordcount.py |     9 +-
 .../python/streaming/stateful_network_wordcount.py |     3 +-
 examples/src/main/python/transitive_closure.py     |     5 +-
 examples/src/main/r/RSparkSQLExample.R             |    14 +-
 ...apache.spark.sql.SparkSessionExtensionsProvider |    17 +
 ...rg.apache.spark.sql.jdbc.JdbcConnectionProvider |    17 +
 examples/src/main/resources/people.xml             |    15 +
 .../apache/spark/examples/DFSReadWriteTest.scala   |     4 +-
 .../spark/examples/DriverSubmissionTest.scala      |     6 +-
 .../spark/examples/ExceptionHandlingTest.scala     |     4 +-
 .../org/apache/spark/examples/GroupByTest.scala    |     2 +-
 .../scala/org/apache/spark/examples/HdfsTest.scala |     2 +-
 .../scala/org/apache/spark/examples/LocalALS.scala |     4 +-
 .../org/apache/spark/examples/LocalKMeans.scala    |     2 +-
 .../scala/org/apache/spark/examples/LocalPi.scala  |     4 +-
 .../apache/spark/examples/MiniReadWriteTest.scala  |    12 +-
 .../apache/spark/examples/MultiBroadcastTest.scala |     6 +-
 .../spark/examples/SimpleSkewedGroupByTest.scala   |    10 +-
 .../apache/spark/examples/SkewedGroupByTest.scala  |     2 +-
 .../scala/org/apache/spark/examples/SparkALS.scala |     6 +-
 .../org/apache/spark/examples/SparkHdfsLR.scala    |     2 +-
 .../org/apache/spark/examples/SparkKMeans.scala    |     6 +-
 .../scala/org/apache/spark/examples/SparkLR.scala  |     7 +-
 .../org/apache/spark/examples/SparkPageRank.scala  |     2 +-
 .../scala/org/apache/spark/examples/SparkPi.scala  |     6 +-
 .../scala/org/apache/spark/examples/SparkTC.scala  |     2 +-
 .../spark/examples/extensions/AgeExample.scala     |    13 +-
 .../examples/graphx/AggregateMessagesExample.scala |     4 +-
 .../apache/spark/examples/graphx/Analytics.scala   |    10 +-
 .../examples/graphx/ComprehensiveExample.scala     |     2 +-
 .../graphx/ConnectedComponentsExample.scala        |     2 +-
 .../spark/examples/graphx/PageRankExample.scala    |     2 +-
 .../apache/spark/examples/graphx/SSSPExample.scala |     4 +-
 .../spark/examples/graphx/SynthBenchmark.scala     |     2 +-
 .../examples/graphx/TriangleCountingExample.scala  |     2 +-
 .../examples/ml/AFTSurvivalRegressionExample.scala |     2 +-
 .../org/apache/spark/examples/ml/ALSExample.scala  |     2 +-
 .../spark/examples/ml/BinarizerExample.scala       |     6 +-
 .../spark/examples/ml/BisectingKMeansExample.scala |     2 +-
 .../ml/BucketedRandomProjectionLSHExample.scala    |     2 +-
 .../spark/examples/ml/BucketizerExample.scala      |    10 +-
 .../spark/examples/ml/ChiSqSelectorExample.scala   |     2 +-
 .../spark/examples/ml/ChiSquareTestExample.scala   |     4 +-
 .../spark/examples/ml/CorrelationExample.scala     |     6 +-
 .../spark/examples/ml/CountVectorizerExample.scala |     2 +-
 .../org/apache/spark/examples/ml/DCTExample.scala  |     2 +-
 .../spark/examples/ml/DataFrameExample.scala       |     2 +-
 .../ml/DecisionTreeClassificationExample.scala     |     2 +-
 .../spark/examples/ml/DecisionTreeExample.scala    |     4 +-
 .../ml/DecisionTreeRegressionExample.scala         |     2 +-
 .../spark/examples/ml/DeveloperApiExample.scala    |     9 +-
 .../examples/ml/ElementwiseProductExample.scala    |     2 +-
 .../ml/EstimatorTransformerParamExample.scala      |     6 +-
 .../spark/examples/ml/FMClassifierExample.scala    |     2 +-
 .../spark/examples/ml/FMRegressorExample.scala     |     2 +-
 .../apache/spark/examples/ml/FPGrowthExample.scala |     2 +-
 .../spark/examples/ml/FeatureHasherExample.scala   |     2 +-
 .../org/apache/spark/examples/ml/GBTExample.scala  |     2 +-
 .../spark/examples/ml/GaussianMixtureExample.scala |     2 +-
 .../ml/GeneralizedLinearRegressionExample.scala    |     2 +-
 .../ml/GradientBoostedTreeClassifierExample.scala  |     2 +-
 .../ml/GradientBoostedTreeRegressorExample.scala   |     2 +-
 .../apache/spark/examples/ml/ImputerExample.scala  |     2 +-
 .../spark/examples/ml/IndexToStringExample.scala   |     2 +-
 .../spark/examples/ml/InteractionExample.scala     |     2 +-
 .../examples/ml/IsotonicRegressionExample.scala    |     2 +-
 .../apache/spark/examples/ml/KMeansExample.scala   |     2 +-
 .../org/apache/spark/examples/ml/LDAExample.scala  |     2 +-
 .../examples/ml/LinearRegressionExample.scala      |     2 +-
 .../ml/LinearRegressionWithElasticNetExample.scala |     2 +-
 .../spark/examples/ml/LinearSVCExample.scala       |     2 +-
 .../examples/ml/LogisticRegressionExample.scala    |     2 +-
 .../ml/LogisticRegressionSummaryExample.scala      |     2 +-
 .../LogisticRegressionWithElasticNetExample.scala  |     2 +-
 .../spark/examples/ml/MaxAbsScalerExample.scala    |     2 +-
 .../spark/examples/ml/MinHashLSHExample.scala      |     2 +-
 .../spark/examples/ml/MinMaxScalerExample.scala    |     2 +-
 .../ModelSelectionViaCrossValidationExample.scala  |     2 +-
 ...elSelectionViaTrainValidationSplitExample.scala |     2 +-
 ...ssLogisticRegressionWithElasticNetExample.scala |     2 +-
 .../ml/MultilayerPerceptronClassifierExample.scala |     2 +-
 .../apache/spark/examples/ml/NGramExample.scala    |     2 +-
 .../spark/examples/ml/NaiveBayesExample.scala      |     2 +-
 .../spark/examples/ml/NormalizerExample.scala      |     2 +-
 .../spark/examples/ml/OneHotEncoderExample.scala   |     2 +-
 .../spark/examples/ml/OneVsRestExample.scala       |     2 +-
 .../org/apache/spark/examples/ml/PCAExample.scala  |     7 +-
 .../apache/spark/examples/ml/PipelineExample.scala |     2 +-
 .../examples/ml/PolynomialExpansionExample.scala   |     7 +-
 .../ml/PowerIterationClusteringExample.scala       |     2 +-
 .../spark/examples/ml/PrefixSpanExample.scala      |     2 +-
 .../examples/ml/QuantileDiscretizerExample.scala   |     6 +-
 .../apache/spark/examples/ml/RFormulaExample.scala |     2 +-
 .../ml/RandomForestClassifierExample.scala         |     2 +-
 .../spark/examples/ml/RandomForestExample.scala    |     2 +-
 .../examples/ml/RandomForestRegressorExample.scala |     2 +-
 .../spark/examples/ml/RobustScalerExample.scala    |     2 +-
 .../spark/examples/ml/SQLTransformerExample.scala  |     2 +-
 .../spark/examples/ml/StandardScalerExample.scala  |     2 +-
 .../examples/ml/StopWordsRemoverExample.scala      |     2 +-
 .../spark/examples/ml/StringIndexerExample.scala   |     2 +-
 .../spark/examples/ml/SummarizerExample.scala      |     2 +-
 .../spark/examples/ml/TargetEncoderExample.scala   |    71 +
 .../apache/spark/examples/ml/TfIdfExample.scala    |     2 +-
 .../spark/examples/ml/TokenizerExample.scala       |     2 +-
 .../ml/UnivariateFeatureSelectorExample.scala      |     2 +-
 .../ml/VarianceThresholdSelectorExample.scala      |     2 +-
 .../spark/examples/ml/VectorAssemblerExample.scala |     2 +-
 .../spark/examples/ml/VectorIndexerExample.scala   |     2 +-
 .../spark/examples/ml/VectorSizeHintExample.scala  |     2 +-
 .../spark/examples/ml/VectorSlicerExample.scala    |     2 +-
 .../apache/spark/examples/ml/Word2VecExample.scala |     2 +-
 .../examples/mllib/BinaryClassification.scala      |     5 +-
 .../mllib/BinaryClassificationMetricsExample.scala |    24 +-
 .../examples/mllib/ChiSqSelectorExample.scala      |     2 +-
 .../spark/examples/mllib/CorrelationsExample.scala |     8 +-
 .../apache/spark/examples/mllib/DenseKMeans.scala  |     5 +-
 .../examples/mllib/ElementwiseProductExample.scala |     4 +-
 .../GradientBoostingClassificationExample.scala    |     2 +-
 .../apache/spark/examples/mllib/LDAExample.scala   |    20 +-
 .../mllib/LatentDirichletAllocationExample.scala   |     2 +-
 .../apache/spark/examples/mllib/MovieLensALS.scala |     5 +-
 .../spark/examples/mllib/NormalizerExample.scala   |     4 +-
 .../examples/mllib/PCAOnRowMatrixExample.scala     |     4 +-
 .../examples/mllib/PMMLModelExportExample.scala    |     2 +-
 .../mllib/PowerIterationClusteringExample.scala    |     7 +-
 .../mllib/RandomForestClassificationExample.scala  |     2 +-
 .../examples/mllib/RankingMetricsExample.scala     |     2 +-
 .../apache/spark/examples/mllib/SVDExample.scala   |     4 +-
 .../spark/examples/mllib/SparseNaiveBayes.scala    |     5 +-
 .../examples/mllib/StandardScalerExample.scala     |     4 +-
 .../apache/spark/examples/mllib/TFIDFExample.scala |     4 +-
 .../spark/examples/mllib/Word2VecExample.scala     |     2 +-
 .../examples/pythonconverters/AvroConverters.scala |     2 +-
 .../apache/spark/examples/sql/RDDRelation.scala    |     2 +-
 .../spark/examples/sql/SQLDataSourceExample.scala  |    73 +-
 .../spark/examples/sql/SimpleTypedAggregator.scala |     2 +-
 .../spark/examples/sql/SparkDataFramePi.scala      |    42 +
 .../spark/examples/sql/SparkSQLExample.scala       |     5 +-
 .../spark/examples/sql/hive/SparkHiveExample.scala |     4 +-
 .../sql/jdbc/ExampleJdbcConnectionProvider.scala   |     5 +
 .../StructuredComplexSessionization.scala          |     2 +-
 .../sql/streaming/StructuredKafkaWordCount.scala   |     2 +-
 .../StructuredKerberizedKafkaWordCount.scala       |     2 +-
 .../sql/streaming/StructuredNetworkWordCount.scala |     2 +-
 .../StructuredNetworkWordCountWindowed.scala       |     2 +-
 .../sql/streaming/StructuredSessionization.scala   |     4 +-
 .../spark/examples/streaming/CustomReceiver.scala  |     2 +-
 .../spark/examples/streaming/RawNetworkGrep.scala  |     4 +-
 .../streaming/RecoverableNetworkWordCount.scala    |     7 +-
 .../examples/streaming/SqlNetworkWordCount.scala   |     2 +-
 .../streaming/StatefulNetworkWordCount.scala       |     2 +-
 .../examples/streaming/StreamingExamples.scala     |    10 +-
 examples/src/main/scripts/submit-pi.sh             |    50 +
 examples/src/main/scripts/submit-sql.sh            |    39 +
 .../benchmarks/AvroReadBenchmark-jdk11-results.txt |   130 -
 .../avro/benchmarks/AvroReadBenchmark-results.txt  |   141 -
 .../AvroWriteBenchmark-jdk11-results.txt           |    10 -
 .../avro/benchmarks/AvroWriteBenchmark-results.txt |    16 -
 external/avro/pom.xml                              |    93 -
 .../spark/sql/avro/SparkAvroKeyOutputFormat.java   |    94 -
 ...org.apache.spark.sql.sources.DataSourceRegister |     1 -
 .../apache/spark/sql/avro/AvroDataToCatalyst.scala |   140 -
 .../apache/spark/sql/avro/AvroDeserializer.scala   |   444 -
 .../org/apache/spark/sql/avro/AvroFileFormat.scala |   172 -
 .../org/apache/spark/sql/avro/AvroOptions.scala    |   147 -
 .../apache/spark/sql/avro/AvroOutputWriter.scala   |    88 -
 .../spark/sql/avro/AvroOutputWriterFactory.scala   |    49 -
 .../org/apache/spark/sql/avro/AvroSerializer.scala |   334 -
 .../org/apache/spark/sql/avro/AvroUtils.scala      |   333 -
 .../apache/spark/sql/avro/CatalystDataToAvro.scala |    70 -
 .../apache/spark/sql/avro/SchemaConverters.scala   |   216 -
 .../org/apache/spark/sql/avro/functions.scala      |    92 -
 .../sql/v2/avro/AvroPartitionReaderFactory.scala   |   122 -
 .../org/apache/spark/sql/v2/avro/AvroScan.scala    |    84 -
 .../apache/spark/sql/v2/avro/AvroScanBuilder.scala |    57 -
 .../org/apache/spark/sql/v2/avro/AvroTable.scala   |    53 -
 .../spark/sql/avro/JavaAvroFunctionsSuite.java     |    66 -
 external/avro/src/test/resources/log4j.properties  |    28 -
 .../sql/avro/AvroCatalystDataConversionSuite.scala |   362 -
 .../apache/spark/sql/avro/AvroFunctionsSuite.scala |   241 -
 .../spark/sql/avro/AvroLogicalTypeSuite.scala      |   455 -
 .../apache/spark/sql/avro/AvroRowReaderSuite.scala |    96 -
 .../spark/sql/avro/AvroSchemaHelperSuite.scala     |   136 -
 .../org/apache/spark/sql/avro/AvroSerdeSuite.scala |   236 -
 .../org/apache/spark/sql/avro/AvroSuite.scala      |  2380 --
 .../sql/avro/DeprecatedAvroFunctionsSuite.scala    |   123 -
 .../execution/benchmark/AvroReadBenchmark.scala    |   308 -
 .../execution/benchmark/AvroWriteBenchmark.scala   |    69 -
 external/docker-integration-tests/pom.xml          |   166 -
 .../src/test/resources/log4j.properties            |    36 -
 .../test/resources/mariadb_docker_entrypoint.sh    |    26 -
 .../src/test/resources/mariadb_krb_setup.sh        |    20 -
 .../src/test/resources/postgres_krb_setup.sh       |    21 -
 .../spark/sql/jdbc/DB2IntegrationSuite.scala       |   201 -
 .../spark/sql/jdbc/DB2KrbIntegrationSuite.scala    |   101 -
 .../spark/sql/jdbc/DockerIntegrationFunSuite.scala |    54 -
 .../sql/jdbc/DockerJDBCIntegrationSuite.scala      |   241 -
 .../sql/jdbc/DockerKrbJDBCIntegrationSuite.scala   |   201 -
 .../sql/jdbc/MariaDBKrbIntegrationSuite.scala      |    74 -
 .../sql/jdbc/MsSqlServerIntegrationSuite.scala     |   359 -
 .../spark/sql/jdbc/MySQLIntegrationSuite.scala     |   195 -
 .../spark/sql/jdbc/OracleIntegrationSuite.scala    |   521 -
 .../spark/sql/jdbc/PostgresIntegrationSuite.scala  |   382 -
 .../sql/jdbc/PostgresKrbIntegrationSuite.scala     |    67 -
 .../spark/sql/jdbc/v2/DB2IntegrationSuite.scala    |    89 -
 .../sql/jdbc/v2/MsSqlServerIntegrationSuite.scala  |    93 -
 .../spark/sql/jdbc/v2/MySQLIntegrationSuite.scala  |   118 -
 .../spark/sql/jdbc/v2/OracleIntegrationSuite.scala |    96 -
 .../sql/jdbc/v2/PostgresIntegrationSuite.scala     |    78 -
 .../spark/sql/jdbc/v2/PostgresNamespaceSuite.scala |    59 -
 .../spark/sql/jdbc/v2/V2JDBCNamespaceTest.scala    |    63 -
 .../org/apache/spark/sql/jdbc/v2/V2JDBCTest.scala  |   184 -
 .../scala/org/apache/spark/util/DockerUtils.scala  |    68 -
 external/docker/README.md                          |     7 -
 external/docker/build                              |    22 -
 external/docker/spark-test/README.md               |    11 -
 external/docker/spark-test/base/Dockerfile         |    28 -
 external/docker/spark-test/build                   |    22 -
 external/docker/spark-test/master/Dockerfile       |    21 -
 external/docker/spark-test/master/default_cmd      |    25 -
 external/docker/spark-test/worker/Dockerfile       |    22 -
 external/docker/spark-test/worker/default_cmd      |    25 -
 external/kafka-0-10-assembly/pom.xml               |   175 -
 external/kafka-0-10-sql/pom.xml                    |   188 -
 ...org.apache.spark.sql.sources.DataSourceRegister |     1 -
 .../spark/sql/kafka010/ConsumerStrategy.scala      |   149 -
 .../org/apache/spark/sql/kafka010/JsonUtils.scala  |   118 -
 .../sql/kafka010/KafkaBatchPartitionReader.scala   |   121 -
 .../spark/sql/kafka010/KafkaBatchWrite.scala       |    67 -
 .../spark/sql/kafka010/KafkaContinuousStream.scala |   249 -
 .../spark/sql/kafka010/KafkaMicroBatchStream.scala |   335 -
 .../sql/kafka010/KafkaOffsetRangeCalculator.scala  |   136 -
 .../spark/sql/kafka010/KafkaOffsetReader.scala     |   169 -
 .../sql/kafka010/KafkaOffsetReaderAdmin.scala      |   548 -
 .../sql/kafka010/KafkaOffsetReaderConsumer.scala   |   630 -
 .../sql/kafka010/KafkaRecordToRowConverter.scala   |    93 -
 .../apache/spark/sql/kafka010/KafkaRelation.scala  |    91 -
 .../org/apache/spark/sql/kafka010/KafkaSink.scala  |    43 -
 .../apache/spark/sql/kafka010/KafkaSource.scala    |   350 -
 .../kafka010/KafkaSourceInitialOffsetWriter.scala  |    63 -
 .../spark/sql/kafka010/KafkaSourceOffset.scala     |    79 -
 .../spark/sql/kafka010/KafkaSourceProvider.scala   |   732 -
 .../apache/spark/sql/kafka010/KafkaSourceRDD.scala |   144 -
 .../spark/sql/kafka010/KafkaStreamingWrite.scala   |    72 -
 .../apache/spark/sql/kafka010/KafkaWriteTask.scala |   137 -
 .../apache/spark/sql/kafka010/KafkaWriter.scala    |   125 -
 .../sql/kafka010/consumer/FetchedDataPool.scala    |   185 -
 .../consumer/InternalKafkaConsumerPool.scala       |   220 -
 .../sql/kafka010/consumer/KafkaDataConsumer.scala  |   691 -
 .../kafka010/producer/CachedKafkaProducer.scala    |    41 -
 .../producer/InternalKafkaProducerPool.scala       |   206 -
 .../src/test/resources/log4j.properties            |    30 -
 .../spark/sql/kafka010/ConsumerStrategySuite.scala |   147 -
 .../sql/kafka010/KafkaContinuousSourceSuite.scala  |   241 -
 .../spark/sql/kafka010/KafkaContinuousTest.scala   |    97 -
 .../sql/kafka010/KafkaDelegationTokenSuite.scala   |   118 -
 .../kafka010/KafkaDontFailOnDataLossSuite.scala    |   289 -
 .../sql/kafka010/KafkaMicroBatchSourceSuite.scala  |  2469 --
 .../kafka010/KafkaOffsetRangeCalculatorSuite.scala |   259 -
 .../sql/kafka010/KafkaOffsetReaderSuite.scala      |   198 -
 .../spark/sql/kafka010/KafkaRelationSuite.scala    |   659 -
 .../apache/spark/sql/kafka010/KafkaSinkSuite.scala |   602 -
 .../sql/kafka010/KafkaSourceOffsetSuite.scala      |   105 -
 .../sql/kafka010/KafkaSourceProviderSuite.scala    |   136 -
 .../org/apache/spark/sql/kafka010/KafkaTest.scala  |    33 -
 .../apache/spark/sql/kafka010/KafkaTestUtils.scala |   680 -
 .../apache/spark/sql/kafka010/RecordBuilder.scala  |    74 -
 .../kafka010/consumer/FetchedDataPoolSuite.scala   |   305 -
 .../consumer/InternalKafkaConsumerPoolSuite.scala  |   286 -
 .../kafka010/consumer/KafkaDataConsumerSuite.scala |   390 -
 external/kafka-0-10-token-provider/pom.xml         |    95 -
 ...he.spark.security.HadoopDelegationTokenProvider |     1 -
 .../apache/spark/kafka010/KafkaConfigUpdater.scala |    89 -
 .../kafka010/KafkaDelegationTokenProvider.scala    |    85 -
 .../apache/spark/kafka010/KafkaRedactionUtil.scala |    52 -
 .../spark/kafka010/KafkaTokenSparkConf.scala       |   110 -
 .../org/apache/spark/kafka010/KafkaTokenUtil.scala |   288 -
 .../src/test/resources/log4j.properties            |    28 -
 .../spark/kafka010/KafkaConfigUpdaterSuite.scala   |   164 -
 .../spark/kafka010/KafkaTokenSparkConfSuite.scala  |   148 -
 .../spark/kafka010/KafkaTokenUtilSuite.scala       |   292 -
 external/kafka-0-10/pom.xml                        |   156 -
 .../streaming/kafka010/ConsumerStrategy.scala      |   468 -
 .../kafka010/DirectKafkaInputDStream.scala         |   349 -
 .../streaming/kafka010/KafkaDataConsumer.scala     |   366 -
 .../apache/spark/streaming/kafka010/KafkaRDD.scala |   312 -
 .../spark/streaming/kafka010/KafkaUtils.scala      |   208 -
 .../streaming/kafka010/LocationStrategy.scala      |    71 -
 .../streaming/kafka010/PerPartitionConfig.scala    |    47 -
 .../kafka010/JavaConsumerStrategySuite.java        |    96 -
 .../kafka010/JavaDirectKafkaStreamSuite.java       |   155 -
 .../streaming/kafka010/JavaKafkaRDDSuite.java      |   121 -
 .../kafka010/JavaLocationStrategySuite.java        |    59 -
 .../kafka-0-10/src/test/resources/log4j.properties |    28 -
 .../kafka010/DirectKafkaStreamSuite.scala          |   815 -
 .../kafka010/KafkaDataConsumerSuite.scala          |   163 -
 .../spark/streaming/kafka010/KafkaRDDSuite.scala   |   293 -
 .../spark/streaming/kafka010/KafkaTestUtils.scala  |   348 -
 .../streaming/kafka010/mocks/MockScheduler.scala   |    72 -
 external/kinesis-asl-assembly/pom.xml              |   209 -
 external/kinesis-asl/pom.xml                       |   110 -
 .../streaming/JavaKinesisWordCountASL.java         |   196 -
 .../streaming/kinesis/KinesisInitialPositions.java |    93 -
 .../examples/streaming/kinesis_wordcount_asl.py    |    82 -
 .../src/main/resources/log4j.properties            |    37 -
 .../examples/streaming/KinesisExampleUtils.scala   |    35 -
 .../examples/streaming/KinesisWordCountASL.scala   |   283 -
 .../streaming/kinesis/KinesisBackedBlockRDD.scala  |   298 -
 .../streaming/kinesis/KinesisCheckpointer.scala    |   145 -
 .../streaming/kinesis/KinesisInputDStream.scala    |   371 -
 .../kinesis/KinesisReadConfigurations.scala        |    78 -
 .../spark/streaming/kinesis/KinesisReceiver.scala  |   371 -
 .../streaming/kinesis/KinesisRecordProcessor.scala |   187 -
 .../spark/streaming/kinesis/KinesisTestUtils.scala |   294 -
 .../kinesis/KinesisUtilsPythonHelper.scala         |    93 -
 .../streaming/kinesis/SparkAWSCredentials.scala    |   178 -
 .../JavaKinesisInputDStreamBuilderSuite.java       |    99 -
 .../src/test/resources/log4j.properties            |    27 -
 .../kinesis/KPLBasedKinesisTestUtils.scala         |    77 -
 .../kinesis/KinesisBackedBlockRDDSuite.scala       |   259 -
 .../kinesis/KinesisCheckpointerSuite.scala         |   150 -
 .../spark/streaming/kinesis/KinesisFunSuite.scala  |    46 -
 .../kinesis/KinesisInputDStreamBuilderSuite.scala  |   189 -
 .../streaming/kinesis/KinesisReceiverSuite.scala   |   222 -
 .../streaming/kinesis/KinesisStreamSuite.scala     |   435 -
 external/spark-ganglia-lgpl/pom.xml                |    48 -
 .../codahale/metrics/ganglia/GangliaReporter.java  |   426 -
 graphx/pom.xml                                     |     8 +-
 .../org/apache/spark/graphx/GraphLoader.scala      |     6 +-
 .../scala/org/apache/spark/graphx/GraphOps.scala   |     2 +-
 .../scala/org/apache/spark/graphx/Pregel.scala     |     5 +-
 .../org/apache/spark/graphx/impl/GraphImpl.scala   |    11 +-
 .../graphx/impl/ShippableVertexPartition.scala     |     2 +-
 .../spark/graphx/impl/VertexPartitionBase.scala    |     1 -
 .../spark/graphx/impl/VertexPartitionBaseOps.scala |     1 -
 .../org/apache/spark/graphx/lib/PageRank.scala     |    11 +-
 .../org/apache/spark/graphx/lib/SVDPlusPlus.scala  |    12 +-
 .../apache/spark/graphx/lib/TriangleCount.scala    |     2 +-
 .../apache/spark/graphx/util/BytecodeUtils.scala   |   134 -
 .../apache/spark/graphx/util/GraphGenerators.scala |     4 +-
 .../collection/GraphXPrimitiveKeyOpenHashMap.scala |     9 +-
 graphx/src/test/resources/log4j.properties         |    27 -
 graphx/src/test/resources/log4j2.properties        |    31 +
 .../org/apache/spark/graphx/EdgeRDDSuite.scala     |     4 +-
 .../scala/org/apache/spark/graphx/EdgeSuite.scala  |     2 +-
 .../org/apache/spark/graphx/GraphLoaderSuite.scala |     2 +-
 .../org/apache/spark/graphx/GraphOpsSuite.scala    |    55 +-
 .../scala/org/apache/spark/graphx/GraphSuite.scala |    19 +-
 .../org/apache/spark/graphx/PregelSuite.scala      |    10 +-
 .../org/apache/spark/graphx/VertexRDDSuite.scala   |    16 +-
 .../spark/graphx/lib/LabelPropagationSuite.scala   |     4 +-
 .../apache/spark/graphx/lib/PageRankSuite.scala    |     2 +-
 .../apache/spark/graphx/lib/SVDPlusPlusSuite.scala |     4 +-
 .../spark/graphx/lib/ShortestPathsSuite.scala      |     4 +-
 .../lib/StronglyConnectedComponentsSuite.scala     |     3 +-
 .../spark/graphx/lib/TriangleCountSuite.scala      |     4 +-
 hadoop-cloud/README.md                             |    20 +
 hadoop-cloud/pom.xml                               |   270 +-
 .../io/cloud/BindingParquetOutputCommitter.scala   |   122 -
 .../io/cloud/PathOutputCommitProtocol.scala        |   166 -
 .../internal/io/cloud/CommitterBindingSuite.scala  |   146 -
 .../io/cloud/StubPathOutputCommitter.scala         |   120 -
 ...AbortableStreamBasedCheckpointFileManager.scala |    98 +
 .../io/cloud/BindingParquetOutputCommitter.scala   |   126 +
 .../io/cloud/PathOutputCommitProtocol.scala        |   239 +
 .../internal/io/cloud/IntegrationTestSuite.java    |    29 +
 .../io/cloud/abortable/AbortableFileSystem.java    |   113 +
 .../abortable/AbstractAbortableFileSystem.java     |    44 +
 hadoop-cloud/src/test/resources/log4j.properties   |    36 -
 hadoop-cloud/src/test/resources/log4j2.properties  |    40 +
 .../scala/org/apache/spark/SparkContextSuite.scala |    79 +
 ...ableStreamBasedCheckpointFileManagerSuite.scala |    83 +
 .../internal/io/cloud/CommitterBindingSuite.scala  |   383 +
 .../io/cloud/StubPathOutputCommitter.scala         |   164 +
 launcher/pom.xml                                   |    41 +-
 .../spark/launcher/AbstractCommandBuilder.java     |   166 +-
 .../apache/spark/launcher/AbstractLauncher.java    |    18 +-
 .../apache/spark/launcher/CommandBuilderUtils.java |   139 +-
 .../apache/spark/launcher/InProcessLauncher.java   |     2 +-
 .../apache/spark/launcher/JavaModuleOptions.java   |    65 +
 .../org/apache/spark/launcher/LauncherServer.java  |    18 +-
 .../main/java/org/apache/spark/launcher/Main.java  |     8 +-
 .../org/apache/spark/launcher/SparkAppHandle.java  |     2 +-
 .../spark/launcher/SparkClassCommandBuilder.java   |    60 +-
 .../org/apache/spark/launcher/SparkLauncher.java   |    16 +-
 .../spark/launcher/SparkSubmitCommandBuilder.java  |   193 +-
 .../spark/launcher/SparkSubmitOptionParser.java    |     8 +
 .../java/org/apache/spark/launcher/BaseSuite.java  |    17 +-
 .../spark/launcher/ChildProcAppHandleSuite.java    |   100 +-
 .../spark/launcher/CommandBuilderUtilsSuite.java   |    25 +-
 .../spark/launcher/InProcessLauncherSuite.java     |    51 +-
 .../apache/spark/launcher/LauncherServerSuite.java |    18 +-
 .../launcher/SparkClassCommandBuilderSuite.java    |    40 +
 .../launcher/SparkSubmitCommandBuilderSuite.java   |   235 +-
 .../launcher/SparkSubmitOptionParserSuite.java     |    12 +-
 launcher/src/test/resources/log4j.properties       |    37 -
 launcher/src/test/resources/log4j2.properties      |    43 +
 licenses-binary/LICENSE-d3.min.js.txt              |    39 +-
 licenses-binary/LICENSE-datatables.txt             |    25 +-
 licenses-binary/LICENSE-icu4j.txt                  |   519 +
 licenses-binary/LICENSE-jakarta-servlet-api.txt    |   277 +
 licenses-binary/LICENSE-jline3.txt                 |    34 +
 licenses-binary/LICENSE-jodd.txt                   |    24 -
 licenses-binary/LICENSE-loose-version.txt          |   279 +
 licenses-binary/LICENSE-mustache.txt               |    11 -
 licenses-binary/LICENSE-netlib.txt                 |    49 -
 licenses-binary/LICENSE-protobuf.txt               |    12 +-
 licenses-binary/LICENSE-scala.txt                  |    30 -
 licenses-binary/LICENSE-vis-timeline.txt           |    29 +-
 licenses-binary/LICENSE-xz.txt                     |    11 +
 licenses/LICENSE-copybutton.txt                    |    49 -
 licenses/LICENSE-d3.min.js.txt                     |    39 +-
 licenses/LICENSE-datatables.txt                    |    25 +-
 licenses/LICENSE-jdom.txt                          |    54 +
 licenses/LICENSE-loose-version.txt                 |   279 +
 licenses/LICENSE-mustache.txt                      |    11 -
 licenses/LICENSE-vis-timeline.txt                  |    29 +-
 licenses/LICENSE-xz.txt                            |    11 +
 .../benchmarks/BLASBenchmark-jdk11-results.txt     |   252 -
 .../benchmarks/BLASBenchmark-jdk21-results.txt     |   312 +
 .../benchmarks/BLASBenchmark-jdk25-results.txt     |   312 +
 mllib-local/benchmarks/BLASBenchmark-results.txt   |   256 +-
 mllib-local/pom.xml                                |    18 +-
 .../scala/org/apache/spark/ml/linalg/BLAS.scala    |    89 +-
 .../org/apache/spark/ml/linalg/Matrices.scala      |    81 +-
 .../scala/org/apache/spark/ml/linalg/Vectors.scala |    60 +-
 .../scala/scala/collection/compat/package.scala    |    26 +
 .../org/apache/spark/ml/linalg/BLASBenchmark.scala |   230 +-
 .../org/apache/spark/ml/linalg/BLASSuite.scala     |     2 +-
 .../org/apache/spark/ml/linalg/MatricesSuite.scala |    13 +-
 .../org/apache/spark/ml/linalg/VectorsSuite.scala  |    20 +-
 .../UDTSerializationBenchmark-jdk11-results.txt    |    12 -
 .../UDTSerializationBenchmark-jdk21-results.txt    |    12 +
 .../UDTSerializationBenchmark-jdk25-results.txt    |    12 +
 .../UDTSerializationBenchmark-results.txt          |     8 +-
 mllib/pom.xml                                      |    38 +-
 .../services/org.apache.spark.ml.Estimator         |    73 +
 .../services/org.apache.spark.ml.Transformer       |    95 +
 .../org.apache.spark.ml.evaluation.Evaluator       |    26 +
 .../org.apache.spark.ml.util.MLFormatRegister      |    17 +
 ...apache.spark.sql.SparkSessionExtensionsProvider |    18 +
 ...org.apache.spark.sql.sources.DataSourceRegister |    17 +
 .../main/scala/org/apache/spark/ml/Estimator.scala |    31 +
 .../src/main/scala/org/apache/spark/ml/Model.scala |    17 +-
 .../main/scala/org/apache/spark/ml/Pipeline.scala  |    49 +-
 .../main/scala/org/apache/spark/ml/Predictor.scala |    58 +-
 .../scala/org/apache/spark/ml/Transformer.scala    |     3 +-
 .../main/scala/org/apache/spark/ml/ann/Layer.scala |     8 +-
 .../apache/spark/ml/attribute/AttributeGroup.scala |     7 +-
 .../org/apache/spark/ml/attribute/attributes.scala |    15 +-
 .../ml/classification/ClassificationSummary.scala  |     7 +-
 .../spark/ml/classification/Classifier.scala       |   117 +-
 .../ml/classification/DecisionTreeClassifier.scala |    27 +-
 .../spark/ml/classification/FMClassifier.scala     |   134 +-
 .../spark/ml/classification/GBTClassifier.scala    |    26 +-
 .../apache/spark/ml/classification/LinearSVC.scala |   130 +-
 .../ml/classification/LogisticRegression.scala     |   285 +-
 .../MultilayerPerceptronClassifier.scala           |   113 +-
 .../spark/ml/classification/NaiveBayes.scala       |   165 +-
 .../apache/spark/ml/classification/OneVsRest.scala |    66 +-
 .../classification/ProbabilisticClassifier.scala   |     7 +-
 .../ml/classification/RandomForestClassifier.scala |    71 +-
 .../spark/ml/clustering/BisectingKMeans.scala      |    39 +-
 .../spark/ml/clustering/ClusteringSummary.scala    |     3 +-
 .../spark/ml/clustering/GaussianMixture.scala      |   145 +-
 .../org/apache/spark/ml/clustering/KMeans.scala    |   520 +-
 .../scala/org/apache/spark/ml/clustering/LDA.scala |   146 +-
 .../ml/clustering/PowerIterationClustering.scala   |    60 +-
 .../evaluation/BinaryClassificationEvaluator.scala |     7 +-
 .../spark/ml/evaluation/ClusteringEvaluator.scala  |    23 +-
 .../spark/ml/evaluation/ClusteringMetrics.scala    |    19 +-
 .../MulticlassClassificationEvaluator.scala        |     8 +-
 .../spark/ml/evaluation/RegressionEvaluator.scala  |    16 +-
 .../org/apache/spark/ml/feature/Binarizer.scala    |    57 +-
 .../ml/feature/BucketedRandomProjectionLSH.scala   |    41 +-
 .../org/apache/spark/ml/feature/Bucketizer.scala   |     5 +-
 .../apache/spark/ml/feature/ChiSqSelector.scala    |    33 +-
 .../apache/spark/ml/feature/CountVectorizer.scala  |    51 +-
 .../scala/org/apache/spark/ml/feature/DCT.scala    |     4 +-
 .../spark/ml/feature/ElementwiseProduct.scala      |     4 +-
 .../apache/spark/ml/feature/FeatureHasher.scala    |    18 +-
 .../org/apache/spark/ml/feature/HashingTF.scala    |     4 +-
 .../scala/org/apache/spark/ml/feature/IDF.scala    |    40 +-
 .../org/apache/spark/ml/feature/Imputer.scala      |    92 +-
 .../org/apache/spark/ml/feature/Instance.scala     |     3 +-
 .../org/apache/spark/ml/feature/Interaction.scala  |    23 +-
 .../scala/org/apache/spark/ml/feature/LSH.scala    |     8 +-
 .../org/apache/spark/ml/feature/MaxAbsScaler.scala |    31 +-
 .../org/apache/spark/ml/feature/MinHashLSH.scala   |    46 +-
 .../org/apache/spark/ml/feature/MinMaxScaler.scala |    38 +-
 .../scala/org/apache/spark/ml/feature/NGram.scala  |     3 +-
 .../org/apache/spark/ml/feature/Normalizer.scala   |     4 +-
 .../apache/spark/ml/feature/OneHotEncoder.scala    |    44 +-
 .../scala/org/apache/spark/ml/feature/PCA.scala    |    34 +-
 .../spark/ml/feature/QuantileDiscretizer.scala     |    19 +-
 .../org/apache/spark/ml/feature/RFormula.scala     |   127 +-
 .../apache/spark/ml/feature/RFormulaParser.scala   |    29 +-
 .../org/apache/spark/ml/feature/RobustScaler.scala |    46 +-
 .../org/apache/spark/ml/feature/Selector.scala     |    10 +-
 .../apache/spark/ml/feature/StandardScaler.scala   |    35 +-
 .../apache/spark/ml/feature/StopWordsRemover.scala |    47 +-
 .../apache/spark/ml/feature/StringIndexer.scala    |   251 +-
 .../apache/spark/ml/feature/TargetEncoder.scala    |   468 +
 .../org/apache/spark/ml/feature/Tokenizer.scala    |    22 +-
 .../ml/feature/UnivariateFeatureSelector.scala     |    48 +-
 .../ml/feature/VarianceThresholdSelector.scala     |    37 +-
 .../apache/spark/ml/feature/VectorAssembler.scala  |    44 +-
 .../apache/spark/ml/feature/VectorIndexer.scala    |    78 +-
 .../apache/spark/ml/feature/VectorSizeHint.scala   |    55 +-
 .../org/apache/spark/ml/feature/VectorSlicer.scala |    10 +-
 .../org/apache/spark/ml/feature/Word2Vec.scala     |    48 +-
 .../scala/org/apache/spark/ml/fpm/FPGrowth.scala   |    33 +-
 .../scala/org/apache/spark/ml/fpm/PrefixSpan.scala |    76 +-
 .../main/scala/org/apache/spark/ml/functions.scala |    92 +-
 .../org/apache/spark/ml/image/ImageSchema.scala    |    27 +-
 .../spark/ml/linalg/JsonMatrixConverter.scala      |    14 +-
 .../spark/ml/linalg/JsonVectorConverter.scala      |    12 +-
 .../org/apache/spark/ml/linalg/MatrixUDT.scala     |    39 +-
 .../org/apache/spark/ml/linalg/VectorUDT.scala     |     9 +-
 .../optim/IterativelyReweightedLeastSquares.scala  |    11 +-
 .../spark/ml/optim/WeightedLeastSquares.scala      |    29 +-
 .../ml/optim/aggregator/AFTBlockAggregator.scala   |    16 +-
 .../aggregator/BinaryLogisticBlockAggregator.scala |    14 +-
 .../ml/optim/aggregator/HingeBlockAggregator.scala |    16 +-
 .../ml/optim/aggregator/HuberBlockAggregator.scala |    16 +-
 .../aggregator/LeastSquaresBlockAggregator.scala   |    14 +-
 .../MultinomialLogisticBlockAggregator.scala       |    53 +-
 .../spark/ml/optim/loss/RDDLossFunction.scala      |     2 +-
 .../scala/org/apache/spark/ml/param/params.scala   |    81 +-
 .../ml/param/shared/SharedParamsCodeGen.scala      |     4 +-
 .../scala/org/apache/spark/ml/python/MLUtil.scala  |    43 +
 .../spark/ml/r/AFTSurvivalRegressionWrapper.scala  |    12 +-
 .../scala/org/apache/spark/ml/r/ALSWrapper.scala   |     9 +-
 .../apache/spark/ml/r/BisectingKMeansWrapper.scala |    12 +-
 .../spark/ml/r/DecisionTreeClassifierWrapper.scala |    10 +-
 .../spark/ml/r/DecisionTreeRegressorWrapper.scala  |    10 +-
 .../apache/spark/ml/r/FMClassifierWrapper.scala    |    12 +-
 .../org/apache/spark/ml/r/FMRegressorWrapper.scala |    10 +-
 .../org/apache/spark/ml/r/FPGrowthWrapper.scala    |     5 +-
 .../apache/spark/ml/r/GBTClassifierWrapper.scala   |    10 +-
 .../apache/spark/ml/r/GBTRegressorWrapper.scala    |    10 +-
 .../apache/spark/ml/r/GaussianMixtureWrapper.scala |     7 +-
 .../ml/r/GeneralizedLinearRegressionWrapper.scala  |    12 +-
 .../spark/ml/r/IsotonicRegressionWrapper.scala     |    12 +-
 .../org/apache/spark/ml/r/KMeansWrapper.scala      |    14 +-
 .../org/apache/spark/ml/r/KSTestWrapper.scala      |     3 +-
 .../scala/org/apache/spark/ml/r/LDAWrapper.scala   |     9 +-
 .../spark/ml/r/LinearRegressionWrapper.scala       |    10 +-
 .../org/apache/spark/ml/r/LinearSVCWrapper.scala   |    12 +-
 .../spark/ml/r/LogisticRegressionWrapper.scala     |    12 +-
 .../r/MultilayerPerceptronClassifierWrapper.scala  |     4 +-
 .../org/apache/spark/ml/r/NaiveBayesWrapper.scala  |    12 +-
 .../org/apache/spark/ml/r/RWrapperUtils.scala      |     9 +-
 .../scala/org/apache/spark/ml/r/RWrappers.scala    |     3 +-
 .../spark/ml/r/RandomForestClassifierWrapper.scala |    11 +-
 .../spark/ml/r/RandomForestRegressorWrapper.scala  |    11 +-
 .../org/apache/spark/ml/recommendation/ALS.scala   |   182 +-
 .../ml/recommendation/TopByKeyAggregator.scala     |    59 -
 .../ml/regression/AFTSurvivalRegression.scala      |   126 +-
 .../ml/regression/DecisionTreeRegressor.scala      |    34 +-
 .../apache/spark/ml/regression/FMRegressor.scala   |    81 +-
 .../apache/spark/ml/regression/GBTRegressor.scala  |    23 +-
 .../regression/GeneralizedLinearRegression.scala   |   151 +-
 .../spark/ml/regression/IsotonicRegression.scala   |    61 +-
 .../spark/ml/regression/LinearRegression.scala     |   280 +-
 .../ml/regression/RandomForestRegressor.scala      |    24 +-
 .../spark/ml/source/image/ImageDataSource.scala    |     4 +-
 .../spark/ml/source/image/ImageFileFormat.scala    |    20 +-
 .../spark/ml/source/libsvm/LibSVMRelation.scala    |    23 +-
 .../scala/org/apache/spark/ml/stat/ANOVATest.scala |    23 +-
 .../org/apache/spark/ml/stat/ChiSquareTest.scala   |    25 +-
 .../org/apache/spark/ml/stat/Correlation.scala     |     4 +-
 .../org/apache/spark/ml/stat/FValueTest.scala      |    24 +-
 .../spark/ml/stat/KolmogorovSmirnovTest.scala      |     2 +-
 .../org/apache/spark/ml/stat/Summarizer.scala      |    80 +-
 .../main/scala/org/apache/spark/ml/tree/Node.scala |     9 +-
 .../spark/ml/tree/impl/DecisionTreeMetadata.scala  |     8 +-
 .../spark/ml/tree/impl/GradientBoostedTrees.scala  |    67 +-
 .../apache/spark/ml/tree/impl/RandomForest.scala   |    84 +-
 .../org/apache/spark/ml/tree/treeModels.scala      |   162 +-
 .../apache/spark/ml/tuning/CrossValidator.scala    |    48 +-
 .../spark/ml/tuning/TrainValidationSplit.scala     |    45 +-
 .../apache/spark/ml/tuning/ValidatorParams.scala   |    19 +-
 .../org/apache/spark/ml/util/ConnectHelper.scala   |   126 +
 .../org/apache/spark/ml/util/DatasetUtils.scala    |   153 +-
 .../apache/spark/ml/util/HasTrainingSummary.scala  |    13 +-
 .../org/apache/spark/ml/util/Instrumentation.scala |    58 +-
 .../org/apache/spark/ml/util/MetadataUtils.scala   |    14 +-
 .../scala/org/apache/spark/ml/util/ReadWrite.scala |   553 +-
 .../org/apache/spark/ml/util/SchemaUtils.scala     |    57 +-
 .../scala/org/apache/spark/ml/util/Summary.scala   |    28 +
 .../api/python/GaussianMixtureModelWrapper.scala   |     5 +-
 .../spark/mllib/api/python/LDAModelWrapper.scala   |     9 +-
 .../spark/mllib/api/python/PythonMLLibAPI.scala    |    23 +-
 .../mllib/api/python/Word2VecModelWrapper.scala    |     2 +-
 .../mllib/classification/ClassificationModel.scala |     4 +-
 .../spark/mllib/classification/NaiveBayes.scala    |    14 +-
 .../apache/spark/mllib/classification/SVM.scala    |     4 +-
 .../impl/GLMClassificationModel.scala              |     4 +-
 .../spark/mllib/clustering/BisectingKMeans.scala   |    20 +-
 .../mllib/clustering/BisectingKMeansModel.scala    |    23 +-
 .../spark/mllib/clustering/DistanceMeasure.scala   |    25 +-
 .../spark/mllib/clustering/GaussianMixture.scala   |     9 +-
 .../mllib/clustering/GaussianMixtureModel.scala    |    10 +-
 .../org/apache/spark/mllib/clustering/KMeans.scala |    52 +-
 .../spark/mllib/clustering/KMeansModel.scala       |    36 +-
 .../apache/spark/mllib/clustering/LDAModel.scala   |    87 +-
 .../spark/mllib/clustering/LDAOptimizer.scala      |     6 +-
 .../spark/mllib/clustering/LocalKMeans.scala       |    10 +-
 .../clustering/PowerIterationClustering.scala      |    17 +-
 .../spark/mllib/clustering/StreamingKMeans.scala   |    19 +-
 .../spark/mllib/evaluation/AreaUnderCurve.scala    |     5 +-
 .../evaluation/BinaryClassificationMetrics.scala   |     9 +-
 .../spark/mllib/evaluation/MulticlassMetrics.scala |    10 +-
 .../spark/mllib/evaluation/MultilabelMetrics.scala |    14 +-
 .../spark/mllib/evaluation/RankingMetrics.scala    |   100 +-
 .../spark/mllib/evaluation/RegressionMetrics.scala |     4 +-
 .../apache/spark/mllib/feature/ChiSqSelector.scala |     8 +-
 .../org/apache/spark/mllib/feature/HashingTF.scala |     2 +-
 .../scala/org/apache/spark/mllib/feature/IDF.scala |    14 +-
 .../org/apache/spark/mllib/feature/Word2Vec.scala  |    53 +-
 .../apache/spark/mllib/fpm/AssociationRules.scala  |    11 +-
 .../org/apache/spark/mllib/fpm/FPGrowth.scala      |    17 +-
 .../apache/spark/mllib/fpm/LocalPrefixSpan.scala   |     2 +-
 .../org/apache/spark/mllib/fpm/PrefixSpan.scala    |    38 +-
 .../org/apache/spark/mllib/linalg/ARPACK.scala     |     4 +-
 .../scala/org/apache/spark/mllib/linalg/BLAS.scala |     4 +-
 .../org/apache/spark/mllib/linalg/LAPACK.scala     |     4 +-
 .../org/apache/spark/mllib/linalg/Matrices.scala   |    13 +-
 .../org/apache/spark/mllib/linalg/Vectors.scala    |    22 +-
 .../mllib/linalg/distributed/BlockMatrix.scala     |    27 +-
 .../linalg/distributed/IndexedRowMatrix.scala      |     4 +-
 .../spark/mllib/linalg/distributed/RowMatrix.scala |    85 +-
 .../spark/mllib/optimization/GradientDescent.scala |    26 +-
 .../apache/spark/mllib/optimization/LBFGS.scala    |     9 +-
 .../apache/spark/mllib/pmml/PMMLExportable.scala   |     5 +-
 .../BinaryClassificationPMMLModelExport.scala      |    12 +-
 .../export/GeneralizedLinearPMMLModelExport.scala  |    12 +-
 .../mllib/pmml/export/KMeansPMMLModelExport.scala  |    10 +-
 .../spark/mllib/pmml/export/PMMLModelExport.scala  |    18 +-
 .../mllib/pmml/export/PMMLModelExportFactory.scala |     2 +-
 .../spark/mllib/rdd/MLPairRDDFunctions.scala       |    34 +-
 .../recommendation/MatrixFactorizationModel.scala  |    15 +-
 .../mllib/regression/IsotonicRegression.scala      |   152 +-
 .../org/apache/spark/mllib/regression/Lasso.scala  |     4 +-
 .../spark/mllib/regression/LinearRegression.scala  |     4 +-
 .../spark/mllib/regression/RegressionModel.scala   |     4 +-
 .../spark/mllib/regression/RidgeRegression.scala   |     4 +-
 .../regression/StreamingLinearAlgorithm.scala      |     7 +-
 .../mllib/regression/impl/GLMRegressionModel.scala |     4 +-
 .../org/apache/spark/mllib/stat/Statistics.scala   |    11 +-
 .../stat/correlation/SpearmanCorrelation.scala     |     2 +-
 .../apache/spark/mllib/stat/test/ChiSqTest.scala   |    17 +-
 .../mllib/stat/test/StreamingTestMethod.scala      |     2 +-
 .../org/apache/spark/mllib/tree/DecisionTree.scala |     2 +-
 .../org/apache/spark/mllib/tree/RandomForest.scala |     2 +-
 .../spark/mllib/tree/configuration/Strategy.scala  |     2 +-
 .../apache/spark/mllib/tree/impurity/Entropy.scala |     2 +-
 .../apache/spark/mllib/tree/impurity/Gini.scala    |     2 +-
 .../spark/mllib/tree/impurity/Variance.scala       |     2 +-
 .../spark/mllib/tree/model/DecisionTreeModel.scala |    31 +-
 .../mllib/tree/model/InformationGainStats.scala    |     4 +-
 .../org/apache/spark/mllib/tree/model/Node.scala   |     2 +-
 .../apache/spark/mllib/tree/model/Predict.scala    |     4 +-
 .../mllib/tree/model/treeEnsembleModels.scala      |    32 +-
 .../apache/spark/mllib/util/DataValidators.scala   |     9 +-
 .../spark/mllib/util/LinearDataGenerator.scala     |     2 +-
 .../apache/spark/mllib/util/MFDataGenerator.scala  |    11 +-
 .../org/apache/spark/mllib/util/MLUtils.scala      |    49 +-
 .../apache/spark/mllib/util/modelSaveLoad.scala    |     5 +-
 .../sql/ml/InternalFunctionRegistration.scala      |   109 +
 .../java/org/apache/spark/SharedSparkSession.java  |     8 +-
 .../org/apache/spark/ml/JavaPipelineSuite.java     |     4 +-
 .../ml/attribute/JavaAttributeGroupSuite.java      |    16 +-
 .../spark/ml/attribute/JavaAttributeSuite.java     |    16 +-
 .../JavaDecisionTreeClassifierSuite.java           |     2 +-
 .../ml/classification/JavaGBTClassifierSuite.java  |     2 +-
 .../JavaLogisticRegressionSuite.java               |    57 +-
 .../JavaMultilayerPerceptronClassifierSuite.java   |     6 +-
 .../ml/classification/JavaNaiveBayesSuite.java     |     4 +-
 .../ml/classification/JavaOneVsRestSuite.java      |    21 +-
 .../JavaRandomForestClassifierSuite.java           |    12 +-
 .../spark/ml/clustering/JavaKMeansSuite.java       |     8 +-
 .../spark/ml/feature/JavaBucketizerSuite.java      |    10 +-
 .../org/apache/spark/ml/feature/JavaDCTSuite.java  |     6 +-
 .../spark/ml/feature/JavaHashingTFSuite.java       |     6 +-
 .../spark/ml/feature/JavaNormalizerSuite.java      |     2 +-
 .../org/apache/spark/ml/feature/JavaPCASuite.java  |     6 +-
 .../ml/feature/JavaPolynomialExpansionSuite.java   |     6 +-
 .../spark/ml/feature/JavaStandardScalerSuite.java  |     2 +-
 .../ml/feature/JavaStopWordsRemoverSuite.java      |     2 +-
 .../spark/ml/feature/JavaStringIndexerSuite.java   |     6 +-
 .../spark/ml/feature/JavaTargetEncoderSuite.java   |   157 +
 .../spark/ml/feature/JavaTokenizerSuite.java       |     6 +-
 .../spark/ml/feature/JavaVectorAssemblerSuite.java |     6 +-
 .../spark/ml/feature/JavaVectorIndexerSuite.java   |     8 +-
 .../spark/ml/feature/JavaVectorSlicerSuite.java    |     6 +-
 .../apache/spark/ml/feature/JavaWord2VecSuite.java |     6 +-
 .../spark/ml/linalg/JavaSQLDataTypesSuite.java     |     8 +-
 .../org/apache/spark/ml/param/JavaParamsSuite.java |    12 +-
 .../regression/JavaDecisionTreeRegressorSuite.java |     2 +-
 .../spark/ml/regression/JavaGBTRegressorSuite.java |     2 +-
 .../ml/regression/JavaLinearRegressionSuite.java   |     6 +-
 .../regression/JavaRandomForestRegressorSuite.java |    12 +-
 .../ml/source/libsvm/JavaLibSVMRelationSuite.java  |    20 +-
 .../ml/stat/JavaKolmogorovSmirnovTestSuite.java    |    10 +-
 .../apache/spark/ml/stat/JavaSummarizerSuite.java  |     8 +-
 .../spark/ml/tuning/JavaCrossValidatorSuite.java   |    10 +-
 .../spark/ml/util/JavaDefaultReadWriteSuite.java   |    22 +-
 .../JavaLogisticRegressionSuite.java               |     8 +-
 .../mllib/classification/JavaNaiveBayesSuite.java  |    10 +-
 .../spark/mllib/classification/JavaSVMSuite.java   |     8 +-
 .../JavaStreamingLogisticRegressionSuite.java      |    11 +-
 .../mllib/clustering/JavaBisectingKMeansSuite.java |    16 +-
 .../mllib/clustering/JavaGaussianMixtureSuite.java |     4 +-
 .../spark/mllib/clustering/JavaKMeansSuite.java    |     4 +-
 .../spark/mllib/clustering/JavaLDASuite.java       |     7 +-
 .../mllib/clustering/JavaStreamingKMeansSuite.java |    11 +-
 .../mllib/evaluation/JavaRankingMetricsSuite.java  |    38 +-
 .../apache/spark/mllib/feature/JavaTfIdfSuite.java |    10 +-
 .../spark/mllib/feature/JavaWord2VecSuite.java     |    15 +-
 .../spark/mllib/fpm/JavaAssociationRulesSuite.java |     3 +-
 .../apache/spark/mllib/fpm/JavaFPGrowthSuite.java  |     6 +-
 .../spark/mllib/fpm/JavaPrefixSpanSuite.java       |     8 +-
 .../spark/mllib/linalg/JavaMatricesSuite.java      |     6 +-
 .../spark/mllib/linalg/JavaVectorsSuite.java       |     5 +-
 .../linalg/distributed/JavaRowMatrixSuite.java     |     2 +-
 .../spark/mllib/random/JavaRandomRDDsSuite.java    |    55 +-
 .../spark/mllib/recommendation/JavaALSSuite.java   |    22 +-
 .../regression/JavaIsotonicRegressionSuite.java    |    16 +-
 .../spark/mllib/regression/JavaLassoSuite.java     |     8 +-
 .../regression/JavaLinearRegressionSuite.java      |     8 +-
 .../mllib/regression/JavaRidgeRegressionSuite.java |     8 +-
 .../JavaStreamingLinearRegressionSuite.java        |    11 +-
 .../spark/mllib/stat/JavaStatisticsSuite.java      |    12 +-
 .../spark/mllib/tree/JavaDecisionTreeSuite.java    |     8 +-
 .../apache/spark/mllib/util/JavaMLUtilsSuite.java  |    16 +-
 .../org.apache.spark.ml.util.MLFormatRegister      |    17 +
 .../date=2018-01/29.5.a_b_EGDP022204.jpg           |   Bin
 .../cls=kittens/date=2018-01/not-image.txt         |     0
 .../partitioned/cls=kittens/date=2018-02/54893.jpg |   Bin
 .../cls=kittens/date=2018-02/DP153539.jpg          |   Bin
 .../cls=kittens/date=2018-02/DP802813.jpg          |   Bin
 .../cls=multichannel/date=2018-01/BGRA.png         |   Bin
 .../date=2018-01/BGRA_alpha_60.png                 |   Bin
 .../cls=multichannel/date=2018-02/chr30.4.184.jpg  |   Bin
 .../cls=multichannel/date=2018-02/grayscale.jpg    |   Bin
 .../src/test/resources}/iris_libsvm.txt            |     0
 mllib/src/test/resources/log4j.properties          |    28 -
 mllib/src/test/resources/log4j2.properties         |    32 +
 .../scala/org/apache/spark/ml/FunctionsSuite.scala |    43 +-
 .../scala/org/apache/spark/ml/MLEventsSuite.scala  |    59 +-
 .../scala/org/apache/spark/ml/PipelineSuite.scala  |    16 +-
 .../scala/org/apache/spark/ml/ann/ANNSuite.scala   |     5 +-
 .../org/apache/spark/ml/ann/GradientSuite.scala    |     2 +-
 .../spark/ml/classification/ClassifierSuite.scala  |    52 +-
 .../DecisionTreeClassifierSuite.scala              |    41 +-
 .../ml/classification/FMClassifierSuite.scala      |    66 +-
 .../ml/classification/GBTClassifierSuite.scala     |    55 +-
 .../spark/ml/classification/LinearSVCSuite.scala   |    65 +-
 .../classification/LogisticRegressionSuite.scala   |   161 +-
 .../MultilayerPerceptronClassifierSuite.scala      |    45 +-
 .../spark/ml/classification/NaiveBayesSuite.scala  |   188 +-
 .../RandomForestClassifierSuite.scala              |    19 +-
 .../spark/ml/clustering/BisectingKMeansSuite.scala |    15 +-
 .../spark/ml/clustering/GaussianMixtureSuite.scala |     7 +-
 .../apache/spark/ml/clustering/KMeansSuite.scala   |   386 +-
 .../org/apache/spark/ml/clustering/LDASuite.scala  |     4 +
 .../clustering/PowerIterationClusteringSuite.scala |     8 +-
 .../ml/evaluation/ClusteringEvaluatorSuite.scala   |     6 +-
 .../apache/spark/ml/feature/BinarizerSuite.scala   |    20 +-
 .../apache/spark/ml/feature/BucketizerSuite.scala  |    34 +-
 .../spark/ml/feature/ChiSqSelectorSuite.scala      |    10 +-
 .../spark/ml/feature/CountVectorizerSuite.scala    |    17 +-
 .../org/apache/spark/ml/feature/DCTSuite.scala     |     4 +-
 .../spark/ml/feature/FeatureHasherSuite.scala      |    26 +-
 .../org/apache/spark/ml/feature/IDFSuite.scala     |    14 +-
 .../org/apache/spark/ml/feature/ImputerSuite.scala |    61 +-
 .../apache/spark/ml/feature/InstanceSuite.scala    |     7 +-
 .../apache/spark/ml/feature/InteractionSuite.scala |    39 +-
 .../org/apache/spark/ml/feature/LSHTest.scala      |    11 +-
 .../apache/spark/ml/feature/NormalizerSuite.scala  |     4 +-
 .../spark/ml/feature/OneHotEncoderSuite.scala      |    18 +-
 .../org/apache/spark/ml/feature/PCASuite.scala     |    11 +-
 .../ml/feature/QuantileDiscretizerSuite.scala      |    65 +-
 .../apache/spark/ml/feature/RFormulaSuite.scala    |    11 +
 .../spark/ml/feature/RobustScalerSuite.scala       |    21 +-
 .../spark/ml/feature/StopWordsRemoverSuite.scala   |    21 +
 .../spark/ml/feature/StringIndexerSuite.scala      |    51 +-
 .../spark/ml/feature/TargetEncoderSuite.scala      |   524 +
 .../feature/UnivariateFeatureSelectorSuite.scala   |    18 +-
 .../feature/VarianceThresholdSelectorSuite.scala   |     7 +-
 .../spark/ml/feature/VectorAssemblerSuite.scala    |    18 +-
 .../spark/ml/feature/VectorIndexerSuite.scala      |    15 +-
 .../spark/ml/feature/VectorSizeHintSuite.scala     |     4 +-
 .../spark/ml/feature/VectorSlicerSuite.scala       |     4 +-
 .../apache/spark/ml/feature/Word2VecSuite.scala    |    10 +-
 .../IterativelyReweightedLeastSquaresSuite.scala   |     2 +-
 .../BinaryLogisticBlockAggregatorSuite.scala       |    26 +-
 .../aggregator/HingeBlockAggregatorSuite.scala     |    22 +-
 .../aggregator/HuberBlockAggregatorSuite.scala     |    24 +-
 .../LeastSquaresBlockAggregatorSuite.scala         |    11 +-
 .../MultinomialLogisticBlockAggregatorSuite.scala  |    34 +-
 .../apache/spark/ml/recommendation/ALSSuite.scala  |   289 +-
 .../spark/ml/recommendation/CollectTopKSuite.scala |   112 +
 .../recommendation/TopByKeyAggregatorSuite.scala   |    73 -
 .../ml/regression/AFTSurvivalRegressionSuite.scala |    67 +-
 .../ml/regression/DecisionTreeRegressorSuite.scala |    12 +-
 .../spark/ml/regression/FMRegressorSuite.scala     |    59 +
 .../spark/ml/regression/GBTRegressorSuite.scala    |    26 +-
 .../GeneralizedLinearRegressionSuite.scala         |    88 +-
 .../ml/regression/IsotonicRegressionSuite.scala    |    32 +
 .../ml/regression/LinearRegressionSuite.scala      |    78 +-
 .../ml/regression/RandomForestRegressorSuite.scala |    13 +-
 .../ml/source/image/ImageFileFormatSuite.scala     |    38 +-
 .../ml/source/libsvm/LibSVMRelationSuite.scala     |    22 +-
 .../apache/spark/ml/stat/ChiSquareTestSuite.scala  |     3 +-
 .../apache/spark/ml/stat/CorrelationSuite.scala    |     4 +-
 .../org/apache/spark/ml/stat/FValueTestSuite.scala |     7 +-
 .../spark/ml/stat/KolmogorovSmirnovTestSuite.scala |     8 +-
 .../spark/ml/tree/impl/BaggedPointSuite.scala      |    13 +-
 .../ml/tree/impl/GradientBoostedTreesSuite.scala   |    10 +-
 .../spark/ml/tree/impl/RandomForestSuite.scala     |    87 +-
 .../org/apache/spark/ml/tree/impl/TreeTests.scala  |     5 +-
 .../spark/ml/tuning/CrossValidatorSuite.scala      |    16 +-
 .../spark/ml/tuning/ParamGridBuilderSuite.scala    |     2 +-
 .../ml/tuning/TrainValidationSplitSuite.scala      |     2 +-
 .../spark/ml/util/DefaultReadWriteTest.scala       |    59 +-
 .../scala/org/apache/spark/ml/util/MLTest.scala    |   166 +-
 .../org/apache/spark/ml/util/MLTestingUtils.scala  |     2 +-
 .../apache/spark/ml/util/PMMLReadWriteTest.scala   |     2 +-
 .../scala/org/apache/spark/ml/util/PMMLUtils.scala |     5 +-
 .../mllib/api/python/PythonMLLibAPISuite.scala     |     2 +-
 .../classification/LogisticRegressionSuite.scala   |    23 +-
 .../mllib/classification/NaiveBayesSuite.scala     |    20 +-
 .../spark/mllib/classification/SVMSuite.scala      |    11 +-
 .../mllib/clustering/DistanceMeasureSuite.scala    |     4 +-
 .../mllib/clustering/GaussianMixtureSuite.scala    |    11 +-
 .../spark/mllib/clustering/KMeansSuite.scala       |     7 +-
 .../apache/spark/mllib/clustering/LDASuite.scala   |    27 +-
 .../mllib/clustering/StreamingKMeansSuite.scala    |     2 +-
 .../BinaryClassificationMetricsSuite.scala         |    15 +-
 .../mllib/evaluation/RankingMetricsSuite.scala     |    87 +-
 .../spark/mllib/feature/ChiSqSelectorSuite.scala   |    10 +-
 .../mllib/feature/ElementwiseProductSuite.scala    |     9 +-
 .../spark/mllib/feature/HashingTFSuite.scala       |     7 +-
 .../spark/mllib/feature/NormalizerSuite.scala      |    15 +-
 .../org/apache/spark/mllib/feature/PCASuite.scala  |     8 +-
 .../spark/mllib/feature/StandardScalerSuite.scala  |    53 +-
 .../apache/spark/mllib/feature/Word2VecSuite.scala |    14 +-
 .../spark/mllib/fpm/AssociationRulesSuite.scala    |     4 +-
 .../org/apache/spark/mllib/fpm/FPGrowthSuite.scala |     2 +-
 .../apache/spark/mllib/fpm/PrefixSpanSuite.scala   |     5 +-
 .../org/apache/spark/mllib/linalg/BLASSuite.scala  |     2 +-
 .../apache/spark/mllib/linalg/MatricesSuite.scala  |    10 +-
 .../mllib/linalg/UDTSerializationBenchmark.scala   |     7 +-
 .../apache/spark/mllib/linalg/VectorsSuite.scala   |    26 +-
 .../linalg/distributed/BlockMatrixSuite.scala      |     8 +-
 .../mllib/linalg/distributed/RowMatrixSuite.scala  |     5 +-
 .../mllib/optimization/GradientDescentSuite.scala  |     4 +-
 .../spark/mllib/optimization/LBFGSSuite.scala      |     6 +-
 .../BinaryClassificationPMMLModelExportSuite.scala |     6 +-
 .../GeneralizedLinearPMMLModelExportSuite.scala    |     8 +-
 .../pmml/export/KMeansPMMLModelExportSuite.scala   |     4 +-
 .../pmml/export/PMMLModelExportFactorySuite.scala  |     2 +-
 .../spark/mllib/random/RandomRDDsSuite.scala       |    12 +-
 .../spark/mllib/recommendation/ALSSuite.scala      |     7 +-
 .../mllib/regression/IsotonicRegressionSuite.scala |   184 +-
 .../apache/spark/mllib/regression/LassoSuite.scala |    19 +-
 .../mllib/regression/LinearRegressionSuite.scala   |    10 +-
 .../mllib/regression/RidgeRegressionSuite.scala    |     7 +-
 .../StreamingLinearRegressionSuite.scala           |     2 +-
 .../apache/spark/mllib/stat/CorrelationSuite.scala |    14 +-
 .../spark/mllib/stat/HypothesisTestSuite.scala     |    13 +-
 .../spark/mllib/stat/KernelDensitySuite.scala      |     5 +-
 .../spark/mllib/tree/DecisionTreeSuite.scala       |    61 +-
 .../spark/mllib/tree/EnsembleTestHelper.scala      |     2 +-
 .../mllib/tree/GradientBoostedTreesSuite.scala     |    47 +-
 .../apache/spark/mllib/tree/ImpuritySuite.scala    |     5 +-
 .../spark/mllib/tree/RandomForestSuite.scala       |    17 +-
 .../mllib/util/LocalClusterSparkContext.scala      |     4 +-
 .../org/apache/spark/mllib/util/MLUtilsSuite.scala |    18 +-
 .../spark/mllib/util/MLlibTestSparkContext.scala   |    13 +-
 pom.xml                                            |  2238 +-
 project/MimaBuild.scala                            |     6 +-
 project/MimaExcludes.scala                         |   159 +-
 project/SparkBuild.scala                           |  1351 +-
 project/build.properties                           |     2 +-
 project/plugins.sbt                                |    34 +-
 pyproject.toml                                     |    70 +
 python/.coveragerc                                 |    23 +
 python/.gitignore                                  |     4 +
 python/MANIFEST.in                                 |     9 +-
 python/README.md                                   |     2 +-
 python/asv                                         |    42 +
 python/benchmarks/README.md                        |    86 +
 python/benchmarks/__init__.py                      |    16 +
 python/benchmarks/asv.conf.json                    |    30 +
 python/benchmarks/bench_arrow.py                   |   116 +
 python/benchmarks/bench_eval_type.py               |  1814 ++
 python/conf_viztracer/daemon_viztracer.py          |    47 +
 python/conf_viztracer/spark-defaults.conf          |    22 +
 python/conf_viztracer/worker_viztracer.py          |    32 +
 python/conf_vscode/README.md                       |    43 +
 python/conf_vscode/sitecustomize.py                |    51 +
 python/docs/Makefile                               |    19 +-
 python/docs/make.bat                               |    19 +-
 python/docs/make2.bat                              |    25 +-
 python/docs/source/_static/copybutton.js           |    66 -
 python/docs/source/_static/css/pyspark.css         |     9 +-
 .../_templates/autosummary/accessor_attribute.rst  |     6 +
 .../_templates/autosummary/accessor_method.rst     |     6 +
 .../_templates/autosummary/class_with_docs.rst     |     4 +-
 .../source/_templates/autosummary/plot_class.rst   |    53 +
 python/docs/source/_templates/layout.html          |    30 +
 python/docs/source/_templates/spark_footer.html    |     3 +
 python/docs/source/conf.py                         |   108 +-
 python/docs/source/development/contributing.rst    |   137 +-
 python/docs/source/development/debugging.rst       |   434 +-
 python/docs/source/development/index.rst           |     2 +
 python/docs/source/development/logger.rst          |   169 +
 python/docs/source/development/testing.rst         |    38 +-
 python/docs/source/getting_started/index.rst       |     3 +
 python/docs/source/getting_started/install.rst     |   266 +-
 .../getting_started/quickstart_connect.ipynb       |   144 +
 .../source/getting_started/quickstart_df.ipynb     |    47 +-
 .../source/getting_started/quickstart_ps.ipynb     |    56 +-
 .../source/getting_started/testing_pyspark.ipynb   |   485 +
 python/docs/source/index.rst                       |   194 +-
 python/docs/source/migration_guide/index.rst       |    31 +-
 .../source/migration_guide/koalas_to_pyspark.rst   |     8 +-
 .../migration_guide/pyspark_1.0_1.2_to_1.3.rst     |    23 -
 .../source/migration_guide/pyspark_1.4_to_1.5.rst  |    25 -
 .../source/migration_guide/pyspark_2.2_to_2.3.rst  |    30 -
 .../pyspark_2.3.0_to_2.3.1_above.rst               |    23 -
 .../source/migration_guide/pyspark_2.3_to_2.4.rst  |    23 -
 .../source/migration_guide/pyspark_2.4_to_3.0.rst  |    44 -
 .../source/migration_guide/pyspark_3.1_to_3.2.rst  |    31 -
 .../source/migration_guide/pyspark_upgrade.rst     |   205 +
 python/docs/source/reference/index.rst             |    12 +-
 python/docs/source/reference/pyspark.errors.rst    |    78 +
 python/docs/source/reference/pyspark.logger.rst    |    47 +
 python/docs/source/reference/pyspark.ml.rst        |    26 +
 .../source/reference/pyspark.pandas/extensions.rst |    18 +
 .../docs/source/reference/pyspark.pandas/frame.rst |    61 +-
 .../reference/pyspark.pandas/general_functions.rst |    45 +-
 .../source/reference/pyspark.pandas/groupby.rst    |    25 +-
 .../docs/source/reference/pyspark.pandas/index.rst |     6 +
 .../source/reference/pyspark.pandas/indexing.rst   |    70 +-
 python/docs/source/reference/pyspark.pandas/io.rst |    26 +-
 python/docs/source/reference/pyspark.pandas/ml.rst |    18 +
 .../source/reference/pyspark.pandas/options.rst    |    37 +
 .../source/reference/pyspark.pandas/resampling.rst |    41 +
 .../source/reference/pyspark.pandas/series.rst     |    58 +-
 .../source/reference/pyspark.pandas/testing.rst    |    33 +
 .../source/reference/pyspark.pandas/window.rst     |    37 +-
 python/docs/source/reference/pyspark.pipelines.rst |    33 +
 python/docs/source/reference/pyspark.rst           |    14 +
 python/docs/source/reference/pyspark.sql.rst       |   649 -
 python/docs/source/reference/pyspark.sql/avro.rst  |    28 +
 .../docs/source/reference/pyspark.sql/catalog.rst  |    64 +
 .../docs/source/reference/pyspark.sql/column.rst   |    64 +
 .../source/reference/pyspark.sql/configuration.rst |    27 +
 .../source/reference/pyspark.sql/core_classes.rst  |    54 +
 .../source/reference/pyspark.sql/data_types.rst    |    55 +
 .../source/reference/pyspark.sql/dataframe.rst     |   191 +
 .../source/reference/pyspark.sql/datasource.rst    |    45 +
 .../source/reference/pyspark.sql/functions.rst     |   755 +
 .../docs/source/reference/pyspark.sql/grouping.rst |    42 +
 python/docs/source/reference/pyspark.sql/index.rst |    47 +
 python/docs/source/reference/pyspark.sql/io.rst    |    70 +
 .../source/reference/pyspark.sql/observation.rst   |    27 +
 .../docs/source/reference/pyspark.sql/protobuf.rst |    28 +
 python/docs/source/reference/pyspark.sql/row.rst   |    27 +
 .../source/reference/pyspark.sql/spark_session.rst |    94 +
 .../reference/pyspark.sql/stateful_processor.rst   |    30 +
 python/docs/source/reference/pyspark.sql/udf.rst   |    32 +
 python/docs/source/reference/pyspark.sql/udtf.rst  |    30 +
 .../source/reference/pyspark.sql/variant_val.rst   |    29 +
 .../docs/source/reference/pyspark.sql/window.rst   |    38 +
 python/docs/source/reference/pyspark.ss.rst        |    90 -
 .../source/reference/pyspark.ss/core_classes.rst   |    32 +
 python/docs/source/reference/pyspark.ss/index.rst  |    30 +
 python/docs/source/reference/pyspark.ss/io.rst     |    49 +
 .../reference/pyspark.ss/query_management.rst      |    45 +
 python/docs/source/reference/pyspark.streaming.rst |     6 +-
 python/docs/source/reference/pyspark.testing.rst   |    29 +
 python/docs/source/tutorial/arrow_pandas.rst       |    24 +
 python/docs/source/tutorial/index.rst              |    37 +
 .../tutorial/pandas_on_spark/best_practices.rst    |   405 +
 .../docs/source/tutorial/pandas_on_spark/faq.rst   |    57 +
 .../tutorial/pandas_on_spark/from_to_dbms.rst      |   125 +
 .../docs/source/tutorial/pandas_on_spark/index.rst |    34 +
 .../source/tutorial/pandas_on_spark/options.rst    |   343 +
 .../tutorial/pandas_on_spark/pandas_pyspark.rst    |   136 +
 .../tutorial/pandas_on_spark/transform_apply.rst   |   137 +
 .../source/tutorial/pandas_on_spark/typehints.rst  |   264 +
 .../docs/source/tutorial/pandas_on_spark/types.rst |   242 +
 python/docs/source/tutorial/python_packaging.rst   |   312 +
 python/docs/source/tutorial/sql/arrow_pandas.rst   |   461 +
 .../docs/source/tutorial/sql/arrow_python_udf.rst  |   496 +
 .../docs/source/tutorial/sql/arrow_python_udtf.rst |   655 +
 python/docs/source/tutorial/sql/index.rst          |    32 +
 .../source/tutorial/sql/python_data_source.rst     |   624 +
 python/docs/source/tutorial/sql/python_udtf.rst    |   558 +
 .../docs/source/tutorial/sql/type_conversions.rst  |   254 +
 .../source/user_guide/ansi_migration_guide.ipynb   |   272 +
 python/docs/source/user_guide/arrow_pandas.rst     |    24 -
 .../user_guide/assets/pyspark-udf-profile.png      |   Bin 0 -> 281384 bytes
 .../source/user_guide/assets/pyspark-ui-print.png  |   Bin 0 -> 206336 bytes
 .../user_guide/assets/pyspark-ui-sql-broadcast.png |   Bin 0 -> 84476 bytes
 .../source/user_guide/assets/pyspark-ui-sql.png    |   Bin 0 -> 365322 bytes
 python/docs/source/user_guide/bugbusting.ipynb     |  1307 +
 python/docs/source/user_guide/dataframes.ipynb     |   800 +
 python/docs/source/user_guide/dataprep.ipynb       |  1420 +
 python/docs/source/user_guide/index.rst            |    28 +-
 python/docs/source/user_guide/loadandbehold.ipynb  |   488 +
 .../user_guide/pandas_on_spark/best_practices.rst  |   313 -
 .../docs/source/user_guide/pandas_on_spark/faq.rst |    39 -
 .../user_guide/pandas_on_spark/from_to_dbms.rst    |   107 -
 .../source/user_guide/pandas_on_spark/index.rst    |    33 -
 .../source/user_guide/pandas_on_spark/options.rst  |   276 -
 .../user_guide/pandas_on_spark/pandas_pyspark.rst  |   118 -
 .../user_guide/pandas_on_spark/transform_apply.rst |   121 -
 .../user_guide/pandas_on_spark/typehints.rst       |   136 -
 .../source/user_guide/pandas_on_spark/types.rst    |   228 -
 python/docs/source/user_guide/python_packaging.rst |   253 -
 python/docs/source/user_guide/sql.ipynb            |  1192 +
 python/docs/source/user_guide/sql/arrow_pandas.rst |   421 -
 python/docs/source/user_guide/sql/index.rst        |    27 -
 python/docs/source/user_guide/touroftypes.ipynb    |  1190 +
 python/docs/source/user_guide/udfandudtf.ipynb     |  1040 +
 python/lib/py4j-0.10.9.2-src.zip                   |   Bin 41839 -> 0 bytes
 python/lib/py4j-0.10.9.9-src.zip                   |   Bin 0 -> 43108 bytes
 python/mypy.ini                                    |   132 +-
 python/packaging/classic/setup.cfg                 |    22 +
 python/packaging/classic/setup.py                  |   421 +
 python/packaging/client/setup.cfg                  |    22 +
 python/packaging/client/setup.py                   |   236 +
 .../packaging/connect/pyspark_connect/__init__.py  |    26 +
 python/packaging/connect/setup.cfg                 |    22 +
 python/packaging/connect/setup.py                  |   148 +
 python/pylintrc                                    |   404 -
 python/pyspark/__init__.py                         |   106 +-
 python/pyspark/__init__.pyi                        |    77 -
 python/pyspark/_globals.py                         |    19 +-
 python/pyspark/_typing.pyi                         |    22 +-
 python/pyspark/accumulators.py                     |   234 +-
 python/pyspark/accumulators.pyi                    |    73 -
 python/pyspark/broadcast.py                        |   209 -
 python/pyspark/broadcast.pyi                       |    48 -
 python/pyspark/cloudpickle/__init__.py             |    23 +-
 python/pyspark/cloudpickle/cloudpickle.py          |  1602 +-
 python/pyspark/cloudpickle/cloudpickle_fast.py     |   772 +-
 python/pyspark/cloudpickle/compat.py               |    13 -
 python/pyspark/conf.py                             |   114 +-
 python/pyspark/conf.pyi                            |    44 -
 python/pyspark/context.py                          |  1289 -
 python/pyspark/context.pyi                         |   194 -
 python/pyspark/core/__init__.py                    |    16 +
 python/pyspark/core/broadcast.py                   |   378 +
 python/pyspark/core/context.py                     |  2672 ++
 python/pyspark/core/files.py                       |   163 +
 python/pyspark/core/rdd.py                         |  5386 ++++
 python/pyspark/core/status.py                      |   149 +
 python/pyspark/daemon.py                           |   215 +-
 python/pyspark/errors/__init__.py                  |    90 +
 python/pyspark/errors/error-conditions.json        |  1228 +
 python/pyspark/errors/error_classes.py             |    28 +
 python/pyspark/errors/exceptions/__init__.py       |    33 +
 python/pyspark/errors/exceptions/base.py           |   494 +
 python/pyspark/errors/exceptions/captured.py       |   460 +
 python/pyspark/errors/exceptions/connect.py        |   598 +
 python/pyspark/errors/exceptions/tblib.py          |   340 +
 python/pyspark/errors/tests/__init__.py            |    16 +
 python/pyspark/errors/tests/connect/__init__.py    |    16 +
 .../errors/tests/connect/test_parity_traceback.py  |    28 +
 .../errors/tests/test_connect_errors_conversion.py |   412 +
 python/pyspark/errors/tests/test_errors.py         |   132 +
 python/pyspark/errors/tests/test_traceback.py      |   278 +
 python/pyspark/errors/utils.py                     |   381 +
 python/pyspark/errors_doc_gen.py                   |    62 +
 python/pyspark/files.py                            |    58 -
 python/pyspark/files.pyi                           |    24 -
 python/pyspark/find_spark_home.py                  |    59 +-
 python/pyspark/install.py                          |   140 +-
 python/pyspark/instrumentation_utils.py            |   186 +
 python/pyspark/java_gateway.py                     |    99 +-
 python/pyspark/join.py                             |    16 +-
 python/pyspark/logger/__init__.py                  |    24 +
 python/pyspark/logger/logger.py                    |   330 +
 python/pyspark/logger/tests/__init__.py            |    16 +
 python/pyspark/logger/tests/connect/__init__.py    |    16 +
 .../logger/tests/connect/test_parity_logger.py     |    29 +
 python/pyspark/logger/tests/test_logger.py         |   219 +
 python/pyspark/logger/worker_io.py                 |   300 +
 python/pyspark/loose_version.py                    |    75 +
 python/pyspark/memory_profiler_ext.py              |   190 +
 python/pyspark/messages/README.md                  |    23 +
 python/pyspark/messages/__init__.py                |    22 +
 python/pyspark/messages/zero_copy_byte_stream.py   |   176 +
 python/pyspark/ml/__init__.py                      |    55 +-
 python/pyspark/ml/_typing.pyi                      |    22 +-
 python/pyspark/ml/base.py                          |   137 +-
 python/pyspark/ml/base.pyi                         |   107 -
 python/pyspark/ml/classification.py                |  2296 +-
 python/pyspark/ml/classification.pyi               |   944 -
 python/pyspark/ml/clustering.py                    |  1020 +-
 python/pyspark/ml/clustering.pyi                   |   437 -
 python/pyspark/ml/common.py                        |   116 +-
 python/pyspark/ml/common.pyi                       |    26 -
 python/pyspark/ml/connect/__init__.py              |    47 +
 python/pyspark/ml/connect/base.py                  |   360 +
 python/pyspark/ml/connect/classification.py        |   398 +
 python/pyspark/ml/connect/evaluation.py            |   300 +
 python/pyspark/ml/connect/feature.py               |   428 +
 python/pyspark/ml/connect/functions.py             |    91 +
 python/pyspark/ml/connect/io_utils.py              |   341 +
 python/pyspark/ml/connect/pipeline.py              |   270 +
 python/pyspark/ml/connect/proto.py                 |    85 +
 python/pyspark/ml/connect/readwrite.py             |   366 +
 python/pyspark/ml/connect/serialize.py             |   223 +
 python/pyspark/ml/connect/summarizer.py            |   121 +
 python/pyspark/ml/connect/tuning.py                |   581 +
 python/pyspark/ml/connect/util.py                  |   192 +
 python/pyspark/ml/deepspeed/__init__.py            |    16 +
 .../pyspark/ml/deepspeed/deepspeed_distributor.py  |   164 +
 python/pyspark/ml/deepspeed/tests/__init__.py      |    16 +
 .../deepspeed/tests/test_deepspeed_distributor.py  |   293 +
 python/pyspark/ml/dl_util.py                       |   148 +
 python/pyspark/ml/evaluation.py                    |   596 +-
 python/pyspark/ml/evaluation.pyi                   |   283 -
 python/pyspark/ml/feature.py                       |  3882 ++-
 python/pyspark/ml/feature.pyi                      |  1623 -
 python/pyspark/ml/fpm.py                           |   273 +-
 python/pyspark/ml/fpm.pyi                          |   109 -
 python/pyspark/ml/functions.py                     |   782 +-
 python/pyspark/ml/functions.pyi                    |    24 -
 python/pyspark/ml/image.py                         |   132 +-
 python/pyspark/ml/image.pyi                        |    40 -
 python/pyspark/ml/linalg/__init__.py               |   632 +-
 python/pyspark/ml/linalg/__init__.pyi              |   251 -
 python/pyspark/ml/model_cache.py                   |    55 +
 python/pyspark/ml/param/__init__.py                |   179 +-
 python/pyspark/ml/param/__init__.pyi               |    96 -
 python/pyspark/ml/param/_shared_params_code_gen.py |   404 +-
 .../pyspark/ml/param/_shared_params_code_gen.pyi   |    19 -
 python/pyspark/ml/param/shared.py                  |   573 +-
 python/pyspark/ml/param/shared.pyi                 |   192 -
 python/pyspark/ml/pipeline.py                      |   223 +-
 python/pyspark/ml/pipeline.pyi                     |    97 -
 python/pyspark/ml/recommendation.py                |   331 +-
 python/pyspark/ml/recommendation.pyi               |   152 -
 python/pyspark/ml/regression.py                    |  1581 +-
 python/pyspark/ml/regression.pyi                   |   825 -
 python/pyspark/ml/stat.py                          |   212 +-
 python/pyspark/ml/stat.pyi                         |    77 -
 python/pyspark/ml/tests/connect/__init__.py        |    16 +
 .../pyspark/ml/tests/connect/test_connect_cache.py |   124 +
 .../tests/connect/test_connect_classification.py   |    50 +
 .../ml/tests/connect/test_connect_evaluation.py    |    37 +
 .../ml/tests/connect/test_connect_feature.py       |    41 +
 .../ml/tests/connect/test_connect_function.py      |    72 +
 .../tests/connect/test_connect_model_offloading.py |   312 +
 .../ml/tests/connect/test_connect_pipeline.py      |    51 +
 .../ml/tests/connect/test_connect_summarizer.py    |    37 +
 .../ml/tests/connect/test_connect_tuning.py        |    51 +
 .../connect/test_legacy_mode_classification.py     |   253 +
 .../tests/connect/test_legacy_mode_evaluation.py   |   190 +
 .../ml/tests/connect/test_legacy_mode_feature.py   |   200 +
 .../ml/tests/connect/test_legacy_mode_pipeline.py  |   196 +
 .../tests/connect/test_legacy_mode_summarizer.py   |    79 +
 .../ml/tests/connect/test_legacy_mode_tuning.py    |   314 +
 python/pyspark/ml/tests/connect/test_parity_als.py |    30 +
 .../ml/tests/connect/test_parity_classification.py |    30 +
 .../ml/tests/connect/test_parity_clustering.py     |    30 +
 .../ml/tests/connect/test_parity_evaluation.py     |    30 +
 .../ml/tests/connect/test_parity_feature.py        |    30 +
 python/pyspark/ml/tests/connect/test_parity_fpm.py |    30 +
 .../ml/tests/connect/test_parity_functions.py      |    48 +
 python/pyspark/ml/tests/connect/test_parity_ovr.py |    30 +
 .../ml/tests/connect/test_parity_pipeline.py       |    30 +
 .../ml/tests/connect/test_parity_regression.py     |    30 +
 .../pyspark/ml/tests/connect/test_parity_stat.py   |    30 +
 .../tests/connect/test_parity_torch_data_loader.py |    43 +
 .../tests/connect/test_parity_torch_distributor.py |   161 +
 .../pyspark/ml/tests/connect/test_parity_tuning.py |    30 +
 python/pyspark/ml/tests/test_algorithms.py         |   282 +-
 python/pyspark/ml/tests/test_als.py                |   131 +
 python/pyspark/ml/tests/test_base.py               |    34 +-
 python/pyspark/ml/tests/test_classification.py     |  1019 +
 python/pyspark/ml/tests/test_clustering.py         |   519 +
 python/pyspark/ml/tests/test_dl_util.py            |   174 +
 python/pyspark/ml/tests/test_evaluation.py         |   359 +-
 python/pyspark/ml/tests/test_feature.py            |  1776 +-
 python/pyspark/ml/tests/test_fpm.py                |   109 +
 python/pyspark/ml/tests/test_functions.py          |   584 +
 python/pyspark/ml/tests/test_image.py              |    42 +-
 python/pyspark/ml/tests/test_linalg.py             |   180 +-
 python/pyspark/ml/tests/test_model_cache.py        |    48 +
 python/pyspark/ml/tests/test_ovr.py                |   125 +
 python/pyspark/ml/tests/test_param.py              |   168 +-
 python/pyspark/ml/tests/test_persistence.py        |   241 +-
 python/pyspark/ml/tests/test_pipeline.py           |   265 +-
 python/pyspark/ml/tests/test_regression.py         |   698 +
 python/pyspark/ml/tests/test_stat.py               |   203 +-
 python/pyspark/ml/tests/test_training_summary.py   |   218 +-
 python/pyspark/ml/tests/test_tuning.py             |  1099 +-
 python/pyspark/ml/tests/test_util.py               |    50 +-
 python/pyspark/ml/tests/test_wrapper.py            |    50 +-
 python/pyspark/ml/tests/tuning/__init__.py         |    16 +
 python/pyspark/ml/tests/tuning/test_cv_io_basic.py |   144 +
 .../pyspark/ml/tests/tuning/test_cv_io_nested.py   |    90 +
 .../pyspark/ml/tests/tuning/test_cv_io_pipeline.py |   144 +
 python/pyspark/ml/tests/tuning/test_tuning.py      |   413 +
 .../pyspark/ml/tests/tuning/test_tvs_io_basic.py   |   130 +
 .../pyspark/ml/tests/tuning/test_tvs_io_nested.py  |    89 +
 .../ml/tests/tuning/test_tvs_io_pipeline.py        |   135 +
 .../ml/tests/typing/test_classification.yml        |    38 +
 .../pyspark/ml/tests/typing/test_clustering.yaml   |    33 +
 python/pyspark/ml/tests/typing/test_evaluation.yml |    28 +
 python/pyspark/ml/tests/typing/test_feature.yml    |    55 +
 python/pyspark/ml/tests/typing/test_param.yml      |    30 +
 python/pyspark/ml/tests/typing/test_readable.yml   |    28 +
 python/pyspark/ml/tests/typing/test_regression.yml |    53 +
 python/pyspark/ml/torch/__init__.py                |    16 +
 python/pyspark/ml/torch/data.py                    |   102 +
 python/pyspark/ml/torch/distributor.py             |  1103 +
 python/pyspark/ml/torch/log_communication.py       |   198 +
 python/pyspark/ml/torch/tests/__init__.py          |    16 +
 python/pyspark/ml/torch/tests/test_data_loader.py  |   127 +
 python/pyspark/ml/torch/tests/test_distributor.py  |   578 +
 .../ml/torch/tests/test_log_communication.py       |   166 +
 .../pyspark/ml/torch/torch_run_process_wrapper.py  |    83 +
 python/pyspark/ml/tree.py                          |   331 +-
 python/pyspark/ml/tree.pyi                         |   112 -
 python/pyspark/ml/tuning.py                        |  1115 +-
 python/pyspark/ml/tuning.pyi                       |   225 -
 python/pyspark/ml/util.py                          |   858 +-
 python/pyspark/ml/util.pyi                         |   136 -
 python/pyspark/ml/wrapper.py                       |   192 +-
 python/pyspark/ml/wrapper.pyi                      |    48 -
 python/pyspark/mllib/__init__.py                   |    18 +-
 python/pyspark/mllib/_typing.pyi                   |    20 +-
 python/pyspark/mllib/classification.py             |   426 +-
 python/pyspark/mllib/classification.pyi            |   151 -
 python/pyspark/mllib/clustering.py                 |   446 +-
 python/pyspark/mllib/clustering.pyi                |   198 -
 python/pyspark/mllib/common.py                     |    90 +-
 python/pyspark/mllib/common.pyi                    |    35 -
 python/pyspark/mllib/evaluation.py                 |   242 +-
 python/pyspark/mllib/evaluation.pyi                |    94 -
 python/pyspark/mllib/feature.py                    |   344 +-
 python/pyspark/mllib/feature.pyi                   |   167 -
 python/pyspark/mllib/fpm.py                        |    66 +-
 python/pyspark/mllib/fpm.pyi                       |    57 -
 python/pyspark/mllib/linalg/__init__.py            |   728 +-
 python/pyspark/mllib/linalg/__init__.pyi           |   286 -
 python/pyspark/mllib/linalg/distributed.py         |   307 +-
 python/pyspark/mllib/linalg/distributed.pyi        |   151 -
 python/pyspark/mllib/random.py                     |   216 +-
 python/pyspark/mllib/random.pyi                    |   126 -
 python/pyspark/mllib/recommendation.py             |   118 +-
 python/pyspark/mllib/recommendation.pyi            |    75 -
 python/pyspark/mllib/regression.py                 |   472 +-
 python/pyspark/mllib/regression.pyi                |   155 -
 python/pyspark/mllib/stat/KernelDensity.py         |    23 +-
 python/pyspark/mllib/stat/KernelDensity.pyi        |    27 -
 python/pyspark/mllib/stat/__init__.py              |    11 +-
 python/pyspark/mllib/stat/__init__.pyi             |    29 -
 python/pyspark/mllib/stat/_statistics.py           |   115 +-
 python/pyspark/mllib/stat/_statistics.pyi          |    69 -
 python/pyspark/mllib/stat/distribution.py          |    10 +-
 python/pyspark/mllib/stat/distribution.pyi         |    25 -
 python/pyspark/mllib/stat/test.py                  |    23 +-
 python/pyspark/mllib/stat/test.pyi                 |    39 -
 python/pyspark/mllib/tests/test_algorithms.py      |   170 +-
 python/pyspark/mllib/tests/test_feature.py         |    80 +-
 python/pyspark/mllib/tests/test_linalg.py          |   212 +-
 python/pyspark/mllib/tests/test_stat.py            |    57 +-
 .../mllib/tests/test_streaming_algorithms.py       |   155 +-
 python/pyspark/mllib/tests/test_util.py            |    37 +-
 python/pyspark/mllib/tree.py                       |   350 +-
 python/pyspark/mllib/tree.pyi                      |   126 -
 python/pyspark/mllib/util.py                       |   141 +-
 python/pyspark/mllib/util.pyi                      |    90 -
 python/pyspark/pandas/__init__.py                  |    61 +-
 python/pyspark/pandas/_typing.py                   |    10 +-
 python/pyspark/pandas/accessors.py                 |   229 +-
 python/pyspark/pandas/base.py                      |   481 +-
 python/pyspark/pandas/categorical.py               |   269 +-
 python/pyspark/pandas/config.py                    |   180 +-
 python/pyspark/pandas/correlation.py               |   278 +
 python/pyspark/pandas/data_type_ops/base.py        |   226 +-
 python/pyspark/pandas/data_type_ops/binary_ops.py  |    31 +-
 python/pyspark/pandas/data_type_ops/boolean_ops.py |   180 +-
 .../pandas/data_type_ops/categorical_ops.py        |    37 +-
 python/pyspark/pandas/data_type_ops/complex_ops.py |     2 +-
 python/pyspark/pandas/data_type_ops/date_ops.py    |    16 +-
 .../pyspark/pandas/data_type_ops/datetime_ops.py   |    82 +-
 python/pyspark/pandas/data_type_ops/null_ops.py    |    31 +-
 python/pyspark/pandas/data_type_ops/num_ops.py     |   464 +-
 python/pyspark/pandas/data_type_ops/string_ops.py  |    55 +-
 .../pyspark/pandas/data_type_ops/timedelta_ops.py  |   109 +
 python/pyspark/pandas/datetimes.py                 |   194 +-
 python/pyspark/pandas/exceptions.py                |    38 +-
 python/pyspark/pandas/extensions.py                |    28 +-
 python/pyspark/pandas/frame.py                     |  4645 ++-
 python/pyspark/pandas/generic.py                   |  1274 +-
 python/pyspark/pandas/groupby.py                   |  2062 +-
 python/pyspark/pandas/indexes/__init__.py          |     2 +-
 python/pyspark/pandas/indexes/base.py              |   610 +-
 python/pyspark/pandas/indexes/category.py          |   132 +-
 python/pyspark/pandas/indexes/datetimes.py         |   282 +-
 python/pyspark/pandas/indexes/multi.py             |   321 +-
 python/pyspark/pandas/indexes/numeric.py           |   197 -
 python/pyspark/pandas/indexes/timedelta.py         |   217 +
 python/pyspark/pandas/indexing.py                  |   386 +-
 python/pyspark/pandas/internal.py                  |   395 +-
 python/pyspark/pandas/missing/common.py            |    99 +-
 python/pyspark/pandas/missing/frame.py             |    53 +-
 python/pyspark/pandas/missing/general_functions.py |    43 +
 python/pyspark/pandas/missing/groupby.py           |    24 +-
 python/pyspark/pandas/missing/indexes.py           |    77 +-
 python/pyspark/pandas/missing/resample.py          |   101 +
 python/pyspark/pandas/missing/scalars.py           |    29 +
 python/pyspark/pandas/missing/series.py            |    63 +-
 python/pyspark/pandas/missing/window.py            |    78 +-
 python/pyspark/pandas/ml.py                        |   116 -
 python/pyspark/pandas/mlflow.py                    |    41 +-
 python/pyspark/pandas/namespace.py                 |  1561 +-
 python/pyspark/pandas/numpy_compat.py              |   228 +-
 python/pyspark/pandas/plot/__init__.py             |     2 +-
 python/pyspark/pandas/plot/core.py                 |   416 +-
 python/pyspark/pandas/plot/matplotlib.py           |   259 +-
 python/pyspark/pandas/plot/plotly.py               |   180 +-
 python/pyspark/pandas/resample.py                  |   795 +
 python/pyspark/pandas/series.py                    |  2286 +-
 python/pyspark/pandas/spark/accessors.py           |   114 +-
 python/pyspark/pandas/spark/functions.py           |    74 -
 python/pyspark/pandas/spark/utils.py               |    50 +-
 python/pyspark/pandas/sql_formatter.py             |   332 +
 python/pyspark/pandas/sql_processor.py             |    76 +-
 python/pyspark/pandas/strings.py                   |   323 +-
 python/pyspark/pandas/supported_api_gen.py         |   444 +
 python/pyspark/pandas/testing.py                   |   329 +
 .../pyspark/pandas/tests/computation/__init__.py   |    16 +
 .../pandas/tests/computation/test_any_all.py       |   274 +
 .../pandas/tests/computation/test_apply_func.py    |   582 +
 .../pandas/tests/computation/test_binary_ops.py    |   330 +
 .../pandas/tests/computation/test_combine.py       |   780 +
 .../pandas/tests/computation/test_compute.py       |   581 +
 .../pyspark/pandas/tests/computation/test_corr.py  |   220 +
 .../pandas/tests/computation/test_corrwith.py      |    77 +
 .../pyspark/pandas/tests/computation/test_cov.py   |   113 +
 .../pandas/tests/computation/test_cumulative.py    |   133 +
 .../pandas/tests/computation/test_describe.py      |   255 +
 .../pyspark/pandas/tests/computation/test_eval.py  |    73 +
 .../pandas/tests/computation/test_idxmax_idxmin.py |   291 +
 .../pyspark/pandas/tests/computation/test_melt.py  |   179 +
 .../pandas/tests/computation/test_missing_data.py  |   501 +
 .../pyspark/pandas/tests/computation/test_pivot.py |   202 +
 .../pandas/tests/computation/test_pivot_table.py   |    86 +
 .../tests/computation/test_pivot_table_adv.py      |    86 +
 .../computation/test_pivot_table_multi_idx.py      |    84 +
 .../computation/test_pivot_table_multi_idx_adv.py  |    86 +
 .../pyspark/pandas/tests/computation/test_stats.py |   324 +
 python/pyspark/pandas/tests/connect/__init__.py    |    16 +
 .../pandas/tests/connect/computation/__init__.py   |    16 +
 .../connect/computation/test_parity_any_all.py     |    34 +
 .../connect/computation/test_parity_apply_func.py  |    34 +
 .../connect/computation/test_parity_binary_ops.py  |    34 +
 .../connect/computation/test_parity_combine.py     |    34 +
 .../connect/computation/test_parity_compute.py     |    34 +
 .../tests/connect/computation/test_parity_corr.py  |    34 +
 .../connect/computation/test_parity_corrwith.py    |    34 +
 .../tests/connect/computation/test_parity_cov.py   |    34 +
 .../connect/computation/test_parity_cumulative.py  |    34 +
 .../connect/computation/test_parity_describe.py    |    30 +
 .../tests/connect/computation/test_parity_eval.py  |    34 +
 .../computation/test_parity_idxmax_idxmin.py       |    34 +
 .../tests/connect/computation/test_parity_melt.py  |    34 +
 .../computation/test_parity_missing_data.py        |    34 +
 .../tests/connect/computation/test_parity_pivot.py |    34 +
 .../connect/computation/test_parity_pivot_table.py |    34 +
 .../computation/test_parity_pivot_table_adv.py     |    34 +
 .../test_parity_pivot_table_multi_idx.py           |    34 +
 .../test_parity_pivot_table_multi_idx_adv.py       |    36 +
 .../tests/connect/computation/test_parity_stats.py |    30 +
 .../pandas/tests/connect/data_type_ops/__init__.py |    16 +
 .../connect/data_type_ops/test_parity_as_type.py   |    36 +
 .../connect/data_type_ops/test_parity_base.py      |    32 +
 .../data_type_ops/test_parity_binary_ops.py        |    36 +
 .../data_type_ops/test_parity_boolean_ops.py       |    48 +
 .../data_type_ops/test_parity_categorical_ops.py   |    36 +
 .../data_type_ops/test_parity_complex_ops.py       |    36 +
 .../connect/data_type_ops/test_parity_date_ops.py  |    36 +
 .../data_type_ops/test_parity_datetime_ops.py      |    43 +
 .../connect/data_type_ops/test_parity_null_ops.py  |    36 +
 .../data_type_ops/test_parity_num_arithmetic.py    |    36 +
 .../connect/data_type_ops/test_parity_num_mod.py   |    36 +
 .../data_type_ops/test_parity_num_mul_div.py       |    36 +
 .../connect/data_type_ops/test_parity_num_ops.py   |    58 +
 .../connect/data_type_ops/test_parity_num_pow.py   |    36 +
 .../data_type_ops/test_parity_num_reverse.py       |    36 +
 .../data_type_ops/test_parity_string_ops.py        |    48 +
 .../data_type_ops/test_parity_timedelta_ops.py     |    36 +
 .../connect/data_type_ops/test_parity_udt_ops.py   |    36 +
 .../tests/connect/diff_frames_ops/__init__.py      |    16 +
 .../connect/diff_frames_ops/test_parity_align.py   |    32 +
 .../diff_frames_ops/test_parity_arithmetic.py      |    34 +
 .../test_parity_arithmetic_chain.py                |    34 +
 .../test_parity_arithmetic_chain_ext.py            |    34 +
 .../test_parity_arithmetic_chain_ext_float.py      |    36 +
 .../diff_frames_ops/test_parity_arithmetic_ext.py  |    34 +
 .../test_parity_arithmetic_ext_float.py            |    34 +
 .../diff_frames_ops/test_parity_assign_frame.py    |    34 +
 .../diff_frames_ops/test_parity_assign_series.py   |    34 +
 .../connect/diff_frames_ops/test_parity_basic.py   |    34 +
 .../diff_frames_ops/test_parity_basic_slow.py      |    32 +
 .../connect/diff_frames_ops/test_parity_bitwise.py |    34 +
 .../diff_frames_ops/test_parity_combine_first.py   |    34 +
 .../diff_frames_ops/test_parity_compare_series.py  |    34 +
 .../diff_frames_ops/test_parity_concat_inner.py    |    34 +
 .../diff_frames_ops/test_parity_concat_outer.py    |    34 +
 .../diff_frames_ops/test_parity_corrwith.py        |    34 +
 .../connect/diff_frames_ops/test_parity_cov.py     |    35 +
 .../diff_frames_ops/test_parity_dot_frame.py       |    32 +
 .../diff_frames_ops/test_parity_dot_series.py      |    32 +
 .../connect/diff_frames_ops/test_parity_error.py   |    32 +
 .../connect/diff_frames_ops/test_parity_groupby.py |    34 +
 .../test_parity_groupby_aggregate.py               |    34 +
 .../diff_frames_ops/test_parity_groupby_apply.py   |    34 +
 .../test_parity_groupby_cumulative.py              |    34 +
 .../diff_frames_ops/test_parity_groupby_diff.py    |    34 +
 .../test_parity_groupby_diff_len.py                |    34 +
 .../test_parity_groupby_expanding.py               |    34 +
 .../test_parity_groupby_expanding_adv.py           |    34 +
 .../test_parity_groupby_expanding_count.py         |    36 +
 .../diff_frames_ops/test_parity_groupby_fillna.py  |    34 +
 .../diff_frames_ops/test_parity_groupby_filter.py  |    34 +
 .../diff_frames_ops/test_parity_groupby_rolling.py |    34 +
 .../test_parity_groupby_rolling_adv.py             |    34 +
 .../test_parity_groupby_rolling_count.py           |    34 +
 .../diff_frames_ops/test_parity_groupby_shift.py   |    34 +
 .../test_parity_groupby_split_apply_combine.py     |    34 +
 .../test_parity_groupby_transform.py               |    34 +
 .../connect/diff_frames_ops/test_parity_index.py   |    32 +
 .../connect/diff_frames_ops/test_parity_series.py  |    32 +
 .../diff_frames_ops/test_parity_setitem_frame.py   |    32 +
 .../diff_frames_ops/test_parity_setitem_series.py  |    32 +
 .../pyspark/pandas/tests/connect/frame/__init__.py |    16 +
 .../tests/connect/frame/test_parity_asfreq.py      |    34 +
 .../pandas/tests/connect/frame/test_parity_asof.py |    34 +
 .../tests/connect/frame/test_parity_attrs.py       |    34 +
 .../pandas/tests/connect/frame/test_parity_axis.py |    34 +
 .../tests/connect/frame/test_parity_constructor.py |    34 +
 .../tests/connect/frame/test_parity_conversion.py  |    34 +
 .../tests/connect/frame/test_parity_interpolate.py |    34 +
 .../connect/frame/test_parity_interpolate_error.py |    34 +
 .../tests/connect/frame/test_parity_reindexing.py  |    34 +
 .../tests/connect/frame/test_parity_reshaping.py   |    34 +
 .../tests/connect/frame/test_parity_spark.py       |    34 +
 .../pandas/tests/connect/frame/test_parity_take.py |    37 +
 .../tests/connect/frame/test_parity_take_adv.py    |    37 +
 .../tests/connect/frame/test_parity_time_series.py |    34 +
 .../tests/connect/frame/test_parity_truncate.py    |    34 +
 .../pandas/tests/connect/groupby/__init__.py       |    16 +
 .../tests/connect/groupby/test_parity_aggregate.py |    34 +
 .../connect/groupby/test_parity_apply_func.py      |    37 +
 .../tests/connect/groupby/test_parity_corr.py      |    34 +
 .../connect/groupby/test_parity_cumulative.py      |    34 +
 .../tests/connect/groupby/test_parity_describe.py  |    34 +
 .../tests/connect/groupby/test_parity_groupby.py   |    35 +
 .../tests/connect/groupby/test_parity_grouping.py  |    35 +
 .../tests/connect/groupby/test_parity_head_tail.py |    34 +
 .../tests/connect/groupby/test_parity_index.py     |    34 +
 .../tests/connect/groupby/test_parity_missing.py   |    35 +
 .../connect/groupby/test_parity_missing_data.py    |    34 +
 .../groupby/test_parity_nlargest_nsmallest.py      |    35 +
 .../tests/connect/groupby/test_parity_raises.py    |    35 +
 .../tests/connect/groupby/test_parity_rank.py      |    35 +
 .../tests/connect/groupby/test_parity_size.py      |    35 +
 .../connect/groupby/test_parity_split_apply.py     |    34 +
 .../groupby/test_parity_split_apply_count.py       |    34 +
 .../groupby/test_parity_split_apply_first.py       |    34 +
 .../groupby/test_parity_split_apply_last.py        |    34 +
 .../groupby/test_parity_split_apply_min_max.py     |    34 +
 .../groupby/test_parity_split_apply_skew.py        |    34 +
 .../connect/groupby/test_parity_split_apply_std.py |    34 +
 .../connect/groupby/test_parity_split_apply_var.py |    34 +
 .../tests/connect/groupby/test_parity_stat.py      |    34 +
 .../tests/connect/groupby/test_parity_stat_adv.py  |    34 +
 .../tests/connect/groupby/test_parity_stat_ddof.py |    34 +
 .../tests/connect/groupby/test_parity_stat_func.py |    34 +
 .../tests/connect/groupby/test_parity_stat_prod.py |    34 +
 .../connect/groupby/test_parity_value_counts.py    |    35 +
 .../pandas/tests/connect/indexes/__init__.py       |    16 +
 .../tests/connect/indexes/test_parity_align.py     |    34 +
 .../tests/connect/indexes/test_parity_append.py    |    34 +
 .../tests/connect/indexes/test_parity_asof.py      |    34 +
 .../tests/connect/indexes/test_parity_astype.py    |    34 +
 .../tests/connect/indexes/test_parity_basic.py     |    34 +
 .../tests/connect/indexes/test_parity_category.py  |    32 +
 .../connect/indexes/test_parity_conversion.py      |    34 +
 .../tests/connect/indexes/test_parity_datetime.py  |    34 +
 .../connect/indexes/test_parity_datetime_at.py     |    34 +
 .../indexes/test_parity_datetime_between.py        |    34 +
 .../connect/indexes/test_parity_datetime_ceil.py   |    34 +
 .../connect/indexes/test_parity_datetime_floor.py  |    34 +
 .../connect/indexes/test_parity_datetime_iso.py    |    34 +
 .../connect/indexes/test_parity_datetime_map.py    |    34 +
 .../indexes/test_parity_datetime_property.py       |    34 +
 .../connect/indexes/test_parity_datetime_round.py  |    34 +
 .../tests/connect/indexes/test_parity_default.py   |    38 +
 .../tests/connect/indexes/test_parity_delete.py    |    34 +
 .../tests/connect/indexes/test_parity_diff.py      |    34 +
 .../tests/connect/indexes/test_parity_drop.py      |    34 +
 .../tests/connect/indexes/test_parity_getattr.py   |    34 +
 .../tests/connect/indexes/test_parity_indexing.py  |    34 +
 .../connect/indexes/test_parity_indexing_adv.py    |    36 +
 .../connect/indexes/test_parity_indexing_basic.py  |    34 +
 .../connect/indexes/test_parity_indexing_iloc.py   |    34 +
 .../connect/indexes/test_parity_indexing_loc.py    |    34 +
 .../connect/indexes/test_parity_indexing_loc_2d.py |    34 +
 .../indexes/test_parity_indexing_loc_multi_idx.py  |    34 +
 .../tests/connect/indexes/test_parity_insert.py    |    34 +
 .../connect/indexes/test_parity_intersection.py    |    34 +
 .../tests/connect/indexes/test_parity_level.py     |    34 +
 .../tests/connect/indexes/test_parity_map.py       |    34 +
 .../tests/connect/indexes/test_parity_missing.py   |    34 +
 .../tests/connect/indexes/test_parity_monotonic.py |    34 +
 .../tests/connect/indexes/test_parity_name.py      |    34 +
 .../tests/connect/indexes/test_parity_reindex.py   |    34 +
 .../tests/connect/indexes/test_parity_rename.py    |    34 +
 .../tests/connect/indexes/test_parity_repeat.py    |    34 +
 .../connect/indexes/test_parity_reset_index.py     |    34 +
 .../tests/connect/indexes/test_parity_sort.py      |    34 +
 .../tests/connect/indexes/test_parity_stat.py      |    34 +
 .../connect/indexes/test_parity_symmetric_diff.py  |    34 +
 .../tests/connect/indexes/test_parity_take.py      |    34 +
 .../tests/connect/indexes/test_parity_timedelta.py |    32 +
 .../tests/connect/indexes/test_parity_union.py     |    34 +
 .../tests/connect/indexes/test_parity_unique.py    |    34 +
 python/pyspark/pandas/tests/connect/io/__init__.py |    16 +
 .../pandas/tests/connect/io/test_parity_csv.py     |    35 +
 .../connect/io/test_parity_dataframe_conversion.py |    35 +
 .../connect/io/test_parity_dataframe_spark_io.py   |    35 +
 .../pandas/tests/connect/io/test_parity_feather.py |    35 +
 .../pandas/tests/connect/io/test_parity_io.py      |    34 +
 .../connect/io/test_parity_series_conversion.py    |    34 +
 .../pandas/tests/connect/io/test_parity_stata.py   |    35 +
 .../pyspark/pandas/tests/connect/plot/__init__.py  |    16 +
 .../tests/connect/plot/test_parity_frame_plot.py   |    32 +
 .../plot/test_parity_frame_plot_matplotlib.py      |    32 +
 .../connect/plot/test_parity_frame_plot_plotly.py  |    32 +
 .../tests/connect/plot/test_parity_series_plot.py  |    30 +
 .../plot/test_parity_series_plot_matplotlib.py     |    32 +
 .../connect/plot/test_parity_series_plot_plotly.py |    32 +
 .../pandas/tests/connect/resample/__init__.py      |    16 +
 .../tests/connect/resample/test_parity_error.py    |    30 +
 .../tests/connect/resample/test_parity_frame.py    |    32 +
 .../tests/connect/resample/test_parity_missing.py  |    32 +
 .../tests/connect/resample/test_parity_on.py       |    30 +
 .../tests/connect/resample/test_parity_series.py   |    32 +
 .../tests/connect/resample/test_parity_timezone.py |    32 +
 .../pandas/tests/connect/reshape/__init__.py       |    16 +
 .../connect/reshape/test_parity_get_dummies.py     |    34 +
 .../reshape/test_parity_get_dummies_kwargs.py      |    34 +
 .../reshape/test_parity_get_dummies_multiindex.py  |    34 +
 .../reshape/test_parity_get_dummies_object.py      |    34 +
 .../reshape/test_parity_get_dummies_prefix.py      |    34 +
 .../connect/reshape/test_parity_merge_asof.py      |    34 +
 .../pandas/tests/connect/series/__init__.py        |    16 +
 .../tests/connect/series/test_parity_all_any.py    |    34 +
 .../tests/connect/series/test_parity_arg_ops.py    |    34 +
 .../tests/connect/series/test_parity_as_of.py      |    34 +
 .../tests/connect/series/test_parity_as_type.py    |    34 +
 .../tests/connect/series/test_parity_compute.py    |    34 +
 .../tests/connect/series/test_parity_conversion.py |    34 +
 .../tests/connect/series/test_parity_cumulative.py |    34 +
 .../tests/connect/series/test_parity_datetime.py   |    34 +
 .../tests/connect/series/test_parity_index.py      |    34 +
 .../connect/series/test_parity_interpolate.py      |    34 +
 .../connect/series/test_parity_missing_data.py     |    34 +
 .../tests/connect/series/test_parity_series.py     |    34 +
 .../tests/connect/series/test_parity_sort.py       |    34 +
 .../tests/connect/series/test_parity_stat.py       |    34 +
 .../connect/series/test_parity_string_ops_adv.py   |    34 +
 .../connect/series/test_parity_string_ops_basic.py |    34 +
 .../tests/connect/test_parity_arrow_interface.py   |    32 +
 .../tests/connect/test_parity_categorical.py       |    35 +
 .../pandas/tests/connect/test_parity_config.py     |    30 +
 .../pandas/tests/connect/test_parity_extension.py  |    34 +
 .../tests/connect/test_parity_frame_spark.py       |    47 +
 .../tests/connect/test_parity_generic_functions.py |    32 +
 .../tests/connect/test_parity_indexops_spark.py    |    32 +
 .../pandas/tests/connect/test_parity_internal.py   |    32 +
 .../pandas/tests/connect/test_parity_namespace.py  |    30 +
 .../tests/connect/test_parity_numpy_compat.py      |    34 +
 .../pandas/tests/connect/test_parity_repr.py       |    30 +
 .../pandas/tests/connect/test_parity_scalars.py    |    30 +
 .../tests/connect/test_parity_spark_functions.py   |    32 +
 .../pandas/tests/connect/test_parity_sql.py        |    30 +
 .../pandas/tests/connect/test_parity_typedef.py    |    30 +
 .../pandas/tests/connect/test_parity_utils.py      |    30 +
 .../pandas/tests/connect/window/__init__.py        |    16 +
 .../tests/connect/window/test_parity_ewm_error.py  |    35 +
 .../tests/connect/window/test_parity_ewm_mean.py   |    35 +
 .../tests/connect/window/test_parity_expanding.py  |    34 +
 .../connect/window/test_parity_expanding_adv.py    |    34 +
 .../connect/window/test_parity_expanding_error.py  |    34 +
 .../connect/window/test_parity_groupby_ewm_mean.py |    35 +
 .../window/test_parity_groupby_expanding.py        |    34 +
 .../window/test_parity_groupby_expanding_adv.py    |    34 +
 .../connect/window/test_parity_groupby_rolling.py  |    34 +
 .../window/test_parity_groupby_rolling_adv.py      |    34 +
 .../window/test_parity_groupby_rolling_count.py    |    34 +
 .../tests/connect/window/test_parity_missing.py    |    34 +
 .../tests/connect/window/test_parity_rolling.py    |    34 +
 .../connect/window/test_parity_rolling_adv.py      |    34 +
 .../connect/window/test_parity_rolling_count.py    |    34 +
 .../connect/window/test_parity_rolling_error.py    |    34 +
 .../pandas/tests/data_type_ops/test_as_type.py     |    92 +
 .../pandas/tests/data_type_ops/test_base.py        |    26 +-
 .../pandas/tests/data_type_ops/test_binary_ops.py  |    34 +-
 .../pandas/tests/data_type_ops/test_boolean_ops.py |   194 +-
 .../tests/data_type_ops/test_categorical_ops.py    |    64 +-
 .../pandas/tests/data_type_ops/test_complex_ops.py |    39 +-
 .../pandas/tests/data_type_ops/test_date_ops.py    |    57 +-
 .../tests/data_type_ops/test_datetime_ops.py       |    46 +-
 .../pandas/tests/data_type_ops/test_null_ops.py    |    32 +-
 .../tests/data_type_ops/test_num_arithmetic.py     |   115 +
 .../pandas/tests/data_type_ops/test_num_mod.py     |    80 +
 .../pandas/tests/data_type_ops/test_num_mul_div.py |   162 +
 .../pandas/tests/data_type_ops/test_num_ops.py     |   490 +-
 .../pandas/tests/data_type_ops/test_num_pow.py     |    65 +
 .../pandas/tests/data_type_ops/test_num_reverse.py |   142 +
 .../pandas/tests/data_type_ops/test_string_ops.py  |    73 +-
 .../tests/data_type_ops/test_timedelta_ops.py      |   217 +
 .../pandas/tests/data_type_ops/test_udt_ops.py     |    51 +-
 .../pandas/tests/data_type_ops/testing_utils.py    |    77 +-
 .../pandas/tests/diff_frames_ops/__init__.py       |    16 +
 .../pandas/tests/diff_frames_ops/test_align.py     |   111 +
 .../tests/diff_frames_ops/test_arithmetic.py       |   194 +
 .../tests/diff_frames_ops/test_arithmetic_chain.py |   187 +
 .../diff_frames_ops/test_arithmetic_chain_ext.py   |   113 +
 .../test_arithmetic_chain_ext_float.py             |   115 +
 .../tests/diff_frames_ops/test_arithmetic_ext.py   |    92 +
 .../diff_frames_ops/test_arithmetic_ext_float.py   |    92 +
 .../tests/diff_frames_ops/test_assign_frame.py     |   236 +
 .../tests/diff_frames_ops/test_assign_series.py    |   234 +
 .../pandas/tests/diff_frames_ops/test_basic.py     |   232 +
 .../tests/diff_frames_ops/test_basic_slow.py       |   191 +
 .../pandas/tests/diff_frames_ops/test_bitwise.py   |   103 +
 .../tests/diff_frames_ops/test_combine_first.py    |   103 +
 .../tests/diff_frames_ops/test_compare_series.py   |   148 +
 .../tests/diff_frames_ops/test_concat_inner.py     |   116 +
 .../tests/diff_frames_ops/test_concat_outer.py     |    74 +
 .../pandas/tests/diff_frames_ops/test_corrwith.py  |   128 +
 .../pandas/tests/diff_frames_ops/test_cov.py       |    84 +
 .../pandas/tests/diff_frames_ops/test_dot_frame.py |    95 +
 .../tests/diff_frames_ops/test_dot_series.py       |   136 +
 .../pandas/tests/diff_frames_ops/test_error.py     |   247 +
 .../pandas/tests/diff_frames_ops/test_groupby.py   |   120 +
 .../diff_frames_ops/test_groupby_aggregate.py      |   132 +
 .../tests/diff_frames_ops/test_groupby_apply.py    |    79 +
 .../diff_frames_ops/test_groupby_cumulative.py     |   175 +
 .../tests/diff_frames_ops/test_groupby_diff.py     |    75 +
 .../tests/diff_frames_ops/test_groupby_diff_len.py |    98 +
 .../diff_frames_ops/test_groupby_expanding.py      |    93 +
 .../diff_frames_ops/test_groupby_expanding_adv.py  |    55 +
 .../test_groupby_expanding_count.py                |    52 +
 .../tests/diff_frames_ops/test_groupby_fillna.py   |   111 +
 .../tests/diff_frames_ops/test_groupby_filter.py   |    75 +
 .../tests/diff_frames_ops/test_groupby_rolling.py  |    93 +
 .../diff_frames_ops/test_groupby_rolling_adv.py    |    53 +
 .../diff_frames_ops/test_groupby_rolling_count.py  |    79 +
 .../tests/diff_frames_ops/test_groupby_shift.py    |    78 +
 .../test_groupby_split_apply_combine.py            |   106 +
 .../diff_frames_ops/test_groupby_transform.py      |    75 +
 .../pandas/tests/diff_frames_ops/test_index.py     |   102 +
 .../pandas/tests/diff_frames_ops/test_series.py    |   129 +
 .../tests/diff_frames_ops/test_setitem_frame.py    |   126 +
 .../tests/diff_frames_ops/test_setitem_series.py   |   175 +
 python/pyspark/pandas/tests/frame/__init__.py      |    16 +
 python/pyspark/pandas/tests/frame/test_asfreq.py   |    80 +
 python/pyspark/pandas/tests/frame/test_asof.py     |    99 +
 python/pyspark/pandas/tests/frame/test_attrs.py    |   358 +
 python/pyspark/pandas/tests/frame/test_axis.py     |   132 +
 .../pyspark/pandas/tests/frame/test_constructor.py |   587 +
 .../pyspark/pandas/tests/frame/test_conversion.py  |    68 +
 .../pyspark/pandas/tests/frame/test_interpolate.py |    96 +
 .../pandas/tests/frame/test_interpolate_error.py   |    70 +
 .../pyspark/pandas/tests/frame/test_reindexing.py  |   971 +
 .../pyspark/pandas/tests/frame/test_reshaping.py   |   479 +
 python/pyspark/pandas/tests/frame/test_spark.py    |   317 +
 python/pyspark/pandas/tests/frame/test_take.py     |    86 +
 python/pyspark/pandas/tests/frame/test_take_adv.py |    97 +
 .../pyspark/pandas/tests/frame/test_time_series.py |   133 +
 python/pyspark/pandas/tests/frame/test_truncate.py |   126 +
 python/pyspark/pandas/tests/groupby/__init__.py    |    16 +
 .../pyspark/pandas/tests/groupby/test_aggregate.py |   283 +
 .../pandas/tests/groupby/test_apply_func.py        |   723 +
 python/pyspark/pandas/tests/groupby/test_corr.py   |    77 +
 .../pandas/tests/groupby/test_cumulative.py        |   401 +
 .../pyspark/pandas/tests/groupby/test_describe.py  |   197 +
 .../pyspark/pandas/tests/groupby/test_groupby.py   |   494 +
 .../pyspark/pandas/tests/groupby/test_grouping.py  |   167 +
 .../pyspark/pandas/tests/groupby/test_head_tail.py |   215 +
 python/pyspark/pandas/tests/groupby/test_index.py  |   293 +
 .../pyspark/pandas/tests/groupby/test_missing.py   |   140 +
 .../pandas/tests/groupby/test_missing_data.py      |   333 +
 .../tests/groupby/test_nlargest_nsmallest.py       |   115 +
 python/pyspark/pandas/tests/groupby/test_raises.py |    48 +
 python/pyspark/pandas/tests/groupby/test_rank.py   |    87 +
 python/pyspark/pandas/tests/groupby/test_size.py   |    66 +
 .../pandas/tests/groupby/test_split_apply.py       |   180 +
 .../pandas/tests/groupby/test_split_apply_count.py |    42 +
 .../pandas/tests/groupby/test_split_apply_first.py |    42 +
 .../pandas/tests/groupby/test_split_apply_last.py  |    42 +
 .../tests/groupby/test_split_apply_min_max.py      |    42 +
 .../pandas/tests/groupby/test_split_apply_skew.py  |    42 +
 .../pandas/tests/groupby/test_split_apply_std.py   |    42 +
 .../pandas/tests/groupby/test_split_apply_var.py   |    42 +
 python/pyspark/pandas/tests/groupby/test_stat.py   |   171 +
 .../pyspark/pandas/tests/groupby/test_stat_adv.py  |   188 +
 .../pyspark/pandas/tests/groupby/test_stat_ddof.py |    84 +
 .../pyspark/pandas/tests/groupby/test_stat_func.py |   130 +
 .../pyspark/pandas/tests/groupby/test_stat_prod.py |    87 +
 .../pandas/tests/groupby/test_value_counts.py      |   100 +
 python/pyspark/pandas/tests/indexes/test_align.py  |    83 +
 python/pyspark/pandas/tests/indexes/test_append.py |   122 +
 python/pyspark/pandas/tests/indexes/test_asof.py   |    82 +
 python/pyspark/pandas/tests/indexes/test_astype.py |    97 +
 python/pyspark/pandas/tests/indexes/test_base.py   |  2382 --
 python/pyspark/pandas/tests/indexes/test_basic.py  |   208 +
 .../pyspark/pandas/tests/indexes/test_category.py  |   107 +-
 .../pandas/tests/indexes/test_conversion.py        |   317 +
 .../pyspark/pandas/tests/indexes/test_datetime.py  |   239 +-
 .../pandas/tests/indexes/test_datetime_at.py       |    61 +
 .../pandas/tests/indexes/test_datetime_between.py  |    72 +
 .../pandas/tests/indexes/test_datetime_ceil.py     |    41 +
 .../pandas/tests/indexes/test_datetime_floor.py    |    41 +
 .../pandas/tests/indexes/test_datetime_iso.py      |    40 +
 .../pandas/tests/indexes/test_datetime_map.py      |    58 +
 .../pandas/tests/indexes/test_datetime_property.py |    63 +
 .../pandas/tests/indexes/test_datetime_round.py    |    41 +
 .../pyspark/pandas/tests/indexes/test_default.py   |   105 +
 python/pyspark/pandas/tests/indexes/test_delete.py |    80 +
 python/pyspark/pandas/tests/indexes/test_diff.py   |   119 +
 python/pyspark/pandas/tests/indexes/test_drop.py   |   167 +
 .../pyspark/pandas/tests/indexes/test_getattr.py   |    72 +
 .../pyspark/pandas/tests/indexes/test_indexing.py  |   432 +
 .../pandas/tests/indexes/test_indexing_adv.py      |   404 +
 .../pandas/tests/indexes/test_indexing_basic.py    |   164 +
 .../pandas/tests/indexes/test_indexing_iloc.py     |   298 +
 .../pandas/tests/indexes/test_indexing_loc.py      |   427 +
 .../pandas/tests/indexes/test_indexing_loc_2d.py   |   242 +
 .../tests/indexes/test_indexing_loc_multi_idx.py   |   142 +
 python/pyspark/pandas/tests/indexes/test_insert.py |   102 +
 .../pandas/tests/indexes/test_intersection.py      |   160 +
 python/pyspark/pandas/tests/indexes/test_level.py  |   167 +
 python/pyspark/pandas/tests/indexes/test_map.py    |   111 +
 .../pyspark/pandas/tests/indexes/test_missing.py   |   237 +
 .../pyspark/pandas/tests/indexes/test_monotonic.py |   177 +
 python/pyspark/pandas/tests/indexes/test_name.py   |   169 +
 .../pyspark/pandas/tests/indexes/test_reindex.py   |   291 +
 python/pyspark/pandas/tests/indexes/test_rename.py |   311 +
 python/pyspark/pandas/tests/indexes/test_repeat.py |    58 +
 .../pandas/tests/indexes/test_reset_index.py       |   158 +
 python/pyspark/pandas/tests/indexes/test_sort.py   |    92 +
 python/pyspark/pandas/tests/indexes/test_stat.py   |   174 +
 .../pandas/tests/indexes/test_symmetric_diff.py    |   110 +
 python/pyspark/pandas/tests/indexes/test_take.py   |    77 +
 .../pyspark/pandas/tests/indexes/test_timedelta.py |   119 +
 python/pyspark/pandas/tests/indexes/test_union.py  |   201 +
 python/pyspark/pandas/tests/indexes/test_unique.py |   157 +
 python/pyspark/pandas/tests/io/__init__.py         |    16 +
 python/pyspark/pandas/tests/io/test_csv.py         |   412 +
 .../pandas/tests/io/test_dataframe_conversion.py   |   274 +
 .../pandas/tests/io/test_dataframe_spark_io.py     |   459 +
 python/pyspark/pandas/tests/io/test_feather.py     |    61 +
 python/pyspark/pandas/tests/io/test_io.py          |   162 +
 .../pandas/tests/io/test_series_conversion.py      |    79 +
 python/pyspark/pandas/tests/io/test_stata.py       |    60 +
 .../pyspark/pandas/tests/plot/test_frame_plot.py   |    46 +-
 .../tests/plot/test_frame_plot_matplotlib.py       |    37 +-
 .../pandas/tests/plot/test_frame_plot_plotly.py    |    45 +-
 .../pyspark/pandas/tests/plot/test_series_plot.py  |    76 +-
 .../tests/plot/test_series_plot_matplotlib.py      |    35 +-
 .../pandas/tests/plot/test_series_plot_plotly.py   |    37 +-
 python/pyspark/pandas/tests/resample/__init__.py   |    16 +
 python/pyspark/pandas/tests/resample/test_error.py |    96 +
 python/pyspark/pandas/tests/resample/test_frame.py |   147 +
 .../pyspark/pandas/tests/resample/test_missing.py  |   134 +
 python/pyspark/pandas/tests/resample/test_on.py    |    59 +
 .../pyspark/pandas/tests/resample/test_series.py   |   146 +
 .../pyspark/pandas/tests/resample/test_timezone.py |    76 +
 python/pyspark/pandas/tests/reshape/__init__.py    |    16 +
 .../pandas/tests/reshape/test_get_dummies.py       |   119 +
 .../tests/reshape/test_get_dummies_kwargs.py       |    62 +
 .../tests/reshape/test_get_dummies_multiindex.py   |   101 +
 .../tests/reshape/test_get_dummies_object.py       |    77 +
 .../tests/reshape/test_get_dummies_prefix.py       |    86 +
 .../pandas/tests/reshape/test_merge_asof.py        |   213 +
 python/pyspark/pandas/tests/series/__init__.py     |    16 +
 python/pyspark/pandas/tests/series/test_all_any.py |    94 +
 python/pyspark/pandas/tests/series/test_arg_ops.py |   208 +
 python/pyspark/pandas/tests/series/test_as_of.py   |   126 +
 python/pyspark/pandas/tests/series/test_as_type.py |   182 +
 python/pyspark/pandas/tests/series/test_compute.py |   614 +
 .../pyspark/pandas/tests/series/test_conversion.py |    89 +
 .../pyspark/pandas/tests/series/test_cumulative.py |   131 +
 .../pyspark/pandas/tests/series/test_datetime.py   |   304 +
 python/pyspark/pandas/tests/series/test_index.py   |   497 +
 .../pandas/tests/series/test_interpolate.py        |   111 +
 .../pandas/tests/series/test_missing_data.py       |   266 +
 python/pyspark/pandas/tests/series/test_series.py  |   889 +
 python/pyspark/pandas/tests/series/test_sort.py    |   151 +
 python/pyspark/pandas/tests/series/test_stat.py    |   739 +
 .../pandas/tests/series/test_string_ops_adv.py     |   247 +
 .../pandas/tests/series/test_string_ops_basic.py   |   181 +
 .../pyspark/pandas/tests/test_arrow_interface.py   |    70 +
 python/pyspark/pandas/tests/test_categorical.py    |   216 +-
 python/pyspark/pandas/tests/test_config.py         |    17 +-
 python/pyspark/pandas/tests/test_csv.py            |   435 -
 python/pyspark/pandas/tests/test_dataframe.py      |  5755 ----
 .../pandas/tests/test_dataframe_conversion.py      |   276 -
 .../pandas/tests/test_dataframe_spark_io.py        |   458 -
 python/pyspark/pandas/tests/test_default_index.py  |    52 -
 python/pyspark/pandas/tests/test_expanding.py      |   345 -
 python/pyspark/pandas/tests/test_extension.py      |    22 +-
 python/pyspark/pandas/tests/test_frame_spark.py    |    19 +-
 .../pyspark/pandas/tests/test_generic_functions.py |   104 +
 python/pyspark/pandas/tests/test_groupby.py        |  2867 --
 python/pyspark/pandas/tests/test_indexing.py       |  1336 -
 python/pyspark/pandas/tests/test_indexops_spark.py |    27 +-
 python/pyspark/pandas/tests/test_internal.py       |    44 +-
 python/pyspark/pandas/tests/test_namespace.py      |   413 +-
 python/pyspark/pandas/tests/test_numpy_compat.py   |    78 +-
 .../pandas/tests/test_ops_on_diff_frames.py        |  1993 --
 .../tests/test_ops_on_diff_frames_groupby.py       |   623 -
 .../test_ops_on_diff_frames_groupby_expanding.py   |   144 -
 .../test_ops_on_diff_frames_groupby_rolling.py     |   107 -
 python/pyspark/pandas/tests/test_repr.py           |   119 +-
 python/pyspark/pandas/tests/test_reshape.py        |   297 -
 python/pyspark/pandas/tests/test_rolling.py        |   220 -
 python/pyspark/pandas/tests/test_scalars.py        |    53 +
 python/pyspark/pandas/tests/test_series.py         |  2918 --
 .../pyspark/pandas/tests/test_series_conversion.py |    77 -
 .../pyspark/pandas/tests/test_series_datetime.py   |   293 -
 python/pyspark/pandas/tests/test_series_string.py  |   341 -
 .../pyspark/pandas/tests/test_spark_functions.py   |    40 +-
 python/pyspark/pandas/tests/test_sql.py            |    85 +-
 python/pyspark/pandas/tests/test_stats.py          |   427 -
 python/pyspark/pandas/tests/test_typedef.py        |   195 +-
 python/pyspark/pandas/tests/test_utils.py          |   166 +-
 python/pyspark/pandas/tests/test_window.py         |   317 -
 python/pyspark/pandas/tests/window/__init__.py     |    16 +
 .../pyspark/pandas/tests/window/test_ewm_error.py  |    90 +
 .../pyspark/pandas/tests/window/test_ewm_mean.py   |   261 +
 .../pyspark/pandas/tests/window/test_expanding.py  |    92 +
 .../pandas/tests/window/test_expanding_adv.py      |    57 +
 .../pandas/tests/window/test_expanding_error.py    |    44 +
 .../pandas/tests/window/test_groupby_ewm_mean.py   |   226 +
 .../pandas/tests/window/test_groupby_expanding.py  |   128 +
 .../tests/window/test_groupby_expanding_adv.py     |    54 +
 .../pandas/tests/window/test_groupby_rolling.py    |   125 +
 .../tests/window/test_groupby_rolling_adv.py       |    56 +
 .../tests/window/test_groupby_rolling_count.py     |   106 +
 python/pyspark/pandas/tests/window/test_missing.py |   461 +
 python/pyspark/pandas/tests/window/test_rolling.py |    84 +
 .../pandas/tests/window/test_rolling_adv.py        |    52 +
 .../pandas/tests/window/test_rolling_count.py      |    65 +
 .../pandas/tests/window/test_rolling_error.py      |    47 +
 python/pyspark/pandas/typedef/__init__.py          |     2 +-
 python/pyspark/pandas/typedef/string_typehints.py  |    40 -
 python/pyspark/pandas/typedef/typehints.py         |   650 +-
 python/pyspark/pandas/usage_logging/__init__.py    |   195 +-
 .../pyspark/pandas/usage_logging/usage_logger.py   |     2 +-
 python/pyspark/pandas/utils.py                     |   413 +-
 python/pyspark/pandas/window.py                    |  1436 +-
 python/pyspark/pipelines/__init__.py               |    33 +
 .../pipelines/add_pipeline_analysis_context.py     |    48 +
 python/pyspark/pipelines/api.py                    |   527 +
 .../pyspark/pipelines/block_session_mutations.py   |   135 +
 python/pyspark/pipelines/cli.py                    |   448 +
 python/pyspark/pipelines/flow.py                   |    43 +
 python/pyspark/pipelines/graph_element_registry.py |    83 +
 python/pyspark/pipelines/init_cli.py               |    83 +
 python/pyspark/pipelines/logging_utils.py          |    42 +
 python/pyspark/pipelines/output.py                 |    87 +
 python/pyspark/pipelines/source_code_location.py   |    72 +
 .../spark_connect_graph_element_registry.py        |   152 +
 python/pyspark/pipelines/spark_connect_pipeline.py |    99 +
 python/pyspark/pipelines/tests/__init__.py         |    16 +
 .../tests/local_graph_element_registry.py          |    58 +
 .../tests/test_add_pipeline_analysis_context.py    |    96 +
 .../tests/test_block_session_mutations.py          |   256 +
 python/pyspark/pipelines/tests/test_cli.py         |   533 +
 python/pyspark/pipelines/tests/test_decorators.py  |    70 +
 .../pipelines/tests/test_graph_element_registry.py |   136 +
 python/pyspark/pipelines/tests/test_init_cli.py    |    93 +
 .../pyspark/pipelines/tests/test_spark_connect.py  |    97 +
 python/pyspark/pipelines/type_error_utils.py       |    53 +
 python/pyspark/profiler.py                         |   345 +-
 python/pyspark/profiler.pyi                        |    56 -
 python/pyspark/rdd.py                              |  2989 --
 python/pyspark/rdd.pyi                             |   483 -
 python/pyspark/rddsampler.py                       |    43 +-
 python/pyspark/resource/__init__.py                |    17 +-
 python/pyspark/resource/information.py             |    27 +-
 python/pyspark/resource/information.pyi            |    26 -
 python/pyspark/resource/profile.py                 |   244 +-
 python/pyspark/resource/profile.pyi                |    60 -
 python/pyspark/resource/requests.py                |   381 +-
 python/pyspark/resource/requests.pyi               |    83 -
 .../resource/tests/test_connect_resources.py       |    82 +
 python/pyspark/resource/tests/test_resources.py    |    37 +-
 python/pyspark/resultiterable.py                   |    20 +-
 python/pyspark/resultiterable.pyi                  |    30 -
 python/pyspark/serializers.py                      |   255 +-
 python/pyspark/shell.py                            |   124 +-
 python/pyspark/shuffle.py                          |   345 +-
 python/pyspark/sql/__init__.py                     |    38 +-
 python/pyspark/sql/__init__.pyi                    |    42 -
 python/pyspark/sql/_typing.pyi                     |    34 +-
 python/pyspark/sql/avro/__init__.py                |     2 +-
 python/pyspark/sql/avro/functions.py               |   116 +-
 python/pyspark/sql/avro/functions.pyi              |    27 -
 python/pyspark/sql/catalog.py                      |  1518 +-
 python/pyspark/sql/catalog.pyi                     |    67 -
 python/pyspark/sql/classic/__init__.py             |    18 +
 python/pyspark/sql/classic/column.py               |   688 +
 python/pyspark/sql/classic/dataframe.py            |  2145 ++
 python/pyspark/sql/classic/table_arg.py            |    51 +
 python/pyspark/sql/classic/window.py               |   149 +
 python/pyspark/sql/column.py                       |  1781 +-
 python/pyspark/sql/column.pyi                      |   118 -
 python/pyspark/sql/conf.py                         |   141 +-
 python/pyspark/sql/conf.pyi                        |    27 -
 python/pyspark/sql/connect/__init__.py             |    22 +
 python/pyspark/sql/connect/_typing.py              |    87 +
 python/pyspark/sql/connect/avro/__init__.py        |    18 +
 python/pyspark/sql/connect/avro/functions.py       |   136 +
 python/pyspark/sql/connect/catalog.py              |   440 +
 python/pyspark/sql/connect/client/__init__.py      |    19 +
 python/pyspark/sql/connect/client/artifact.py      |   505 +
 python/pyspark/sql/connect/client/core.py          |  2492 ++
 python/pyspark/sql/connect/client/reattach.py      |   343 +
 python/pyspark/sql/connect/client/retries.py       |   392 +
 python/pyspark/sql/connect/column.py               |   662 +
 python/pyspark/sql/connect/conf.py                 |   165 +
 python/pyspark/sql/connect/conversion.py           |    59 +
 python/pyspark/sql/connect/dataframe.py            |  2528 ++
 python/pyspark/sql/connect/datasource.py           |    42 +
 python/pyspark/sql/connect/expressions.py          |  1358 +
 python/pyspark/sql/connect/functions/__init__.py   |    24 +
 python/pyspark/sql/connect/functions/builtin.py    |  5577 ++++
 .../pyspark/sql/connect/functions/partitioning.py  |   108 +
 python/pyspark/sql/connect/group.py                |   637 +
 python/pyspark/sql/connect/logging.py              |    67 +
 python/pyspark/sql/connect/merge.py                |   253 +
 python/pyspark/sql/connect/observation.py          |   123 +
 python/pyspark/sql/connect/plan.py                 |  3062 ++
 python/pyspark/sql/connect/profiler.py             |    41 +
 python/pyspark/sql/connect/proto/__init__.py       |    28 +
 python/pyspark/sql/connect/proto/base_pb2.py       |   286 +
 python/pyspark/sql/connect/proto/base_pb2.pyi      |  4632 +++
 python/pyspark/sql/connect/proto/base_pb2_grpc.py  |   641 +
 python/pyspark/sql/connect/proto/catalog_pb2.py    |   142 +
 python/pyspark/sql/connect/proto/catalog_pb2.pyi   |  1508 +
 python/pyspark/sql/connect/proto/commands_pb2.py   |   200 +
 python/pyspark/sql/connect/proto/commands_pb2.pyi  |  2434 ++
 python/pyspark/sql/connect/proto/common_pb2.py     |    92 +
 python/pyspark/sql/connect/proto/common_pb2.pyi    |   731 +
 .../sql/connect/proto/example_plugins_pb2.py       |    62 +
 .../sql/connect/proto/example_plugins_pb2.pyi      |   113 +
 .../pyspark/sql/connect/proto/expressions_pb2.py   |   157 +
 .../pyspark/sql/connect/proto/expressions_pb2.pyi  |  2168 ++
 python/pyspark/sql/connect/proto/ml_common_pb2.py  |    67 +
 python/pyspark/sql/connect/proto/ml_common_pb2.pyi |   178 +
 python/pyspark/sql/connect/proto/ml_pb2.py         |    83 +
 python/pyspark/sql/connect/proto/ml_pb2.pyi        |   679 +
 python/pyspark/sql/connect/proto/pipelines_pb2.py  |   149 +
 python/pyspark/sql/connect/proto/pipelines_pb2.pyi |  1735 ++
 python/pyspark/sql/connect/proto/relations_pb2.py  |   258 +
 python/pyspark/sql/connect/proto/relations_pb2.pyi |  4659 +++
 python/pyspark/sql/connect/proto/types_pb2.py      |   110 +
 python/pyspark/sql/connect/proto/types_pb2.pyi     |  1127 +
 python/pyspark/sql/connect/protobuf/__init__.py    |    18 +
 python/pyspark/sql/connect/protobuf/functions.py   |   161 +
 python/pyspark/sql/connect/readwriter.py           |  1098 +
 python/pyspark/sql/connect/resource/__init__.py    |    16 +
 python/pyspark/sql/connect/resource/profile.py     |    69 +
 python/pyspark/sql/connect/session.py              |  1355 +
 python/pyspark/sql/connect/shell/__init__.py       |    26 +
 python/pyspark/sql/connect/shell/progress.py       |   213 +
 python/pyspark/sql/connect/sql_formatter.py        |    76 +
 python/pyspark/sql/connect/streaming/__init__.py   |    16 +
 python/pyspark/sql/connect/streaming/query.py      |   460 +
 python/pyspark/sql/connect/streaming/readwriter.py |   794 +
 .../sql/connect/streaming/worker/__init__.py       |    18 +
 .../streaming/worker/foreach_batch_worker.py       |    95 +
 .../connect/streaming/worker/listener_worker.py    |   109 +
 python/pyspark/sql/connect/table_arg.py            |   106 +
 python/pyspark/sql/connect/tvf.py                  |   168 +
 python/pyspark/sql/connect/types.py                |   431 +
 python/pyspark/sql/connect/udf.py                  |   346 +
 python/pyspark/sql/connect/udtf.py                 |   253 +
 python/pyspark/sql/connect/utils.py                |   123 +
 python/pyspark/sql/connect/window.py               |   185 +
 python/pyspark/sql/context.py                      |   398 +-
 python/pyspark/sql/context.pyi                     |   140 -
 python/pyspark/sql/conversion.py                   |  1782 ++
 python/pyspark/sql/dataframe.py                    |  7854 +++--
 python/pyspark/sql/dataframe.pyi                   |   332 -
 python/pyspark/sql/datasource.py                   |  1304 +
 python/pyspark/sql/datasource_internal.py          |   199 +
 python/pyspark/sql/functions.py                    |  5103 ---
 python/pyspark/sql/functions.pyi                   |   372 -
 python/pyspark/sql/functions/__init__.py           |   593 +
 python/pyspark/sql/functions/builtin.py            | 30883 +++++++++++++++++++
 python/pyspark/sql/functions/partitioning.py       |   253 +
 python/pyspark/sql/geo_utils.py                    |   103 +
 python/pyspark/sql/group.py                        |   465 +-
 python/pyspark/sql/group.pyi                       |    44 -
 python/pyspark/sql/interchange.py                  |    89 +
 python/pyspark/sql/internal.py                     |   176 +
 python/pyspark/sql/merge.py                        |   253 +
 python/pyspark/sql/metrics.py                      |   311 +
 python/pyspark/sql/observation.py                  |    89 +-
 python/pyspark/sql/observation.pyi                 |    27 -
 python/pyspark/sql/pandas/_typing/__init__.pyi     |   386 +-
 .../pyspark/sql/pandas/_typing/protocols/frame.pyi |   412 +-
 .../sql/pandas/_typing/protocols/series.pyi        |   237 +-
 python/pyspark/sql/pandas/conversion.py            |  1172 +-
 python/pyspark/sql/pandas/conversion.pyi           |    58 -
 python/pyspark/sql/pandas/functions.py             |   832 +-
 python/pyspark/sql/pandas/functions.pyi            |   145 +-
 python/pyspark/sql/pandas/group_ops.py             |  1037 +-
 python/pyspark/sql/pandas/group_ops.pyi            |    49 -
 python/pyspark/sql/pandas/map_ops.py               |   144 +-
 python/pyspark/sql/pandas/map_ops.pyi              |    30 -
 python/pyspark/sql/pandas/serializers.py           |  1633 +-
 python/pyspark/sql/pandas/typehints.py             |   633 +-
 python/pyspark/sql/pandas/types.py                 |  1521 +-
 python/pyspark/sql/pandas/utils.py                 |   116 +-
 python/pyspark/sql/plot/__init__.py                |    22 +
 python/pyspark/sql/plot/core.py                    |   788 +
 python/pyspark/sql/plot/plotly.py                  |   282 +
 python/pyspark/sql/profiler.py                     |   698 +
 python/pyspark/sql/protobuf/__init__.py            |    18 +
 python/pyspark/sql/protobuf/functions.py           |   335 +
 python/pyspark/sql/readwriter.py                   |  1992 +-
 python/pyspark/sql/readwriter.pyi                  |   257 -
 python/pyspark/sql/session.py                      |  2333 +-
 python/pyspark/sql/session.pyi                     |   131 -
 python/pyspark/sql/sql_formatter.py                |    86 +
 python/pyspark/sql/srs_registry.csv                | 10717 +++++++
 python/pyspark/sql/streaming.py                    |  1312 -
 python/pyspark/sql/streaming.pyi                   |   197 -
 python/pyspark/sql/streaming/__init__.py           |    27 +
 python/pyspark/sql/streaming/benchmark/__init__.py |    16 +
 .../benchmark/benchmark_tws_state_server.py        |   375 +
 .../pyspark/sql/streaming/benchmark/tws_utils.py   |    73 +
 python/pyspark/sql/streaming/benchmark/utils.py    |    26 +
 python/pyspark/sql/streaming/datasource.py         |   119 +
 python/pyspark/sql/streaming/list_state_client.py  |   209 +
 python/pyspark/sql/streaming/listener.py           |  1143 +
 python/pyspark/sql/streaming/map_state_client.py   |   338 +
 .../sql/streaming/proto/StateMessage_pb2.py        |   150 +
 .../sql/streaming/proto/StateMessage_pb2.pyi       |  1455 +
 python/pyspark/sql/streaming/proto/__init__.py     |    16 +
 .../streaming/python_streaming_source_runner.py    |   314 +
 python/pyspark/sql/streaming/query.py              |   810 +
 python/pyspark/sql/streaming/readwriter.py         |  1955 ++
 python/pyspark/sql/streaming/state.py              |   299 +
 python/pyspark/sql/streaming/stateful_processor.py |   475 +
 .../sql/streaming/stateful_processor_api_client.py |   628 +
 .../sql/streaming/stateful_processor_util.py       |   237 +
 .../transform_with_state_driver_worker.py          |    98 +
 python/pyspark/sql/streaming/tws_tester.py         |   689 +
 python/pyspark/sql/streaming/value_state_client.py |   109 +
 python/pyspark/sql/table_arg.py                    |   223 +
 python/pyspark/sql/tests/arrow/__init__.py         |    16 +
 python/pyspark/sql/tests/arrow/test_arrow.py       |  1971 ++
 .../sql/tests/arrow/test_arrow_cogrouped_map.py    |   438 +
 .../sql/tests/arrow/test_arrow_grouped_map.py      |   497 +
 python/pyspark/sql/tests/arrow/test_arrow_map.py   |   364 +
 .../sql/tests/arrow/test_arrow_python_udf.py       |   549 +
 python/pyspark/sql/tests/arrow/test_arrow_udf.py   |   320 +
 .../sql/tests/arrow/test_arrow_udf_grouped_agg.py  |  1293 +
 .../sql/tests/arrow/test_arrow_udf_scalar.py       |  1297 +
 .../sql/tests/arrow/test_arrow_udf_typehints.py    |   466 +
 .../sql/tests/arrow/test_arrow_udf_window.py       |   858 +
 python/pyspark/sql/tests/arrow/test_arrow_udtf.py  |  1729 ++
 python/pyspark/sql/tests/coercion/__init__.py      |    16 +
 .../golden_pandas_udf_input_type_coercion_base.csv |    40 +
 .../golden_pandas_udf_input_type_coercion_base.md  |    41 +
 ...golden_pandas_udf_return_type_coercion_base.csv |    16 +
 .../golden_pandas_udf_return_type_coercion_base.md |    17 +
 ...lden_python_udf_input_type_coercion_vanilla.csv |    40 +
 ...olden_python_udf_input_type_coercion_vanilla.md |    41 +
 ...n_python_udf_input_type_coercion_with_arrow.csv |    40 +
 ...en_python_udf_input_type_coercion_with_arrow.md |    41 +
 ...f_input_type_coercion_with_arrow_and_pandas.csv |    40 +
 ...df_input_type_coercion_with_arrow_and_pandas.md |    41 +
 ...den_python_udf_return_type_coercion_vanilla.csv |    16 +
 ...lden_python_udf_return_type_coercion_vanilla.md |    17 +
 ..._python_udf_return_type_coercion_with_arrow.csv |    16 +
 ...n_python_udf_return_type_coercion_with_arrow.md |    17 +
 ..._return_type_coercion_with_arrow_and_pandas.csv |    16 +
 ...f_return_type_coercion_with_arrow_and_pandas.md |    17 +
 .../tests/coercion/test_pandas_udf_input_type.py   |   320 +
 .../tests/coercion/test_pandas_udf_return_type.py  |   235 +
 .../tests/coercion/test_python_udf_input_type.py   |   351 +
 .../tests/coercion/test_python_udf_return_type.py  |   241 +
 python/pyspark/sql/tests/connect/__init__.py       |    16 +
 python/pyspark/sql/tests/connect/arrow/__init__.py |    16 +
 .../sql/tests/connect/arrow/test_parity_arrow.py   |   177 +
 .../arrow/test_parity_arrow_cogrouped_map.py       |    30 +
 .../connect/arrow/test_parity_arrow_grouped_map.py |    30 +
 .../tests/connect/arrow/test_parity_arrow_map.py   |    30 +
 .../connect/arrow/test_parity_arrow_python_udf.py  |   115 +
 .../tests/connect/arrow/test_parity_arrow_udf.py   |    35 +
 .../arrow/test_parity_arrow_udf_grouped_agg.py     |    50 +
 .../connect/arrow/test_parity_arrow_udf_scalar.py  |    50 +
 .../connect/arrow/test_parity_arrow_udf_window.py  |    50 +
 .../tests/connect/arrow/test_parity_arrow_udtf.py  |    30 +
 .../pyspark/sql/tests/connect/client/__init__.py   |    16 +
 .../sql/tests/connect/client/test_artifact.py      |   461 +
 .../connect/client/test_artifact_localcluster.py   |    43 +
 .../sql/tests/connect/client/test_client.py        |   971 +
 .../connect/client/test_client_call_stack_trace.py |   312 +
 .../tests/connect/client/test_client_retries.py    |   242 +
 .../sql/tests/connect/client/test_reattach.py      |    94 +
 .../pyspark/sql/tests/connect/pandas/__init__.py   |    16 +
 .../sql/tests/connect/pandas/streaming/__init__.py |    16 +
 .../test_parity_pandas_grouped_map_with_state.py   |    33 +
 .../test_parity_pandas_transform_with_state.py     |    59 +
 ...y_pandas_transform_with_state_state_variable.py |    54 +
 .../streaming/test_parity_transform_with_state.py  |    59 +
 ...t_parity_transform_with_state_state_variable.py |    54 +
 .../pandas/test_parity_pandas_cogrouped_map.py     |    32 +
 .../pandas/test_parity_pandas_grouped_map.py       |    29 +
 .../tests/connect/pandas/test_parity_pandas_map.py |    32 +
 .../tests/connect/pandas/test_parity_pandas_udf.py |    29 +
 .../pandas/test_parity_pandas_udf_grouped_agg.py   |    32 +
 .../pandas/test_parity_pandas_udf_scalar.py        |    28 +
 .../pandas/test_parity_pandas_udf_window.py        |    32 +
 python/pyspark/sql/tests/connect/shell/__init__.py |    16 +
 .../sql/tests/connect/shell/test_progress.py       |   145 +
 .../sql/tests/connect/streaming/__init__.py        |    16 +
 .../tests/connect/streaming/test_parity_foreach.py |    29 +
 .../connect/streaming/test_parity_foreach_batch.py |   151 +
 .../connect/streaming/test_parity_listener.py      |   322 +
 .../connect/streaming/test_parity_streaming.py     |    30 +
 .../sql/tests/connect/test_connect_basic.py        |  1668 +
 .../sql/tests/connect/test_connect_channel.py      |   156 +
 .../tests/connect/test_connect_clone_session.py    |   146 +
 .../sql/tests/connect/test_connect_collection.py   |   375 +
 .../sql/tests/connect/test_connect_column.py       |  1109 +
 .../sql/tests/connect/test_connect_creation.py     |   738 +
 .../connect/test_connect_dataframe_property.py     |   464 +
 .../sql/tests/connect/test_connect_error.py        |   272 +
 .../sql/tests/connect/test_connect_function.py     |  2636 ++
 .../pyspark/sql/tests/connect/test_connect_plan.py |  1225 +
 .../sql/tests/connect/test_connect_readwriter.py   |   652 +
 .../sql/tests/connect/test_connect_retry.py        |   171 +
 .../sql/tests/connect/test_connect_session.py      |   416 +
 .../pyspark/sql/tests/connect/test_connect_stat.py |   816 +
 python/pyspark/sql/tests/connect/test_df_debug.py  |    73 +
 .../sql/tests/connect/test_parity_catalog.py       |    29 +
 .../sql/tests/connect/test_parity_collection.py    |    52 +
 .../sql/tests/connect/test_parity_column.py        |    75 +
 .../pyspark/sql/tests/connect/test_parity_conf.py  |    29 +
 .../sql/tests/connect/test_parity_creation.py      |    32 +
 .../sql/tests/connect/test_parity_dataframe.py     |    45 +
 .../connect/test_parity_dataframe_query_context.py |    30 +
 .../sql/tests/connect/test_parity_datasources.py   |    41 +
 .../sql/tests/connect/test_parity_errors.py        |    29 +
 .../sql/tests/connect/test_parity_frame_plot.py    |    29 +
 .../tests/connect/test_parity_frame_plot_plotly.py |    29 +
 .../sql/tests/connect/test_parity_functions.py     |    45 +
 .../sql/tests/connect/test_parity_geographytype.py |    30 +
 .../sql/tests/connect/test_parity_geometrytype.py  |    30 +
 .../pyspark/sql/tests/connect/test_parity_group.py |    29 +
 .../tests/connect/test_parity_job_cancellation.py  |    39 +
 .../tests/connect/test_parity_memory_profiler.py   |    67 +
 .../sql/tests/connect/test_parity_observation.py   |    33 +
 .../tests/connect/test_parity_python_datasource.py |    27 +
 .../test_parity_python_streaming_datasource.py     |    32 +
 .../sql/tests/connect/test_parity_readwriter.py    |    41 +
 .../sql/tests/connect/test_parity_repartition.py   |    32 +
 .../sql/tests/connect/test_parity_resources.py     |    32 +
 .../pyspark/sql/tests/connect/test_parity_serde.py |    37 +
 .../pyspark/sql/tests/connect/test_parity_sql.py   |    30 +
 .../pyspark/sql/tests/connect/test_parity_stat.py  |    32 +
 .../sql/tests/connect/test_parity_subquery.py      |    51 +
 .../pyspark/sql/tests/connect/test_parity_tvf.py   |    29 +
 .../pyspark/sql/tests/connect/test_parity_types.py |   113 +
 .../pyspark/sql/tests/connect/test_parity_udf.py   |    75 +
 .../tests/connect/test_parity_udf_combinations.py  |    33 +
 .../sql/tests/connect/test_parity_udf_profiler.py  |    74 +
 .../pyspark/sql/tests/connect/test_parity_udtf.py  |   161 +
 .../sql/tests/connect/test_parity_unified_udf.py   |    30 +
 .../pyspark/sql/tests/connect/test_parity_utils.py |    29 +
 python/pyspark/sql/tests/pandas/__init__.py        |    16 +
 .../sql/tests/pandas/bench_arrow_columnar_udf.py   |   168 +
 python/pyspark/sql/tests/pandas/helper/__init__.py |    16 +
 .../helper/helper_pandas_transform_with_state.py   |  2592 ++
 .../pyspark/sql/tests/pandas/streaming/__init__.py |    16 +
 .../streaming/test_pandas_transform_with_state.py  |  1658 +
 ...st_pandas_transform_with_state_checkpoint_v2.py |    42 +
 ...t_pandas_transform_with_state_state_variable.py |  1012 +
 ...form_with_state_state_variable_checkpoint_v2.py |    44 +
 .../pandas/streaming/test_transform_with_state.py  |    62 +
 .../test_transform_with_state_checkpoint_v2.py     |    42 +
 .../test_transform_with_state_state_variable.py    |    65 +
 ...form_with_state_state_variable_checkpoint_v2.py |    44 +
 .../sql/tests/pandas/streaming/test_tws_tester.py  |  1154 +
 python/pyspark/sql/tests/pandas/test_converter.py  |   587 +
 .../sql/tests/pandas/test_pandas_cogrouped_map.py  |   773 +
 .../sql/tests/pandas/test_pandas_grouped_map.py    |  1558 +
 .../pandas/test_pandas_grouped_map_with_state.py   |   350 +
 python/pyspark/sql/tests/pandas/test_pandas_map.py |   570 +
 .../sql/tests/pandas/test_pandas_sqlmetrics.py     |    60 +
 python/pyspark/sql/tests/pandas/test_pandas_udf.py |   489 +
 .../tests/pandas/test_pandas_udf_grouped_agg.py    |  1157 +
 .../sql/tests/pandas/test_pandas_udf_scalar.py     |  2093 ++
 .../sql/tests/pandas/test_pandas_udf_typehints.py  |   539 +
 ...pandas_udf_typehints_with_future_annotations.py |   372 +
 .../sql/tests/pandas/test_pandas_udf_window.py     |   715 +
 python/pyspark/sql/tests/plot/__init__.py          |    16 +
 python/pyspark/sql/tests/plot/test_frame_plot.py   |    93 +
 .../sql/tests/plot/test_frame_plot_plotly.py       |   522 +
 .../pyspark/sql/tests/streaming/KAFKA_TESTING.md   |   426 +
 python/pyspark/sql/tests/streaming/__init__.py     |    16 +
 python/pyspark/sql/tests/streaming/kafka_utils.py  |   379 +
 .../pyspark/sql/tests/streaming/test_streaming.py  |   684 +
 .../sql/tests/streaming/test_streaming_foreach.py  |   287 +
 .../streaming/test_streaming_foreach_batch.py      |   234 +
 .../tests/streaming/test_streaming_kafka_rtm.py    |   181 +
 .../sql/tests/streaming/test_streaming_listener.py |   676 +
 .../test_streaming_offline_state_repartition.py    |   560 +
 python/pyspark/sql/tests/test_arrow.py             |   603 -
 python/pyspark/sql/tests/test_artifact.py          |    87 +
 python/pyspark/sql/tests/test_catalog.py           |   563 +-
 python/pyspark/sql/tests/test_collection.py        |   447 +
 python/pyspark/sql/tests/test_column.py            |   557 +-
 python/pyspark/sql/tests/test_conf.py              |    72 +-
 .../sql/tests/test_connect_compatibility.py        |   527 +
 python/pyspark/sql/tests/test_context.py           |   106 +-
 python/pyspark/sql/tests/test_conversion.py        |   794 +
 python/pyspark/sql/tests/test_creation.py          |   334 +
 python/pyspark/sql/tests/test_dataframe.py         |  1667 +-
 .../sql/tests/test_dataframe_query_context.py      |   518 +
 python/pyspark/sql/tests/test_datasources.py       |   423 +-
 python/pyspark/sql/tests/test_errors.py            |    62 +
 python/pyspark/sql/tests/test_functions.py         |  3996 ++-
 python/pyspark/sql/tests/test_geographytype.py     |   130 +
 python/pyspark/sql/tests/test_geometrytype.py      |   121 +
 python/pyspark/sql/tests/test_group.py             |   188 +-
 python/pyspark/sql/tests/test_job_cancellation.py  |   209 +
 python/pyspark/sql/tests/test_listener.py          |   110 +
 python/pyspark/sql/tests/test_observation.py       |   391 +
 .../pyspark/sql/tests/test_pandas_cogrouped_map.py |   266 -
 .../pyspark/sql/tests/test_pandas_grouped_map.py   |   621 -
 python/pyspark/sql/tests/test_pandas_map.py        |   132 -
 python/pyspark/sql/tests/test_pandas_udf.py        |   251 -
 .../sql/tests/test_pandas_udf_grouped_agg.py       |   523 -
 python/pyspark/sql/tests/test_pandas_udf_scalar.py |  1164 -
 .../pyspark/sql/tests/test_pandas_udf_typehints.py |   253 -
 python/pyspark/sql/tests/test_pandas_udf_window.py |   366 -
 python/pyspark/sql/tests/test_python_datasource.py |  1377 +
 .../sql/tests/test_python_streaming_datasource.py  |   767 +
 python/pyspark/sql/tests/test_readwriter.py        |   413 +-
 python/pyspark/sql/tests/test_repartition.py       |   223 +
 python/pyspark/sql/tests/test_resources.py         |   100 +
 python/pyspark/sql/tests/test_serde.py             |    65 +-
 python/pyspark/sql/tests/test_session.py           |   595 +-
 python/pyspark/sql/tests/test_sql.py               |   184 +
 python/pyspark/sql/tests/test_stat.py              |   688 +
 python/pyspark/sql/tests/test_streaming.py         |   606 -
 python/pyspark/sql/tests/test_subquery.py          |  1164 +
 python/pyspark/sql/tests/test_tvf.py               |   621 +
 python/pyspark/sql/tests/test_types.py             |  2903 +-
 python/pyspark/sql/tests/test_udf.py               |  1566 +-
 python/pyspark/sql/tests/test_udf_combinations.py  |   202 +
 python/pyspark/sql/tests/test_udf_profiler.py      |   761 +
 python/pyspark/sql/tests/test_udtf.py              |  3989 +++
 python/pyspark/sql/tests/test_unified_udf.py       |   461 +
 python/pyspark/sql/tests/test_utils.py             |  1865 +-
 python/pyspark/sql/tests/typing/test_column.yml    |    37 +
 python/pyspark/sql/tests/typing/test_dataframe.yml |   134 +
 python/pyspark/sql/tests/typing/test_functions.yml |   101 +
 .../pyspark/sql/tests/typing/test_readwriter.yml   |    45 +
 python/pyspark/sql/tests/typing/test_session.yml   |   100 +
 python/pyspark/sql/tests/typing/test_udf.yml       |   170 +
 python/pyspark/sql/tvf.py                          |   751 +
 python/pyspark/sql/tvf_argument.py                 |    27 +
 python/pyspark/sql/types.py                        |  3104 +-
 python/pyspark/sql/types.pyi                       |   205 -
 python/pyspark/sql/udf.py                          |   674 +-
 python/pyspark/sql/udf.pyi                         |    58 -
 python/pyspark/sql/udtf.py                         |   574 +
 python/pyspark/sql/utils.py                        |   571 +-
 python/pyspark/sql/variant_utils.py                |   821 +
 python/pyspark/sql/window.py                       |   283 +-
 python/pyspark/sql/window.pyi                      |    41 -
 python/pyspark/sql/worker/__init__.py              |    16 +
 python/pyspark/sql/worker/analyze_udtf.py          |   234 +
 .../pyspark/sql/worker/commit_data_source_write.py |    80 +
 python/pyspark/sql/worker/create_data_source.py    |   148 +
 .../sql/worker/data_source_pushdown_filters.py     |   229 +
 python/pyspark/sql/worker/lookup_data_sources.py   |    62 +
 python/pyspark/sql/worker/plan_data_source_read.py |   394 +
 .../sql/worker/python_streaming_sink_runner.py     |   116 +
 python/pyspark/sql/worker/utils.py                 |   111 +
 .../pyspark/sql/worker/write_into_data_source.py   |   240 +
 python/pyspark/statcounter.py                      |    92 +-
 python/pyspark/statcounter.pyi                     |    44 -
 python/pyspark/status.py                           |    96 -
 python/pyspark/status.pyi                          |    42 -
 python/pyspark/storagelevel.py                     |    46 +-
 python/pyspark/storagelevel.pyi                    |    43 -
 python/pyspark/streaming/__init__.py               |     2 +-
 python/pyspark/streaming/context.py                |   146 +-
 python/pyspark/streaming/context.pyi               |    75 -
 python/pyspark/streaming/dstream.py                |   454 +-
 python/pyspark/streaming/dstream.pyi               |   216 -
 python/pyspark/streaming/kinesis.py                |   122 +-
 python/pyspark/streaming/kinesis.pyi               |    49 -
 python/pyspark/streaming/listener.py               |    25 +-
 python/pyspark/streaming/listener.pyi              |    35 -
 python/pyspark/streaming/tests/test_context.py     |    23 +-
 python/pyspark/streaming/tests/test_dstream.py     |   262 +-
 python/pyspark/streaming/tests/test_kinesis.py     |    70 +-
 python/pyspark/streaming/tests/test_listener.py    |    17 +-
 python/pyspark/streaming/util.py                   |    30 +-
 python/pyspark/taskcontext.py                      |   410 +-
 python/pyspark/taskcontext.pyi                     |    46 -
 python/pyspark/testing/__init__.py                 |     5 +
 python/pyspark/testing/connectutils.py             |   327 +
 python/pyspark/testing/goldenutils.py              |   383 +
 python/pyspark/testing/mllibutils.py               |     2 +-
 python/pyspark/testing/mlutils.py                  |   103 +-
 python/pyspark/testing/objects.py                  |   121 +
 python/pyspark/testing/pandasutils.py              |   650 +-
 python/pyspark/testing/sqlutils.py                 |   339 +-
 python/pyspark/testing/streamingutils.py           |    32 +-
 python/pyspark/testing/{ => tests}/__init__.py     |     0
 python/pyspark/testing/tests/test_fail.py          |    36 +
 .../testing/tests/test_fail_in_set_up_class.py     |    41 +
 python/pyspark/testing/tests/test_no_tests.py      |    21 +
 python/pyspark/testing/tests/test_pass_all.py      |    28 +
 python/pyspark/testing/tests/test_skip_all.py      |    29 +
 python/pyspark/testing/tests/test_skip_class.py    |    29 +
 .../testing/tests/test_skip_set_up_class.py        |    33 +
 python/pyspark/testing/unittestutils.py            |    55 +
 python/pyspark/testing/utils.py                    |  1210 +-
 python/pyspark/tests/test_appsubmit.py             |   182 +-
 python/pyspark/tests/test_broadcast.py             |    57 +-
 python/pyspark/tests/test_conf.py                  |    17 +-
 python/pyspark/tests/test_context.py               |   111 +-
 python/pyspark/tests/test_daemon.py                |    24 +-
 python/pyspark/tests/test_import_spark.py          |    69 +
 python/pyspark/tests/test_install_spark.py         |   122 +-
 python/pyspark/tests/test_join.py                  |    11 +-
 python/pyspark/tests/test_memory_profiler.py       |   621 +
 python/pyspark/tests/test_pin_thread.py            |    58 +-
 python/pyspark/tests/test_profiler.py              |   106 +-
 python/pyspark/tests/test_rdd.py                   |   242 +-
 python/pyspark/tests/test_rddbarrier.py            |    13 +-
 python/pyspark/tests/test_rddsampler.py            |    59 +
 python/pyspark/tests/test_readwrite.py             |   302 +-
 python/pyspark/tests/test_serializers.py           |   119 +-
 python/pyspark/tests/test_shuffle.py               |   188 +-
 python/pyspark/tests/test_stage_sched.py           |   147 +
 python/pyspark/tests/test_statcounter.py           |   124 +
 python/pyspark/tests/test_taskcontext.py           |   100 +-
 python/pyspark/tests/test_util.py                  |   153 +-
 python/pyspark/tests/test_worker.py                |    82 +-
 python/pyspark/tests/test_zero_copy_byte_stream.py |   272 +
 python/pyspark/tests/typing/test_context.yml       |    21 +
 python/pyspark/tests/typing/test_core.yml          |    20 +
 python/pyspark/tests/typing/test_rdd.yml           |   129 +
 .../pyspark/tests/typing/test_resultiterable.yml   |    22 +
 .../{testing => tests/upstream}/__init__.py        |     0
 python/pyspark/tests/upstream/pyarrow/__init__.py  |    16 +
 .../golden_pyarrow_arrow_to_pandas_default.csv     |   122 +
 .../golden_pyarrow_arrow_to_pandas_default.md      |   123 +
 .../pyarrow/golden_pyarrow_nested_cast_safe.csv    |    15 +
 .../pyarrow/golden_pyarrow_nested_cast_safe.md     |    16 +
 .../pyarrow/golden_pyarrow_nested_cast_unsafe.csv  |    15 +
 .../pyarrow/golden_pyarrow_nested_cast_unsafe.md   |    16 +
 .../pyarrow/golden_pyarrow_scalar_cast_safe.csv    |    84 +
 .../pyarrow/golden_pyarrow_scalar_cast_safe.md     |    85 +
 .../pyarrow/golden_pyarrow_scalar_cast_unsafe.csv  |    84 +
 .../pyarrow/golden_pyarrow_scalar_cast_unsafe.md   |    85 +
 .../upstream/pyarrow/test_pyarrow_array_cast.py    |   845 +
 .../pyarrow/test_pyarrow_array_type_inference.py   |   600 +
 .../test_pyarrow_arrow_to_pandas_default.py        |   414 +
 .../pyarrow/test_pyarrow_ignore_timezone.py        |   186 +
 .../pyarrow/test_pyarrow_scalar_type_coercion.py   |   391 +
 .../pyarrow/test_pyarrow_scalar_type_inference.py  |   492 +
 .../upstream/pyarrow/test_pyarrow_type_coercion.py |  1158 +
 python/pyspark/threaddump.py                       |    62 +
 python/pyspark/traceback_utils.py                  |    10 +-
 python/pyspark/util.py                             |   891 +-
 python/pyspark/util.pyi                            |    29 -
 python/pyspark/version.py                          |     2 +-
 python/pyspark/version.pyi                         |    19 -
 python/pyspark/worker.py                           |  3787 ++-
 python/pyspark/worker_util.py                      |   254 +
 python/run-tests                                   |     4 +-
 python/run-tests-with-coverage                     |     5 +-
 python/run-tests.py                                |   453 +-
 python/run-with-viztracer                          |    69 +
 python/run-with-vscode-breakpoints                 |    79 +
 python/setup.cfg                                   |    22 -
 python/setup.py                                    |   309 -
 python/test_coverage/conf/spark-defaults.conf      |    21 -
 python/test_coverage/coverage_daemon.py            |    48 -
 python/test_coverage/sitecustomize.py              |    55 +-
 python/test_support/log4j2.properties              |    31 +
 python/test_support/pytorch_training_test_file.py  |   115 +
 python/test_support/sql/ages.csv                   |     4 -
 .../sql/parquet_partitioned/_common_metadata       |   Bin 210 -> 0 bytes
 .../test_support/sql/parquet_partitioned/_metadata |   Bin 743 -> 0 bytes
 .../month=9/day=1/.part-r-00008.gz.parquet.crc     |   Bin 12 -> 0 bytes
 .../month=9/day=1/part-r-00008.gz.parquet          |   Bin 322 -> 0 bytes
 .../month=10/day=25/.part-r-00002.gz.parquet.crc   |   Bin 12 -> 0 bytes
 .../month=10/day=25/.part-r-00004.gz.parquet.crc   |   Bin 12 -> 0 bytes
 .../month=10/day=25/part-r-00002.gz.parquet        |   Bin 343 -> 0 bytes
 .../month=10/day=25/part-r-00004.gz.parquet        |   Bin 343 -> 0 bytes
 .../month=10/day=26/.part-r-00005.gz.parquet.crc   |   Bin 12 -> 0 bytes
 .../month=10/day=26/part-r-00005.gz.parquet        |   Bin 333 -> 0 bytes
 .../month=9/day=1/.part-r-00007.gz.parquet.crc     |   Bin 12 -> 0 bytes
 .../month=9/day=1/part-r-00007.gz.parquet          |   Bin 343 -> 0 bytes
 .../sql/streaming/time/text-with-time-test.txt     |    10 +
 python/test_support/userlibrary.py                 |     3 +-
 repl/pom.xml                                       |    55 +-
 .../scala-2.12/org/apache/spark/repl/Main.scala    |   130 -
 .../org/apache/spark/repl/SparkILoop.scala         |   273 -
 .../scala-2.13/org/apache/spark/repl/Main.scala    |   138 -
 .../org/apache/spark/repl/SparkILoop.scala         |   149 -
 .../apache/spark/repl/ExecutorClassLoader.scala    |   273 -
 .../main/scala/org/apache/spark/repl/Main.scala    |   142 +
 .../scala/org/apache/spark/repl/SparkILoop.scala   |   158 +
 repl/src/test/resources/log4j.properties           |    27 -
 repl/src/test/resources/log4j2.properties          |    31 +
 .../org/apache/spark/repl/Repl2Suite.scala         |    53 -
 .../apache/spark/repl/SingletonRepl2Suite.scala    |   171 -
 .../org/apache/spark/repl/Repl2Suite.scala         |    53 -
 .../apache/spark/repl/SingletonRepl2Suite.scala    |   171 -
 .../spark/repl/ExecutorClassLoaderSuite.scala      |   358 -
 .../scala/org/apache/spark/repl/IntSumUdf.scala    |    24 +
 .../scala/org/apache/spark/repl/ReplSuite.scala    |   217 +-
 .../org/apache/spark/repl/SingletonReplSuite.scala |    81 +-
 .../org/apache/spark/repl/SparkShellSuite.scala    |   165 +
 resource-managers/kubernetes/core/pom.xml          |   111 +-
 .../org.apache.spark.deploy.SparkDiagnosticsSetter |    18 +
 .../org.apache.spark.deploy.SparkSubmitOperation   |    17 +
 ...g.apache.spark.scheduler.ExternalClusterManager |    17 +
 .../scala/org/apache/spark/deploy/k8s/Config.scala |   370 +-
 .../org/apache/spark/deploy/k8s/Constants.scala    |    22 +-
 .../apache/spark/deploy/k8s/KubernetesConf.scala   |   125 +-
 .../spark/deploy/k8s/KubernetesDriverSpec.scala    |    53 +-
 .../apache/spark/deploy/k8s/KubernetesUtils.scala  |    81 +-
 .../spark/deploy/k8s/KubernetesVolumeSpec.scala    |     7 +-
 .../spark/deploy/k8s/KubernetesVolumeUtils.scala   |    69 +-
 .../deploy/k8s/SparkKubernetesClientFactory.scala  |    55 +-
 .../k8s/SparkKubernetesDiagnosticsSetter.scala     |    81 +
 .../org/apache/spark/deploy/k8s/SparkPod.scala     |     6 +-
 .../k8s/features/BasicDriverFeatureStep.scala      |    71 +-
 .../k8s/features/BasicExecutorFeatureStep.scala    |   168 +-
 .../k8s/features/DriverCommandFeatureStep.scala    |    49 +-
 .../DriverKubernetesCredentialsFeatureStep.scala   |     9 +-
 .../k8s/features/DriverServiceFeatureStep.scala    |    39 +-
 .../k8s/features/EnvSecretsFeatureStep.scala       |     2 +-
 .../k8s/features/HadoopConfDriverFeatureStep.scala |    18 +-
 .../features/HadoopConfExecutorFeatureStep.scala   |    63 +
 .../features/KerberosConfDriverFeatureStep.scala   |    38 +-
 .../KubernetesDriverCustomFeatureConfigStep.scala  |    79 +
 ...KubernetesExecutorCustomFeatureConfigStep.scala |    79 +
 .../k8s/features/KubernetesFeatureConfigStep.scala |    15 +-
 .../deploy/k8s/features/LocalDirsFeatureStep.scala |    21 +-
 .../k8s/features/MountVolumesFeatureStep.scala     |    35 +-
 .../k8s/features/NetworkPolicyFeatureStep.scala    |    60 +
 .../k8s/features/PodTemplateConfigMapStep.scala    |     5 +-
 .../spark/deploy/k8s/submit/K8sSubmitOps.scala     |    44 +-
 .../k8s/submit/KubernetesClientApplication.scala   |    86 +-
 .../deploy/k8s/submit/KubernetesClientUtils.scala  |    74 +-
 .../k8s/submit/KubernetesDriverBuilder.scala       |    44 +-
 .../k8s/submit/LoggingPodStatusWatcher.scala       |    26 +-
 .../spark/deploy/k8s/submit/MainAppResource.scala  |    46 +-
 .../cluster/k8s/AbstractPodsAllocator.scala        |    60 +-
 .../cluster/k8s/DeploymentPodsAllocator.scala      |   200 +
 .../cluster/k8s/ExecutorPVCResizePlugin.scala      |   262 +
 .../cluster/k8s/ExecutorPodsAllocator.scala        |   318 +-
 .../cluster/k8s/ExecutorPodsLifecycleManager.scala |   115 +-
 .../k8s/ExecutorPodsPollingSnapshotSource.scala    |    31 +-
 .../cluster/k8s/ExecutorPodsSnapshot.scala         |     8 +-
 .../k8s/ExecutorPodsSnapshotsStoreImpl.scala       |    11 +-
 .../k8s/ExecutorPodsWatchSnapshotSource.scala      |    60 +-
 .../cluster/k8s/ExecutorResizePlugin.scala         |   170 +
 .../scheduler/cluster/k8s/ExecutorRollPlugin.scala |   189 +
 .../cluster/k8s/KubernetesClusterManager.scala     |    86 +-
 .../k8s/KubernetesClusterSchedulerBackend.scala    |   174 +-
 .../cluster/k8s/KubernetesExecutorBackend.scala    |    13 +-
 .../cluster/k8s/KubernetesExecutorBuilder.scala    |    29 +-
 .../cluster/k8s/StatefulSetPodsAllocator.scala     |   208 +
 .../cluster/k8s/StatefulsetPodsAllocator.scala     |   201 -
 ...ernetesLocalDiskShuffleExecutorComponents.scala |    95 +-
 .../test/resources/driver-podgroup-template.yml    |    25 +
 .../core/src/test/resources/log4j.properties       |    31 -
 .../core/src/test/resources/log4j2.properties      |    39 +
 .../apache/spark/deploy/k8s/Fabric8Aliases.scala   |    29 +-
 .../spark/deploy/k8s/KubernetesConfSuite.scala     |   148 +-
 .../deploy/k8s/KubernetesDriverSpecSuite.scala     |    86 +
 .../spark/deploy/k8s/KubernetesTestConf.scala      |    32 +-
 .../spark/deploy/k8s/KubernetesUtilsSuite.scala    |   103 +-
 .../deploy/k8s/KubernetesVolumeUtilsSuite.scala    |   138 +-
 .../apache/spark/deploy/k8s/PodBuilderSuite.scala  |   137 +-
 .../spark/deploy/k8s/SecretVolumeUtils.scala       |     2 +-
 .../SparkKubernetesDiagnosticsSetterSuite.scala    |    83 +
 .../k8s/features/BasicDriverFeatureStepSuite.scala |   255 +-
 .../features/BasicExecutorFeatureStepSuite.scala   |   314 +-
 .../features/DriverCommandFeatureStepSuite.scala   |    27 +-
 ...iverKubernetesCredentialsFeatureStepSuite.scala |    11 +-
 .../features/DriverServiceFeatureStepSuite.scala   |   103 +-
 .../k8s/features/EnvSecretsFeatureStepSuite.scala  |     2 +-
 ...utorKubernetesCredentialsFeatureStepSuite.scala |    57 +-
 .../HadoopConfDriverFeatureStepSuite.scala         |     7 +-
 .../HadoopConfExecutorFeatureStepSuite.scala       |    80 +
 .../KerberosConfDriverFeatureStepSuite.scala       |    14 +-
 .../k8s/features/KubernetesFeaturesTestUtils.scala |     2 +-
 .../k8s/features/LocalDirsFeatureStepSuite.scala   |    25 +-
 .../features/MountVolumesFeatureStepSuite.scala    |   249 +-
 .../features/NetworkPolicyFeatureStepSuite.scala   |    60 +
 .../spark/deploy/k8s/submit/ClientSuite.scala      |   141 +-
 .../spark/deploy/k8s/submit/K8sSubmitOpSuite.scala |    44 +-
 .../k8s/submit/KubernetesClientUtilsSuite.scala    |    52 +
 .../k8s/submit/KubernetesDriverBuilderSuite.scala  |    88 +
 .../cluster/k8s/DeploymentAllocatorSuite.scala     |   190 +
 .../DeterministicExecutorPodsSnapshotsStore.scala  |     3 +-
 .../cluster/k8s/ExecutorLifecycleTestUtils.scala   |    47 +-
 .../cluster/k8s/ExecutorPVCResizePluginSuite.scala |   307 +
 .../cluster/k8s/ExecutorPodsAllocatorSuite.scala   |   746 +-
 .../k8s/ExecutorPodsLifecycleManagerSuite.scala    |   154 +-
 .../ExecutorPodsPollingSnapshotSourceSuite.scala   |    33 +-
 .../k8s/ExecutorPodsSnapshotsStoreSuite.scala      |     8 +-
 .../k8s/ExecutorPodsWatchSnapshotSourceSuite.scala |    33 +-
 .../cluster/k8s/ExecutorResizePluginSuite.scala    |   304 +
 .../cluster/k8s/ExecutorRollPluginSuite.scala      |   417 +
 .../k8s/KubernetesClusterManagerSuite.scala        |    74 +-
 .../KubernetesClusterSchedulerBackendSuite.scala   |   207 +-
 .../k8s/KubernetesExecutorBuilderSuite.scala       |    47 +
 .../cluster/k8s/StatefulSetAllocatorSuite.scala    |   166 +
 .../cluster/k8s/StatefulsetAllocatorSuite.scala    |   153 -
 .../KubernetesLocalDiskShuffleDataIOSuite.scala    |    33 +
 .../spark/shuffle/ShuffleChecksumUtilsSuite.scala  |   134 +
 .../deploy/k8s/features/VolcanoFeatureStep.scala   |    98 +
 .../k8s/features/VolcanoFeatureStepSuite.scala     |   104 +
 .../docker/src/main/dockerfiles/spark/Dockerfile   |    30 +-
 .../main/dockerfiles/spark/bindings/R/Dockerfile   |    19 +-
 .../dockerfiles/spark/bindings/python/Dockerfile   |    14 +-
 .../docker/src/main/dockerfiles/spark/decom.sh     |    15 +-
 .../src/main/dockerfiles/spark/entrypoint.sh       |    22 +-
 .../kubernetes/integration-tests/README.md         |   140 +-
 .../dev/dev-run-integration-tests.sh               |    39 +-
 .../integration-tests/dev/spark-rbac.yaml          |     6 +-
 .../kubernetes/integration-tests/pom.xml           |   105 +-
 .../scripts/setup-integration-test-env.sh          |    28 +-
 .../deploy/k8s/integrationtest/YuniKornTag.java    |    27 +
 .../test/resources/driver-schedule-template.yml    |    27 +
 .../resources/log-config-test-log4j.properties     |    13 +-
 .../src/test/resources/log4j.properties            |    31 -
 .../src/test/resources/log4j2.properties           |    39 +
 .../src/test/resources/pagerank_data.txt           |     6 +
 .../volcano/driver-podgroup-template-memory-3g.yml |    23 +
 .../high-priority-driver-podgroup-template.yml     |    21 +
 .../volcano/high-priority-driver-template.yml      |    26 +
 .../low-priority-driver-podgroup-template.yml      |    21 +
 .../volcano/low-priority-driver-template.yml       |    26 +
 .../medium-priority-driver-podgroup-template.yml   |    21 +
 .../volcano/medium-priority-driver-template.yml    |    26 +
 .../src/test/resources/volcano/priorityClasses.yml |    33 +
 .../volcano/queue-driver-podgroup-template.yml     |    20 +
 .../volcano/queue0-driver-podgroup-template.yml    |    20 +
 .../volcano/queue1-driver-podgroup-template.yml    |    20 +
 .../k8s/integrationtest/BasicTestsSuite.scala      |    89 +-
 .../k8s/integrationtest/ClientModeTestsSuite.scala |    16 +-
 .../k8s/integrationtest/DecommissionSuite.scala    |   349 +-
 .../k8s/integrationtest/DepsTestsSuite.scala       |   141 +-
 .../k8s/integrationtest/KubernetesSuite.scala      |   112 +-
 .../integrationtest/KubernetesTestComponents.scala |    19 +-
 .../deploy/k8s/integrationtest/PVTestsSuite.scala  |    70 +-
 .../k8s/integrationtest/PodTemplateSuite.scala     |    28 +-
 .../deploy/k8s/integrationtest/ProcessUtils.scala  |     2 +-
 .../k8s/integrationtest/PythonTestsSuite.scala     |    18 +
 .../k8s/integrationtest/SecretsTestsSuite.scala    |    16 +-
 .../integrationtest/SparkConfPropagateSuite.scala  |    17 +-
 .../integrationtest/SparkReadinessWatcher.scala    |     2 +-
 .../deploy/k8s/integrationtest/TestConstants.scala |     7 +-
 .../spark/deploy/k8s/integrationtest/Utils.scala   |    33 +-
 .../deploy/k8s/integrationtest/VolumeSuite.scala   |   183 +
 .../deploy/k8s/integrationtest/YuniKornSuite.scala |    32 +
 .../backend/IntegrationTestBackend.scala           |    14 +-
 .../backend/cloud/KubeConfigBackend.scala          |    20 +-
 .../backend/docker/DockerForDesktopBackend.scala   |     2 +-
 .../backend/docker/RancherDesktopBackend.scala     |    25 +
 .../backend/minikube/Minikube.scala                |    16 +-
 .../backend/minikube/MinikubeTestBackend.scala     |    15 +-
 .../integration-tests/tests/pyfiles_connect.py     |    43 +
 .../deploy/k8s/integrationtest/VolcanoSuite.scala  |    33 +
 .../k8s/integrationtest/VolcanoTestsSuite.scala    |   504 +
 resource-managers/mesos/pom.xml                    |   128 -
 .../mesos/MesosExternalBlockStoreClient.java       |   124 -
 ...g.apache.spark.scheduler.ExternalClusterManager |     1 -
 .../deploy/mesos/MesosClusterDispatcher.scala      |   136 -
 .../mesos/MesosClusterDispatcherArguments.scala    |   149 -
 .../deploy/mesos/MesosDriverDescription.scala      |    70 -
 .../deploy/mesos/MesosExternalShuffleService.scala |   131 -
 .../org/apache/spark/deploy/mesos/config.scala     |   420 -
 .../apache/spark/deploy/mesos/ui/DriverPage.scala  |   186 -
 .../spark/deploy/mesos/ui/MesosClusterPage.scala   |   156 -
 .../spark/deploy/mesos/ui/MesosClusterUI.scala     |    48 -
 .../spark/deploy/rest/mesos/MesosRestServer.scala  |   173 -
 .../spark/executor/MesosExecutorBackend.scala      |   132 -
 .../cluster/mesos/MesosClusterManager.scala        |    65 -
 .../mesos/MesosClusterPersistenceEngine.scala      |   135 -
 .../cluster/mesos/MesosClusterScheduler.scala      |   904 -
 .../mesos/MesosClusterSchedulerSource.scala        |    41 -
 .../mesos/MesosCoarseGrainedSchedulerBackend.scala |   821 -
 .../mesos/MesosFineGrainedSchedulerBackend.scala   |   466 -
 .../scheduler/cluster/mesos/MesosProtoUtils.scala  |    53 -
 .../scheduler/cluster/mesos/MesosScheduler.scala   |    28 -
 .../cluster/mesos/MesosSchedulerBackendUtil.scala  |   307 -
 .../cluster/mesos/MesosSchedulerUtils.scala        |   620 -
 .../MesosClusterDispatcherArgumentsSuite.scala     |    63 -
 .../deploy/mesos/MesosClusterDispatcherSuite.scala |    40 -
 .../deploy/rest/mesos/MesosRestServerSuite.scala   |    85 -
 .../cluster/mesos/MesosClusterManagerSuite.scala   |    57 -
 .../cluster/mesos/MesosClusterSchedulerSuite.scala |   822 -
 .../MesosCoarseGrainedSchedulerBackendSuite.scala  |   843 -
 .../MesosFineGrainedSchedulerBackendSuite.scala    |   424 -
 .../cluster/mesos/MesosProtoUtilsSuite.scala       |    48 -
 .../mesos/MesosSchedulerBackendUtilSuite.scala     |    71 -
 .../cluster/mesos/MesosSchedulerUtilsSuite.scala   |   398 -
 .../spark/scheduler/cluster/mesos/Utils.scala      |   229 -
 resource-managers/yarn/pom.xml                     |   191 +-
 .../org/apache/spark/deploy/yarn/AmIpFilter.java   |   239 +
 .../apache/spark/deploy/yarn/AmIpPrincipal.java    |    35 +
 .../deploy/yarn/AmIpServletRequestWrapper.java     |    54 +
 .../org/apache/spark/deploy/yarn/ProxyUtils.java   |   126 +
 ...g.apache.spark.scheduler.ExternalClusterManager |    17 +
 .../spark/deploy/yarn/ApplicationMaster.scala      |   206 +-
 .../deploy/yarn/ApplicationMasterArguments.scala   |     4 -
 .../org/apache/spark/deploy/yarn/Client.scala      |   400 +-
 .../apache/spark/deploy/yarn/ClientArguments.scala |    15 +-
 .../yarn/ClientDistributedCacheManager.scala       |    45 +-
 .../spark/deploy/yarn/ExecutorRunnable.scala       |   113 +-
 ...calityPreferredContainerPlacementStrategy.scala |     5 +-
 .../spark/deploy/yarn/ResourceRequestHelper.scala  |   120 +-
 .../spark/deploy/yarn/SparkRackResolver.scala      |    18 +-
 .../apache/spark/deploy/yarn/YarnAllocator.scala   |   307 +-
 .../yarn/YarnAllocatorNodeHealthTracker.scala      |    77 +-
 .../deploy/yarn/YarnProxyRedirectFilter.scala      |     4 +-
 .../apache/spark/deploy/yarn/YarnRMClient.scala    |     4 +-
 .../spark/deploy/yarn/YarnSparkHadoopUtil.scala    |   105 +-
 .../org/apache/spark/deploy/yarn/config.scala      |   426 -
 .../apache/spark/deploy/yarn/config/package.scala  |   516 +
 .../YarnCoarseGrainedExecutorBackend.scala         |     3 +-
 .../cluster/YarnClientSchedulerBackend.scala       |    35 +-
 .../cluster/YarnClusterSchedulerBackend.scala      |     7 +-
 .../scheduler/cluster/YarnSchedulerBackend.scala   |    52 +-
 .../spark/util/YarnContainerInfoHelper.scala       |     7 +-
 .../org/apache/hadoop/net/ServerSocketUtil.java    |   132 -
 .../org/eclipse/jetty/server/SessionManager.java   |   290 -
 .../jetty/server/session/SessionHandler.java       |    90 -
 .../yarn/src/test/resources/certchain.pem          |    17 +
 resource-managers/yarn/src/test/resources/key.pem  |    29 +
 resource-managers/yarn/src/test/resources/keystore |   Bin 0 -> 2247 bytes
 .../yarn/src/test/resources/log4j.properties       |    31 -
 .../yarn/src/test/resources/log4j2.properties      |    39 +
 .../yarn/src/test/resources/truststore             |   Bin 0 -> 957 bytes
 .../src/test/resources/unencrypted-certchain.pem   |    21 +
 .../yarn/src/test/resources/unencrypted-key.pem    |    28 +
 .../yarn/src/test/resources/untrusted-keystore     |   Bin 0 -> 2246 bytes
 .../apache/spark/deploy/yarn/AmIpFilterSuite.scala |   338 +
 .../spark/deploy/yarn/BaseYarnClusterSuite.scala   |    57 +-
 .../yarn/ClientDistributedCacheManagerSuite.scala  |    64 +-
 .../org/apache/spark/deploy/yarn/ClientSuite.scala |   506 +-
 .../yarn/ContainerPlacementStrategySuite.scala     |    30 +-
 .../spark/deploy/yarn/ExecutorRunnableSuite.scala  |   141 +
 .../spark/deploy/yarn/FailureTrackerSuite.scala    |   101 -
 .../deploy/yarn/ResourceRequestHelperSuite.scala   |    58 +-
 .../deploy/yarn/ResourceRequestTestHelper.scala    |    80 +-
 .../yarn/YarnAllocatorHealthTrackerSuite.scala     |    13 +-
 .../spark/deploy/yarn/YarnAllocatorSuite.scala     |   508 +-
 .../spark/deploy/yarn/YarnClusterSuite.scala       |   240 +-
 .../deploy/yarn/YarnProxyRedirectFilterSuite.scala |     4 +-
 .../yarn/YarnShuffleAlternateNameConfigSuite.scala |    18 +-
 .../deploy/yarn/YarnShuffleIntegrationSuite.scala  |    70 +-
 .../deploy/yarn/YarnSparkHadoopUtilSuite.scala     |    53 +-
 .../network/shuffle/ShuffleTestAccessor.scala      |   173 +-
 .../network/yarn/SslYarnShuffleServiceSuite.scala  |    34 +
 .../yarn/YarnShuffleServiceMetricsSuite.scala      |     4 +-
 .../network/yarn/YarnShuffleServiceSuite.scala     |   894 +-
 .../spark/network/yarn/YarnTestAccessor.scala      |     4 +
 .../cluster/YarnSchedulerBackendSuite.scala        |    10 +-
 sbin/decommission-slave.sh                         |    23 -
 sbin/slaves.sh                                     |    23 -
 sbin/spark-config.sh                               |     2 +-
 sbin/spark-daemon.sh                               |    20 +-
 sbin/start-connect-server.sh                       |    46 +
 sbin/start-history-server.sh                       |     2 +-
 sbin/start-master.sh                               |    10 +-
 sbin/start-mesos-dispatcher.sh                     |    51 -
 sbin/start-mesos-shuffle-service.sh                |    36 -
 sbin/start-slave.sh                                |    23 -
 sbin/start-slaves.sh                               |    23 -
 sbin/start-thriftserver.sh                         |     3 +-
 sbin/start-worker.sh                               |     2 +-
 sbin/stop-connect-server.sh                        |    26 +
 sbin/stop-mesos-dispatcher.sh                      |    33 -
 sbin/stop-mesos-shuffle-service.sh                 |    26 -
 sbin/stop-slave.sh                                 |    23 -
 sbin/stop-slaves.sh                                |    23 -
 scalastyle-config.xml                              |   487 +-
 sql/README.md                                      |     3 +-
 sql/api/pom.xml                                    |   110 +
 .../spark/sql/catalyst/parser/SqlBaseLexer.g4      |   712 +
 .../spark/sql/catalyst/parser/SqlBaseParser.g4     |  2714 ++
 .../function/FlatMapGroupsWithStateFunction.java   |    37 +
 .../java/function/MapGroupsWithStateFunction.java  |    36 +
 .../main/java/org/apache/spark/sql/RowFactory.java |     0
 .../main/java/org/apache/spark/sql/SaveMode.java   |     0
 .../java/org/apache/spark/sql/api/java/UDF0.java   |     0
 .../java/org/apache/spark/sql/api/java/UDF1.java   |     0
 .../java/org/apache/spark/sql/api/java/UDF10.java  |     0
 .../java/org/apache/spark/sql/api/java/UDF11.java  |     0
 .../java/org/apache/spark/sql/api/java/UDF12.java  |     0
 .../java/org/apache/spark/sql/api/java/UDF13.java  |     0
 .../java/org/apache/spark/sql/api/java/UDF14.java  |     0
 .../java/org/apache/spark/sql/api/java/UDF15.java  |     0
 .../java/org/apache/spark/sql/api/java/UDF16.java  |     0
 .../java/org/apache/spark/sql/api/java/UDF17.java  |     0
 .../java/org/apache/spark/sql/api/java/UDF18.java  |     0
 .../java/org/apache/spark/sql/api/java/UDF19.java  |     0
 .../java/org/apache/spark/sql/api/java/UDF2.java   |     0
 .../java/org/apache/spark/sql/api/java/UDF20.java  |     0
 .../java/org/apache/spark/sql/api/java/UDF21.java  |     0
 .../java/org/apache/spark/sql/api/java/UDF22.java  |     0
 .../java/org/apache/spark/sql/api/java/UDF3.java   |     0
 .../java/org/apache/spark/sql/api/java/UDF4.java   |     0
 .../java/org/apache/spark/sql/api/java/UDF5.java   |     0
 .../java/org/apache/spark/sql/api/java/UDF6.java   |     0
 .../java/org/apache/spark/sql/api/java/UDF7.java   |     0
 .../java/org/apache/spark/sql/api/java/UDF8.java   |     0
 .../java/org/apache/spark/sql/api/java/UDF9.java   |     0
 .../apache/spark/sql/api/java/package-info.java    |     0
 .../spark/sql/connector/catalog/Identifier.java    |    43 +
 .../sql/connector/catalog/IdentifierImpl.java      |    74 +
 .../sql/connector/catalog/IdentityColumnSpec.java  |    88 +
 .../spark/sql/expressions/javalang/typed.java      |    74 +
 .../CartesianSpatialReferenceSystemMapper.java     |    38 +
 .../GeographicSpatialReferenceSystemMapper.java    |    38 +
 .../types/SpatialReferenceSystemCache.java         |   171 +
 .../types/SpatialReferenceSystemInformation.java   |    33 +
 .../types/SpatialReferenceSystemMapper.java        |    30 +
 .../spark/sql/streaming/GroupStateTimeout.java     |    68 +
 .../org/apache/spark/sql/streaming/OutputMode.java |     0
 .../org/apache/spark/sql/streaming/TimeMode.java   |    48 +
 .../org/apache/spark/sql/streaming/Trigger.java    |   233 +
 .../java/org/apache/spark/sql/types/DataTypes.java |   316 +
 .../java/org/apache/spark/sql/types/Geography.java |    80 +
 .../java/org/apache/spark/sql/types/Geometry.java  |    80 +
 .../apache/spark/sql/types/SQLUserDefinedType.java |     0
 .../resources/gregorian-julian-rebase-micros.json  |  2409 ++
 .../resources/julian-gregorian-rebase-micros.json  |  2409 ++
 .../org/apache/spark/sql/srs_registry.csv          | 10717 +++++++
 .../org/apache/spark/sql/AnalysisException.scala   |   228 +
 .../main/scala/org/apache/spark/sql/Artifact.scala |   157 +
 .../main/scala/org/apache/spark/sql/Column.scala   |  1558 +
 .../apache/spark/sql/DataFrameNaFunctions.scala    |   361 +
 .../org/apache/spark/sql/DataFrameReader.scala     |   622 +
 .../apache/spark/sql/DataFrameStatFunctions.scala  |   585 +
 .../org/apache/spark/sql/DataFrameWriter.scala     |   532 +
 .../org/apache/spark/sql/DataFrameWriterV2.scala   |   265 +
 .../main/scala/org/apache/spark/sql/Dataset.scala  |  3334 ++
 .../scala/org/apache/spark/sql/DatasetHolder.scala |    45 +
 .../main/scala/org/apache/spark/sql/Encoder.scala  |    78 +
 .../main/scala/org/apache/spark/sql/Encoders.scala |   392 +
 .../scala/org/apache/spark/sql/ForeachWriter.scala |   130 +
 .../apache/spark/sql/KeyValueGroupedDataset.scala  |  1022 +
 .../org/apache/spark/sql/MergeIntoWriter.scala     |   325 +
 .../scala/org/apache/spark/sql/Observation.scala   |   159 +
 .../spark/sql/RelationalGroupedDataset.scala       |   359 +
 .../src/main/scala/org/apache/spark/sql/Row.scala  |   697 +
 .../scala/org/apache/spark/sql/RuntimeConfig.scala |   140 +
 .../scala/org/apache/spark/sql/SQLContext.scala    |  1030 +
 .../scala/org/apache/spark/sql/SQLImplicits.scala  |   318 +
 .../scala/org/apache/spark/sql/SparkSession.scala  |  1288 +
 .../org/apache/spark/sql/TableValuedFunction.scala |   192 +
 .../org/apache/spark/sql/UDFRegistration.scala     |  1364 +
 .../org/apache/spark/sql/avro/functions.scala      |   125 +
 .../org/apache/spark/sql/catalog/Catalog.scala     |   893 +
 .../org/apache/spark/sql/catalog/interface.scala   |   278 +
 .../sql/catalyst/DefinedByConstructorParams.scala  |    24 +
 .../spark/sql/catalyst/JavaTypeInference.scala     |   196 +
 .../spark/sql/catalyst/ScalaReflection.scala       |   536 +
 .../apache/spark/sql/catalyst/WalkedTypePath.scala |    58 +
 .../sql/catalyst/analysis/NonEmptyException.scala  |    36 +
 .../sql/catalyst/analysis/SqlApiAnalysis.scala     |    26 +
 .../catalyst/analysis/alreadyExistException.scala  |   199 +
 .../catalyst/analysis/noSuchItemsExceptions.scala  |   243 +
 .../sql/catalyst/encoders/AgnosticEncoder.scala    |   306 +
 .../spark/sql/catalyst/encoders/OuterScopes.scala  |   183 +
 .../spark/sql/catalyst/encoders/RowEncoder.scala   |   139 +
 .../spark/sql/catalyst/encoders/codecs.scala       |    73 +
 .../sql/catalyst/expressions/OrderUtils.scala      |    35 +
 .../spark/sql/catalyst/expressions/rows.scala      |    51 +
 .../sql/catalyst/parser/DataTypeAstBuilder.scala   |   597 +
 .../catalyst/parser/DataTypeParserInterface.scala  |    38 +
 .../catalyst/parser/LegacyTypeStringParser.scala   |    96 +
 .../spark/sql/catalyst/parser/PositionMapper.scala |   172 +
 .../catalyst/parser/SparkParserErrorStrategy.scala |   155 +
 .../parser/SubstituteParmsAstBuilder.scala         |   251 +
 .../apache/spark/sql/catalyst/parser/parsers.scala |   621 +
 .../catalyst/plans/logical/LogicalGroupState.scala |    20 +
 .../sql/catalyst/plans/logical/TimeMode.scala      |    52 +
 .../plans/logical/groupStateTimeouts.scala         |    24 +
 .../catalyst/streaming/InternalOutputModes.scala   |    65 +
 .../spark/sql/catalyst/trees/QueryContexts.scala   |   179 +
 .../apache/spark/sql/catalyst/trees/origin.scala   |   179 +
 .../sql/catalyst/util/AttributeNameParser.scala    |    68 +
 .../sql/catalyst/util/CaseInsensitiveMap.scala     |    64 +
 .../sql/catalyst/util/DataTypeJsonUtils.scala      |    56 +
 .../spark/sql/catalyst/util/DateFormatter.scala    |   232 +
 .../catalyst/util/DateTimeFormatterHelper.scala    |   371 +
 .../apache/spark/sql/catalyst/util/MathUtils.scala |   110 +
 .../spark/sql/catalyst/util/QuotingUtils.scala     |    86 +
 .../spark/sql/catalyst/util/RebaseDateTime.scala   |   556 +
 .../catalyst/util/ResolveDefaultColumnsUtils.scala |    46 +
 .../sql/catalyst/util/SparkCharVarcharUtils.scala  |    63 +
 .../sql/catalyst/util/SparkDateTimeUtils.scala     |   828 +
 .../sql/catalyst/util/SparkIntervalUtils.scala     |   572 +
 .../spark/sql/catalyst/util/SparkParserUtils.scala |   270 +
 .../spark/sql/catalyst/util/StringConcat.scala     |    63 +
 .../spark/sql/catalyst/util/TimeFormatter.scala    |   142 +
 .../sql/catalyst/util/TimestampFormatter.scala     |   660 +
 .../apache/spark/sql/catalyst/util/UDTUtils.scala  |    61 +
 .../spark/sql/errors/CompilationErrors.scala       |   148 +
 .../apache/spark/sql/errors/DataTypeErrors.scala   |   278 +
 .../spark/sql/errors/DataTypeErrorsBase.scala      |   103 +
 .../apache/spark/sql/errors/ExecutionErrors.scala  |   264 +
 .../spark/sql/errors/QueryParsingErrors.scala      |   945 +
 .../spark/sql/execution/streaming/Triggers.scala   |   156 +
 .../apache/spark/sql/expressions/Aggregator.scala  |   101 +
 .../spark/sql/expressions/ReduceAggregator.scala   |    84 +
 .../sql/expressions/UserDefinedFunction.scala      |   206 +
 .../org/apache/spark/sql/expressions/Window.scala  |   241 +
 .../apache/spark/sql/expressions/WindowSpec.scala  |   216 +
 .../spark/sql/expressions/scalalang/typed.scala    |    69 +
 .../org/apache/spark/sql/expressions/udaf.scala    |   162 +
 .../scala/org/apache/spark/sql/functions.scala     | 11863 +++++++
 .../spark/sql/internal/LegacyBehaviorPolicy.scala  |    21 +
 .../org/apache/spark/sql/internal/SqlApiConf.scala |   115 +
 .../spark/sql/internal/SqlApiConfHelper.scala      |    58 +
 .../sql/internal/TableValuedFunctionArgument.scala |    20 +
 .../org/apache/spark/sql/internal/ToScalaUDF.scala |   846 +
 .../apache/spark/sql/internal/columnNodes.scala    |   675 +
 .../spark/sql/internal/typedaggregators.scala      |    98 +
 .../sql/internal/types/AbstractArrayType.scala     |    36 +
 .../spark/sql/internal/types/AbstractMapType.scala |    40 +
 .../sql/internal/types/AbstractStringType.scala    |   121 +
 .../main/scala/org/apache/spark/sql/package.scala  |    63 +
 .../org/apache/spark/sql/protobuf/functions.scala  |   298 +
 .../spark/sql/streaming/DataStreamReader.scala     |   385 +
 .../spark/sql/streaming/DataStreamWriter.scala     |   229 +
 .../spark/sql/streaming/ExpiredTimerInfo.scala     |    36 +
 .../apache/spark/sql/streaming/GroupState.scala    |   340 +
 .../org/apache/spark/sql/streaming/ListState.scala |    78 +
 .../org/apache/spark/sql/streaming/MapState.scala  |   108 +
 .../org/apache/spark/sql/streaming/QueryInfo.scala |    58 +
 .../spark/sql/streaming/StatefulProcessor.scala    |   159 +
 .../sql/streaming/StatefulProcessorHandle.scala    |   233 +
 .../sql/streaming/StreamingCheckpointManager.scala |    58 +
 .../spark/sql/streaming/StreamingQuery.scala       |   176 +
 .../sql/streaming/StreamingQueryListener.scala     |   310 +
 .../sql/streaming/StreamingQueryManager.scala      |   129 +
 .../spark/sql/streaming/StreamingQueryStatus.scala |    71 +
 .../org/apache/spark/sql/streaming/TTLConfig.scala |    45 +
 .../apache/spark/sql/streaming/TimerValues.scala   |    54 +
 .../apache/spark/sql/streaming/ValueState.scala    |    61 +
 .../org/apache/spark/sql/streaming/progress.scala  |   400 +
 .../apache/spark/sql/types/AbstractDataType.scala  |   191 +
 .../org/apache/spark/sql/types/ArrayType.scala     |   122 +
 .../org/apache/spark/sql/types/BinaryType.scala    |    45 +
 .../org/apache/spark/sql/types/BooleanType.scala   |    46 +
 .../org/apache/spark/sql/types/ByteType.scala      |    48 +
 .../spark/sql/types/CalendarIntervalType.scala     |    53 +
 .../org/apache/spark/sql/types/CharType.scala      |    65 +
 .../org/apache/spark/sql/types/DataType.scala      |   606 +
 .../org/apache/spark/sql/types/DateType.scala      |    53 +
 .../spark/sql/types/DayTimeIntervalType.scala      |   105 +
 .../scala/org/apache/spark/sql/types/Decimal.scala |   734 +
 .../org/apache/spark/sql/types/DecimalType.scala   |   249 +
 .../org/apache/spark/sql/types/DoubleType.scala    |    75 +
 .../org/apache/spark/sql/types/FloatType.scala     |    78 +
 .../org/apache/spark/sql/types/GeographyType.scala |   269 +
 .../org/apache/spark/sql/types/GeometryType.scala  |   229 +
 .../org/apache/spark/sql/types/IntegerType.scala   |    48 +
 .../org/apache/spark/sql/types/LongType.scala      |    48 +
 .../scala/org/apache/spark/sql/types/MapType.scala |   126 +
 .../org/apache/spark/sql/types/Metadata.scala      |   357 +
 .../org/apache/spark/sql/types/NullType.scala      |    47 +
 .../org/apache/spark/sql/types/ObjectType.scala    |    48 +
 .../org/apache/spark/sql/types/ShortType.scala     |    48 +
 .../org/apache/spark/sql/types/SpatialType.scala   |    31 +
 .../org/apache/spark/sql/types/StringType.scala    |   229 +
 .../org/apache/spark/sql/types/StructField.scala   |   275 +
 .../org/apache/spark/sql/types/StructType.scala    |   741 +
 .../org/apache/spark/sql/types/TimeType.scala      |    58 +
 .../apache/spark/sql/types/TimestampNTZType.scala  |    57 +
 .../org/apache/spark/sql/types/TimestampType.scala |    54 +
 .../apache/spark/sql/types/UDTRegistration.scala   |    96 +
 .../org/apache/spark/sql/types/UpCastRule.scala    |    82 +
 .../apache/spark/sql/types/UserDefinedType.scala   |   162 +
 .../org/apache/spark/sql/types/VarcharType.scala   |    64 +
 .../org/apache/spark/sql/types/VariantType.scala   |    47 +
 .../spark/sql/types/YearMonthIntervalType.scala    |    97 +
 .../spark/sql/types/ops/TimeTypeApiOps.scala       |    90 +
 .../apache/spark/sql/types/ops/TypeApiOps.scala    |   180 +
 .../org/apache/spark/sql/util/ArrowUtils.scala     |   433 +
 .../org/apache/spark/sql/util/ArtifactUtils.scala  |    59 +
 .../apache/spark/sql/util/CloseableIterator.scala  |    66 +
 .../sql/util/ConcatenatingArrowStreamReader.scala  |   187 +
 .../org/apache/spark/sql/util/ProtobufUtils.scala  |    39 +
 .../spark/sql/types/GeographyClientSuite.java      |    86 +
 .../spark/sql/types/GeometryClientSuite.java       |    86 +
 ...kSessionBuilderImplementationBindingSuite.scala |    87 +
 .../spark/sql/StaticProcedureSuiteBase.scala       |    33 +
 .../CalendarIntervalBenchmark-jdk21-results.txt    |    11 +
 .../CalendarIntervalBenchmark-jdk25-results.txt    |    11 +
 .../CalendarIntervalBenchmark-results.txt          |    11 +
 .../EnumTypeSetBenchmark-jdk21-results.txt         |   105 +
 .../EnumTypeSetBenchmark-jdk25-results.txt         |   105 +
 .../benchmarks/EnumTypeSetBenchmark-results.txt    |   105 +
 .../EscapePathBenchmark-jdk21-results.txt          |    24 +
 .../EscapePathBenchmark-jdk25-results.txt          |    24 +
 .../benchmarks/EscapePathBenchmark-results.txt     |    24 +
 .../GenericArrayDataBenchmark-jdk11-results.txt    |    10 -
 .../GenericArrayDataBenchmark-jdk21-results.txt    |    10 +
 .../GenericArrayDataBenchmark-jdk25-results.txt    |    10 +
 .../GenericArrayDataBenchmark-results.txt          |    14 +-
 .../benchmarks/HashBenchmark-jdk11-results.txt     |    70 -
 .../benchmarks/HashBenchmark-jdk21-results.txt     |    70 +
 .../benchmarks/HashBenchmark-jdk25-results.txt     |    70 +
 sql/catalyst/benchmarks/HashBenchmark-results.txt  |    60 +-
 .../HashByteArrayBenchmark-jdk11-results.txt       |    77 -
 .../HashByteArrayBenchmark-jdk21-results.txt       |    77 +
 .../HashByteArrayBenchmark-jdk25-results.txt       |    77 +
 .../benchmarks/HashByteArrayBenchmark-results.txt  |    90 +-
 .../benchmarks/HexBenchmark-jdk21-results.txt      |    14 +
 .../benchmarks/HexBenchmark-jdk25-results.txt      |    14 +
 sql/catalyst/benchmarks/HexBenchmark-results.txt   |    14 +
 ...RowComparableWrapperBenchmark-jdk21-results.txt |     7 +
 ...RowComparableWrapperBenchmark-jdk25-results.txt |     7 +
 ...ternalRowComparableWrapperBenchmark-results.txt |     7 +
 .../SparkBloomFilterBenchmark-jdk21-results.txt    |   300 +
 .../SparkBloomFilterBenchmark-jdk25-results.txt    |   300 +
 .../SparkBloomFilterBenchmark-results.txt          |   300 +
 .../UnsafeProjectionBenchmark-jdk11-results.txt    |    14 -
 .../UnsafeProjectionBenchmark-jdk21-results.txt    |    14 +
 .../UnsafeProjectionBenchmark-jdk25-results.txt    |    14 +
 .../UnsafeProjectionBenchmark-results.txt          |    12 +-
 sql/catalyst/pom.xml                               |   109 +-
 .../apache/spark/sql/catalyst/parser/SqlBase.g4    |  1894 --
 .../catalyst/expressions/ArrayExpressionUtils.java |   208 +
 .../catalyst/expressions/ArrayOfDecimalsSerDe.java |   259 +
 .../expressions/BitmapExpressionUtils.java         |    72 +
 .../expressions/ExpressionDescription.java         |     2 +-
 .../catalyst/expressions/ExpressionImplUtils.java  |   345 +
 .../sql/catalyst/expressions/ExpressionInfo.java   |    54 +-
 .../FixedLengthRowBasedKeyValueBatch.java          |     4 +-
 .../expressions/RowBasedKeyValueBatch.java         |    18 +-
 .../catalyst/expressions/SpecializedGetters.java   |     9 +
 .../expressions/SpecializedGettersReader.java      |    67 +-
 .../sql/catalyst/expressions/ToJavaArrayUtils.java |   112 +
 .../sql/catalyst/expressions/UnsafeArrayData.java  |    49 +-
 .../sql/catalyst/expressions/UnsafeDataUtils.java  |     6 +-
 .../spark/sql/catalyst/expressions/UnsafeRow.java  |   112 +-
 .../catalyst/expressions/UnsafeRowChecksum.scala   |    53 +
 .../VariableLengthRowBasedKeyValueBatch.java       |     4 +-
 .../expressions/VectorFunctionImplUtils.java       |   465 +
 .../catalyst/expressions/codegen/BufferHolder.java |    21 +-
 .../expressions/codegen/UnsafeArrayWriter.java     |    30 +-
 .../catalyst/expressions/codegen/UnsafeWriter.java |    39 +-
 .../expressions/json/JsonExpressionUtils.java      |    80 +
 .../sql/catalyst/expressions/xml/UDFXPathUtil.java |     4 +-
 .../sql/catalyst/util/CharVarcharCodegenUtils.java |    14 +-
 .../org/apache/spark/sql/catalyst/util/Geo.java    |    84 +
 .../apache/spark/sql/catalyst/util/Geography.java  |   201 +
 .../apache/spark/sql/catalyst/util/Geometry.java   |   204 +
 .../sql/catalyst/util/HadoopCompressionCodec.java  |    67 +
 .../apache/spark/sql/catalyst/util/STUtils.java    |   185 +
 .../spark/sql/catalyst/util/geo/GeoTypeId.java     |    58 +
 .../sql/catalyst/util/geo/GeometryCollection.java  |    59 +
 .../spark/sql/catalyst/util/geo/GeometryModel.java |   248 +
 .../spark/sql/catalyst/util/geo/LineString.java    |    59 +
 .../sql/catalyst/util/geo/MultiLineString.java     |    64 +
 .../spark/sql/catalyst/util/geo/MultiPoint.java    |    64 +
 .../spark/sql/catalyst/util/geo/MultiPolygon.java  |    64 +
 .../apache/spark/sql/catalyst/util/geo/Point.java  |   134 +
 .../spark/sql/catalyst/util/geo/Polygon.java       |    67 +
 .../apache/spark/sql/catalyst/util/geo/Ring.java   |    74 +
 .../sql/catalyst/util/geo/WkbParseException.java   |    45 +
 .../spark/sql/catalyst/util/geo/WkbReader.java     |   496 +
 .../spark/sql/catalyst/util/geo/WkbUtil.java       |   132 +
 .../spark/sql/catalyst/util/geo/WkbWriter.java     |   225 +
 .../spark/sql/connector/ExternalCommandRunner.java |     4 +-
 .../sql/connector/catalog/CatalogExtension.java    |     2 +-
 .../spark/sql/connector/catalog/CatalogPlugin.java |     2 +-
 .../spark/sql/connector/catalog/Changelog.java     |   205 +
 .../spark/sql/connector/catalog/ChangelogInfo.java |    88 +
 .../sql/connector/catalog/ChangelogRange.java      |    87 +
 .../apache/spark/sql/connector/catalog/Column.java |   204 +
 .../sql/connector/catalog/ColumnDefaultValue.java  |    86 +
 .../spark/sql/connector/catalog/DefaultValue.java  |    95 +
 .../catalog/DelegatingCatalogExtension.java        |    65 +-
 .../spark/sql/connector/catalog/Dependency.java    |    58 +
 .../sql/connector/catalog/DependencyList.java      |    75 +
 .../sql/connector/catalog/FunctionCatalog.java     |    18 +
 .../sql/connector/catalog/FunctionDependency.java  |    68 +
 .../spark/sql/connector/catalog/Identifier.java    |    43 -
 .../sql/connector/catalog/IdentifierImpl.java      |    81 -
 .../sql/connector/catalog/MetadataColumn.java      |    74 +-
 .../spark/sql/connector/catalog/MetadataTable.java |   101 +
 .../sql/connector/catalog/NamespaceChange.java     |    10 +
 .../sql/connector/catalog/ProcedureCatalog.java    |    43 +
 .../spark/sql/connector/catalog/StagedTable.java   |    14 +
 .../sql/connector/catalog/StagingTableCatalog.java |   152 +-
 .../catalog/SupportsAtomicPartitionManagement.java |    15 +-
 .../connector/catalog/SupportsCatalogOptions.java  |    16 +
 .../sql/connector/catalog/SupportsDelete.java      |    14 +-
 .../sql/connector/catalog/SupportsDeleteV2.java    |    84 +
 .../connector/catalog/SupportsMetadataColumns.java |    50 +-
 .../sql/connector/catalog/SupportsNamespaces.java  |    16 +-
 .../catalog/SupportsPartitionManagement.java       |    27 +-
 .../catalog/SupportsRowLevelOperations.java        |    42 +
 .../connector/catalog/SupportsSchemaEvolution.java |    47 +
 .../SupportsV1OverwriteWithSaveAsTable.java        |    71 +
 .../apache/spark/sql/connector/catalog/Table.java  |    40 +-
 .../sql/connector/catalog/TableCapability.java     |     8 +-
 .../spark/sql/connector/catalog/TableCatalog.java  |   264 +-
 .../connector/catalog/TableCatalogCapability.java  |    96 +
 .../spark/sql/connector/catalog/TableChange.java   |   320 +-
 .../sql/connector/catalog/TableDependency.java     |    76 +
 .../spark/sql/connector/catalog/TableInfo.java     |   154 +
 .../spark/sql/connector/catalog/TableSummary.java  |    45 +
 .../sql/connector/catalog/TableViewCatalog.java    |   236 +
 .../sql/connector/catalog/TableWritePrivilege.java |    40 +
 .../catalog/TransactionalCatalogPlugin.java        |    41 +
 .../sql/connector/catalog/TruncatableTable.java    |    23 +
 .../spark/sql/connector/catalog/ViewCatalog.java   |   165 +
 .../spark/sql/connector/catalog/ViewInfo.java      |   172 +
 .../catalog/constraints/BaseConstraint.java        |   141 +
 .../sql/connector/catalog/constraints/Check.java   |   134 +
 .../connector/catalog/constraints/Constraint.java  |   127 +
 .../connector/catalog/constraints/ForeignKey.java  |   151 +
 .../connector/catalog/constraints/PrimaryKey.java  |   107 +
 .../sql/connector/catalog/constraints/Unique.java  |   113 +
 .../sql/connector/catalog/functions/Reducer.java   |    59 +
 .../catalog/functions/ReducibleFunction.java       |   106 +
 .../catalog/functions/ScalarFunction.java          |    10 +-
 .../catalog/functions/SimpleFunction.java          |    42 +
 .../sql/connector/catalog/index/SupportsIndex.java |    76 +
 .../sql/connector/catalog/index/TableIndex.java    |    77 +
 .../catalog/procedures/BoundProcedure.java         |    57 +
 .../connector/catalog/procedures/Procedure.java    |    40 +
 .../catalog/procedures/ProcedureParameter.java     |   142 +
 .../catalog/procedures/SimpleProcedure.java        |    42 +
 .../catalog/procedures/UnboundProcedure.java       |    49 +
 .../catalog/transactions/Transaction.java          |    80 +
 .../catalog/transactions/TransactionInfo.java      |    33 +
 .../distributions/ClusteredDistribution.java       |     4 +-
 .../sql/connector/distributions/Distribution.java  |     4 +-
 .../sql/connector/distributions/Distributions.java |     4 +-
 .../distributions/OrderedDistribution.java         |     4 +-
 .../distributions/UnspecifiedDistribution.java     |     4 +-
 .../spark/sql/connector/expressions/Cast.java      |    77 +
 .../sql/connector/expressions/Expression.java      |    33 +-
 .../sql/connector/expressions/Expressions.java     |     4 +-
 .../spark/sql/connector/expressions/Extract.java   |    61 +
 .../expressions/GeneralScalarExpression.java       |   468 +
 .../sql/connector/expressions/GetArrayItem.java    |    55 +
 .../spark/sql/connector/expressions/Literal.java   |     3 +
 .../sql/connector/expressions/NamedReference.java  |     6 +
 .../sql/connector/expressions/NullOrdering.java    |    16 +-
 .../expressions/PartitionFieldReference.java       |    39 +
 .../sql/connector/expressions/SortDirection.java   |    16 +-
 .../spark/sql/connector/expressions/SortOrder.java |     7 +-
 .../spark/sql/connector/expressions/Transform.java |     8 +-
 .../expressions/UserDefinedScalarFunc.java         |    68 +
 .../expressions/aggregate/Aggregation.java         |    17 +-
 .../sql/connector/expressions/aggregate/Avg.java   |    44 +
 .../sql/connector/expressions/aggregate/Count.java |    24 +-
 .../connector/expressions/aggregate/CountStar.java |     9 +-
 .../aggregate/GeneralAggregateFunc.java            |   103 +
 .../sql/connector/expressions/aggregate/Max.java   |    16 +-
 .../sql/connector/expressions/aggregate/Min.java   |    16 +-
 .../sql/connector/expressions/aggregate/Sum.java   |    24 +-
 .../aggregate/UserDefinedAggregateFunc.java        |    77 +
 .../connector/expressions/filter/AlwaysFalse.java  |    49 +
 .../connector/expressions/filter/AlwaysTrue.java   |    49 +
 .../sql/connector/expressions/filter/And.java      |    37 +
 .../sql/connector/expressions/filter/Not.java      |    35 +
 .../spark/sql/connector/expressions/filter/Or.java |    37 +
 .../expressions/filter/PartitionPredicate.java     |    90 +
 .../connector/expressions/filter/Predicate.java    |   158 +
 .../apache/spark/sql/connector/join/JoinType.java  |    32 +
 .../sql/connector/metric/CustomAvgMetric.java      |     7 +-
 .../spark/sql/connector/metric/CustomMetric.java   |     4 +
 .../sql/connector/metric/CustomSumMetric.java      |     8 +-
 .../sql/connector/metric/CustomTaskMetric.java     |    32 +
 .../spark/sql/connector/read/HasPartitionKey.java  |    53 +
 .../sql/connector/read/HasPartitionStatistics.java |    47 +
 .../apache/spark/sql/connector/read/LocalScan.java |     4 +-
 .../spark/sql/connector/read/PartitionReader.java  |    15 +
 .../sql/connector/read/PartitionReaderFactory.java |     5 +-
 .../org/apache/spark/sql/connector/read/Scan.java  |    48 +-
 .../spark/sql/connector/read/ScanBuilder.java      |     7 +-
 .../spark/sql/connector/read/Statistics.java       |     7 +
 .../connector/read/SupportsPushDownAggregates.java |    28 +-
 .../sql/connector/read/SupportsPushDownJoin.java   |    78 +
 .../sql/connector/read/SupportsPushDownLimit.java  |    42 +
 .../sql/connector/read/SupportsPushDownOffset.java |    36 +
 .../read/SupportsPushDownTableSample.java          |    39 +
 .../sql/connector/read/SupportsPushDownTopN.java   |    43 +
 .../connector/read/SupportsPushDownV2Filters.java  |   102 +
 .../read/SupportsPushDownVariantExtractions.java   |    63 +
 .../sql/connector/read/SupportsReportOrdering.java |    40 +
 .../connector/read/SupportsRuntimeFiltering.java   |    17 +-
 .../connector/read/SupportsRuntimeV2Filtering.java |   113 +
 .../sql/connector/read/VariantExtraction.java      |    64 +
 .../connector/read/colstats/ColumnStatistics.java  |    83 +
 .../sql/connector/read/colstats/Histogram.java     |    40 +
 .../sql/connector/read/colstats/HistogramBin.java  |    43 +
 .../read/partitioning/ClusteredDistribution.java   |    41 -
 .../connector/read/partitioning/Distribution.java  |    44 -
 .../read/partitioning/KeyGroupedPartitioning.java  |    55 +
 .../connector/read/partitioning/Partitioning.java  |    26 +-
 .../read/partitioning/UnknownPartitioning.java     |    39 +
 .../read/streaming/AcceptsLatestSeenOffset.java    |    39 +
 .../ContinuousPartitionReaderFactory.java          |     6 +-
 .../spark/sql/connector/read/streaming/Offset.java |     4 +-
 .../sql/connector/read/streaming/ReadLimit.java    |     2 +
 .../sql/connector/read/streaming/ReadMaxBytes.java |    56 +
 .../read/streaming/ReportsSinkMetrics.java         |    36 +
 .../read/streaming/SupportsRealTimeMode.java       |    47 +
 .../read/streaming/SupportsRealTimeRead.java       |    89 +
 .../streaming/SupportsTriggerAvailableNow.java     |    48 +
 .../sql/connector/util/V2ExpressionSQLBuilder.java |   416 +
 .../spark/sql/connector/write/BatchWrite.java      |    29 +
 .../spark/sql/connector/write/DataWriter.java      |    35 +
 .../spark/sql/connector/write/DeleteSummary.java   |    39 +
 .../spark/sql/connector/write/DeltaBatchWrite.java |    31 +
 .../spark/sql/connector/write/DeltaWrite.java      |    37 +
 .../sql/connector/write/DeltaWriteBuilder.java     |    37 +
 .../spark/sql/connector/write/DeltaWriter.java     |    78 +
 .../sql/connector/write/DeltaWriterFactory.java    |    34 +
 .../sql/connector/write/LogicalWriteInfo.java      |    20 +
 .../spark/sql/connector/write/MergeSummary.java    |    72 +
 .../write/RequiresDistributionAndOrdering.java     |    49 +-
 .../sql/connector/write/RowLevelOperation.java     |   108 +
 .../connector/write/RowLevelOperationBuilder.java  |    34 +
 .../sql/connector/write/RowLevelOperationInfo.java |    40 +
 .../spark/sql/connector/write/SupportsDelta.java   |    48 +
 .../connector/write/SupportsDynamicOverwrite.java  |     2 +-
 .../sql/connector/write/SupportsOverwrite.java     |    31 +-
 .../sql/connector/write/SupportsOverwriteV2.java   |    66 +
 .../spark/sql/connector/write/UpdateSummary.java   |    39 +
 .../apache/spark/sql/connector/write/Write.java    |    20 +-
 .../spark/sql/connector/write/WriteBuilder.java    |    15 +-
 .../spark/sql/connector/write/WriteSummary.java    |    29 +
 .../connector/write/streaming/StreamingWrite.java  |    16 +
 .../spark/sql/streaming/GroupStateTimeout.java     |    66 -
 .../java/org/apache/spark/sql/types/DataTypes.java |   257 -
 .../spark/sql/util/CaseInsensitiveStringMap.java   |    31 +-
 .../apache/spark/sql/util/NumericHistogram.java    |   292 +
 .../spark/sql/vectorized/ArrowColumnVector.java    |   319 +-
 .../apache/spark/sql/vectorized/ColumnVector.java  |    59 +-
 .../apache/spark/sql/vectorized/ColumnarArray.java |    66 +-
 .../apache/spark/sql/vectorized/ColumnarBatch.java |   197 +-
 .../spark/sql/vectorized/ColumnarBatchRow.java     |   233 +
 .../apache/spark/sql/vectorized/ColumnarRow.java   |    82 +-
 .../resources/gregorian-julian-rebase-micros.json  |  2401 --
 .../resources/julian-gregorian-rebase-micros.json  |  2401 --
 .../sql/catalyst/expressions/AttributeMap.scala    |    55 -
 .../sql/catalyst/util/CaseInsensitiveMap.scala     |    65 -
 .../sql/catalyst/expressions/AttributeMap.scala    |    56 -
 .../sql/catalyst/util/CaseInsensitiveMap.scala     |    67 -
 .../org/apache/spark/sql/AnalysisException.scala   |    96 -
 .../main/scala/org/apache/spark/sql/Encoder.scala  |    78 -
 .../main/scala/org/apache/spark/sql/Encoders.scala |   352 -
 .../src/main/scala/org/apache/spark/sql/Row.scala  |   615 -
 .../apache/spark/sql/catalyst/CapturesConfig.scala |   100 +
 .../sql/catalyst/CatalystTypeConverters.scala      |   226 +-
 .../spark/sql/catalyst/CurrentUserContext.scala    |     6 +
 .../spark/sql/catalyst/DataSourceOptions.scala     |   114 +
 .../sql/catalyst/DeserializerBuildHelper.scala     |   359 +-
 .../sql/catalyst/ExtendedAnalysisException.scala   |    93 +
 .../spark/sql/catalyst/FileSourceOptions.scala     |    53 +
 .../apache/spark/sql/catalyst/InternalRow.scala    |    48 +-
 .../spark/sql/catalyst/JavaTypeInference.scala     |   481 -
 .../apache/spark/sql/catalyst/OrderedFilters.scala |     6 +-
 .../spark/sql/catalyst/ProjectingInternalRow.scala |   133 +
 .../spark/sql/catalyst/QueryPlanningTracker.scala  |    75 +-
 .../apache/spark/sql/catalyst/SQLConfHelper.scala  |    29 +
 .../spark/sql/catalyst/ScalaReflection.scala       |  1035 -
 .../spark/sql/catalyst/SerializerBuildHelper.scala |   340 +-
 .../sql/catalyst/SqlScriptingContextManager.scala  |    26 +
 .../apache/spark/sql/catalyst/StructFilters.scala  |     7 +-
 .../apache/spark/sql/catalyst/WalkedTypePath.scala |    57 -
 .../sql/catalyst/analysis/AliasResolution.scala    |    82 +
 .../catalyst/analysis/AlreadyExistException.scala  |    99 +-
 .../spark/sql/catalyst/analysis/Analyzer.scala     |  4661 +--
 .../AnsiGetDateFieldOperationsTypeCoercion.scala   |    34 +
 .../analysis/AnsiStringPromotionTypeCoercion.scala |   118 +
 .../sql/catalyst/analysis/AnsiTypeCoercion.scala   |   237 +-
 .../analysis/ApplyCharTypePaddingHelper.scala      |   206 +
 .../catalyst/analysis/ApplyDefaultCollation.scala  |   452 +
 .../sql/catalyst/analysis/ApplyLimitAll.scala      |    52 +
 .../sql/catalyst/analysis/AssignmentUtils.scala    |   394 +
 .../BinaryArithmeticWithDatetimeResolver.scala     |   165 +
 .../analysis/BooleanEqualityTypeCoercion.scala     |    87 +
 .../sql/catalyst/analysis/CTESubstitution.scala    |   368 +-
 .../CannotReplaceMissingTableException.scala       |    29 -
 .../analysis/CdcNetChangesStatefulProcessor.scala  |   200 +
 .../sql/catalyst/analysis/ChangelogInfoUtils.scala |   116 +
 .../sql/catalyst/analysis/CheckAnalysis.scala      |  1664 +-
 .../catalyst/analysis/CollationRulesRunner.scala   |    29 +
 .../sql/catalyst/analysis/CollationTypeCasts.scala |    34 +
 .../catalyst/analysis/CollationTypeCoercion.scala  |   560 +
 .../catalyst/analysis/ColumnResolutionHelper.scala |   699 +
 .../sql/catalyst/analysis/DecimalPrecision.scala   |   293 +-
 .../analysis/DecimalPrecisionTypeCoercion.scala    |   208 +
 .../catalyst/analysis/DeduplicateRelations.scala   |   447 +-
 .../analysis/DeduplicateUnionChildOutput.scala     |    82 +
 .../catalyst/analysis/DivisionTypeCoercion.scala   |    41 +
 .../analysis/EliminateLazyExpression.scala         |    34 +
 .../analysis/ExtractDistributedSequenceID.scala    |    47 +
 .../sql/catalyst/analysis/FunctionRegistry.scala   |   720 +-
 .../sql/catalyst/analysis/FunctionResolution.scala |   774 +
 .../catalyst/analysis/GeneratorResolution.scala    |    51 +
 .../analysis/GroupingAnalyticsTransformer.scala    |   278 +
 .../sql/catalyst/analysis/HintErrorLogger.scala    |    20 +-
 .../catalyst/analysis/IdentifierResolution.scala   |    51 +
 .../analysis/IntegralDivisionTypeCoercion.scala    |    38 +
 .../sql/catalyst/analysis/KeepLegacyOutputs.scala  |    48 +
 .../spark/sql/catalyst/analysis/LambdaBinder.scala |    90 +
 .../analysis/LiteralFunctionResolution.scala       |    52 +
 .../catalyst/analysis/NameStreamingSources.scala   |   142 +
 .../catalyst/analysis/NamedStreamingRelation.scala |   103 +
 .../analysis/NaturalAndUsingJoinResolution.scala   |   195 +
 .../catalyst/analysis/NoSuchItemException.scala    |   121 +-
 .../NondeterministicExpressionCollection.scala     |    48 +
 .../catalyst/analysis/NormalizeableRelation.scala  |    32 +
 .../ParameterizedQueryArgumentsValidator.scala     |    59 +
 .../sql/catalyst/analysis/PivotTransformer.scala   |   223 +
 .../spark/sql/catalyst/analysis/PlanToString.scala |    36 +
 .../analysis/PullOutNondeterministic.scala         |    49 +-
 .../sql/catalyst/analysis/RelationCache.scala      |    28 +
 .../sql/catalyst/analysis/RelationChanges.scala    |    42 +
 .../sql/catalyst/analysis/RelationResolution.scala |   543 +
 .../sql/catalyst/analysis/RelationTimeTravel.scala |    34 +
 .../sql/catalyst/analysis/ResolveCatalogs.scala    |   282 +-
 .../catalyst/analysis/ResolveChangelogTable.scala  |   975 +
 .../catalyst/analysis/ResolveCollationName.scala   |    36 +
 .../ResolveColumnDefaultInCommandInputQuery.scala  |   177 +
 .../analysis/ResolveCommandsWithIfExists.scala     |     6 +-
 .../sql/catalyst/analysis/ResolveCursors.scala     |   109 +
 .../analysis/ResolveDataFrameDropColumns.scala     |    56 +
 .../analysis/ResolveEventTimeWatermark.scala       |    55 +
 .../sql/catalyst/analysis/ResolveFetchCursor.scala |    93 +
 .../spark/sql/catalyst/analysis/ResolveHints.scala |   118 +-
 .../analysis/ResolveIdentifierClause.scala         |   107 +
 .../catalyst/analysis/ResolveInlineTables.scala    |   123 +-
 .../catalyst/analysis/ResolveInsertionBase.scala   |    77 +
 .../ResolveLateralColumnAliasReference.scala       |   311 +
 .../catalyst/analysis/ResolvePartitionSpec.scala   |    13 +-
 .../analysis/ResolveReferencesInAggregate.scala    |   203 +
 .../analysis/ResolveReferencesInSort.scala         |   116 +
 .../analysis/ResolveReferencesInUpdate.scala       |    74 +
 .../ResolveRowLevelCommandAssignments.scala        |   146 +
 .../catalyst/analysis/ResolveSchemaEvolution.scala |   293 +
 .../sql/catalyst/analysis/ResolveSetVariable.scala |    96 +
 .../analysis/ResolveTableConstraints.scala         |    89 +
 .../sql/catalyst/analysis/ResolveTableSpec.scala   |   115 +
 .../analysis/ResolveTableValuedFunctions.scala     |    58 -
 .../sql/catalyst/analysis/ResolveTimeWindows.scala |   363 +
 .../spark/sql/catalyst/analysis/ResolveUnion.scala |    78 +-
 .../ResolveUpdateEventTimeWatermarkColumn.scala    |    52 +
 .../sql/catalyst/analysis/ResolveWithCTE.scala     |   319 +-
 .../catalyst/analysis/RewriteCollationJoin.scala   |    99 +
 .../catalyst/analysis/RewriteDeleteFromTable.scala |   120 +
 .../catalyst/analysis/RewriteMergeIntoTable.scala  |   524 +
 .../catalyst/analysis/RewriteRowLevelCommand.scala |   323 +
 .../sql/catalyst/analysis/RewriteUpdateTable.scala |   229 +
 .../catalyst/analysis/SQLFunctionExpression.scala  |    97 +
 .../sql/catalyst/analysis/SQLFunctionNode.scala    |    64 +
 .../analysis/SequentialUnionAnalysis.scala         |    77 +
 .../sql/catalyst/analysis/StackTypeCoercion.scala  |    40 +
 .../catalyst/analysis/StreamingJoinHelper.scala    |    41 +-
 .../analysis/StringLiteralTypeCoercion.scala       |    48 +
 .../analysis/StringPromotionTypeCoercion.scala     |    83 +
 ...ionInLambdaOrHigherOrderFunctionValidator.scala |    43 +
 .../analysis/SubstituteUnresolvedOrdinals.scala    |    64 -
 .../catalyst/analysis/TableOutputResolver.scala    |   695 +-
 .../sql/catalyst/analysis/TimeTravelSpec.scala     |   122 +
 .../sql/catalyst/analysis/TypeCheckResult.scala    |    26 +
 .../spark/sql/catalyst/analysis/TypeCoercion.scala |   964 +-
 .../sql/catalyst/analysis/TypeCoercionBase.scala   |   469 +
 .../sql/catalyst/analysis/TypeCoercionHelper.scala |   734 +
 .../catalyst/analysis/TypeCoercionValidation.scala |   127 +
 .../sql/catalyst/analysis/UnpivotTransformer.scala |    76 +
 .../catalyst/analysis/UnpivotTypeCoercion.scala    |    52 +
 .../analysis/UnresolveRelationsInTransaction.scala |    68 +
 .../analysis/UnsupportedOperationChecker.scala     |   351 +-
 .../sql/catalyst/analysis/UpCastResolution.scala   |    69 +
 .../sql/catalyst/analysis/V2TableReference.scala   |   188 +
 .../analysis/ValidateSubqueryExpression.scala      |   557 +
 .../sql/catalyst/analysis/VariableResolution.scala |   204 +
 .../sql/catalyst/analysis/ViewResolution.scala     |    63 +
 .../sql/catalyst/analysis/WindowResolution.scala   |   174 +
 .../catalyst/analysis/higherOrderFunctions.scala   |   104 +-
 .../spark/sql/catalyst/analysis/package.scala      |    71 +-
 .../spark/sql/catalyst/analysis/parameters.scala   |   279 +
 .../resolver/AggregateExpressionResolver.scala     |   329 +
 .../resolver/AggregateResolutionResult.scala       |    34 +
 .../analysis/resolver/AggregateResolver.scala      |   338 +
 .../AggregateWithLcaResolutionResult.scala         |    36 +
 .../analysis/resolver/AggregationValidator.scala   |    44 +
 .../sql/catalyst/analysis/resolver/AliasKind.scala |    38 +
 .../catalyst/analysis/resolver/AliasResolver.scala |   171 +
 .../analysis/resolver/AnalyzerBridgeState.scala    |    72 +
 .../analysis/resolver/AttributeScopeStack.scala    |   124 +
 .../resolver/AutoGeneratedAliasProvider.scala      |    85 +
 .../resolver/BinaryArithmeticResolver.scala        |   162 +
 .../analysis/resolver/BridgedRelationId.scala      |    31 +
 .../resolver/BridgedRelationMetadataProvider.scala |   106 +
 .../resolver/CandidatesForResolution.scala         |    26 +
 .../analysis/resolver/CoercesExpressionTypes.scala |   189 +
 .../sql/catalyst/analysis/resolver/CteScope.scala  |   381 +
 .../resolver/DefaultCollationTypeCoercion.scala    |    98 +
 .../resolver/DelegatesResolutionToExtensions.scala |    72 +
 .../ExplicitlyUnsupportedResolverFeature.scala     |    56 +
 .../analysis/resolver/ExpressionIdAssigner.scala   |   941 +
 .../resolver/ExpressionResolutionContext.scala     |   174 +
 .../resolver/ExpressionResolutionValidator.scala   |   213 +
 .../analysis/resolver/ExpressionResolver.scala     |  1384 +
 .../resolver/ExpressionTreeTraversal.scala         |   128 +
 .../analysis/resolver/ExtractValueResolver.scala   |   106 +
 .../analysis/resolver/FilterResolver.scala         |    83 +
 .../analysis/resolver/FunctionResolver.scala       |   227 +
 .../analysis/resolver/FunctionResolverUtils.scala  |   157 +
 .../resolver/GroupingAnalyticsResolver.scala       |   204 +
 .../GroupingAndAggregateExpressionsExtractor.scala |   141 +
 .../analysis/resolver/HavingResolver.scala         |   385 +
 .../resolver/HigherOrderFunctionResolver.scala     |   172 +
 .../analysis/resolver/HybridAnalyzer.scala         |   388 +
 .../resolver/IdentifierAndCteSubstituor.scala      |   208 +
 .../IdentifierFromUnresolvedNodeExtractor.scala    |    44 +
 .../catalyst/analysis/resolver/IdentifierMap.scala |    28 +
 .../catalyst/analysis/resolver/JoinResolver.scala  |   426 +
 .../analysis/resolver/KeyTransformingMap.scala     |    67 +
 .../analysis/resolver/LambdaFunctionResolver.scala |   126 +
 .../LateralColumnAliasProhibitedRegistry.scala     |    49 +
 .../resolver/LateralColumnAliasRegistry.scala      |    50 +
 .../resolver/LateralColumnAliasRegistryImpl.scala  |   212 +
 .../resolver/LateralColumnAliasResolver.scala      |   386 +
 .../resolver/LimitLikeExpressionValidator.scala    |   151 +
 .../analysis/resolver/LogicalPlanDifference.scala  |   220 +
 .../analysis/resolver/MetadataResolver.scala       |   151 +
 .../resolver/NameParameterizedQueryResolver.scala  |   174 +
 .../resolver/NameResolutionParameters.scala        |    55 +
 .../sql/catalyst/analysis/resolver/NameScope.scala |  1496 +
 .../catalyst/analysis/resolver/NameTarget.scala    |   105 +
 .../resolver/NonDeterministicExpressionCheck.scala |    63 +
 .../resolver/OperatorResolutionContext.scala       |   149 +
 .../OperatorWithUncomparableTypeValidator.scala    |    79 +
 .../resolver/OrdinalReplacementExpressions.scala   |    48 +
 .../analysis/resolver/OrdinalResolver.scala        |   124 +
 .../catalyst/analysis/resolver/OutputType.scala    |    28 +
 .../catalyst/analysis/resolver/PivotResolver.scala |   170 +
 .../catalyst/analysis/resolver/PlanLogger.scala    |    94 +
 .../catalyst/analysis/resolver/PlanRewriter.scala  |    78 +
 .../resolver/ProducesUnresolvedSubtree.scala       |   126 +
 .../analysis/resolver/ProhibitedResolver.scala     |    35 +
 .../analysis/resolver/ProjectResolver.scala        |   177 +
 .../analysis/resolver/PruneMetadataColumns.scala   |   153 +
 .../analysis/resolver/RecursiveCteState.scala      |    46 +
 .../catalyst/analysis/resolver/RelationId.scala    |    33 +
 .../resolver/RelationMetadataProvider.scala        |    93 +
 .../resolver/RepartitionByExpressionResolver.scala |   131 +
 .../analysis/resolver/ResolutionCheckRunner.scala  |    69 +
 .../analysis/resolver/ResolutionValidator.scala    |   375 +
 .../resolver/ResolvedAggregateExpressions.scala    |    46 +
 .../analysis/resolver/ResolvedProjectList.scala    |    40 +
 .../resolver/ResolvedSubqueryExpressionPlan.scala  |    37 +
 .../sql/catalyst/analysis/resolver/Resolver.scala  |   898 +
 .../analysis/resolver/ResolverExtension.scala      |    41 +
 .../catalyst/analysis/resolver/ResolverGuard.scala |   762 +
 .../analysis/resolver/ResolverGuardResult.scala    |    26 +
 .../analysis/resolver/ResolverMetricTracker.scala  |    90 +
 .../analysis/resolver/ResolverRunner.scala         |   114 +
 .../analysis/resolver/ResolverRunnerResult.scala   |    39 +
 .../catalyst/analysis/resolver/ResolverTag.scala   |    50 +
 .../resolver/ResolvesExpressionChildren.scala      |   109 +
 .../resolver/ResolvesNameByHiddenOutput.scala      |   428 +
 .../resolver/RetainsOriginalJoinOutput.scala       |   156 +
 .../resolver/RewritesAliasesInTopLcaProject.scala  |   253 +
 .../analysis/resolver/SemanticComparator.scala     |    78 +
 .../resolver/SemiStructuredExtractResolver.scala   |    81 +
 .../resolver/SetOperationLikeResolver.scala        |   491 +
 .../catalyst/analysis/resolver/SortResolver.scala  |   437 +
 .../resolver/SubqueryExpressionResolver.scala      |   263 +
 .../catalyst/analysis/resolver/SubqueryScope.scala |   181 +
 .../resolver/TableValuedFunctionResolver.scala     |    80 +
 .../resolver/TimezoneAwareExpressionResolver.scala |   166 +
 .../analysis/resolver/TreeNodeResolver.scala       |    32 +
 .../analysis/resolver/TryExtractOrdinal.scala      |    37 +
 .../analysis/resolver/UnpivotResolver.scala        |   278 +
 .../resolver/UnresolvedCteRelationRef.scala        |    29 +
 ...UnsupportedExpressionInOperatorValidation.scala |    77 +
 .../analysis/resolver/ValidatesFilter.scala        |    78 +
 .../catalyst/analysis/resolver/ViewResolver.scala  |   202 +
 .../sql/catalyst/analysis/resolver/package.scala   |    26 +
 .../sql/catalyst/analysis/timeZoneAnalysis.scala   |     8 +-
 .../spark/sql/catalyst/analysis/unresolved.scala   |   856 +-
 .../sql/catalyst/analysis/v2ResolutionPlans.scala  |   225 +-
 .../apache/spark/sql/catalyst/analysis/view.scala  |    52 +-
 .../sql/catalyst/catalog/ExternalCatalog.scala     |    32 +-
 .../catalyst/catalog/ExternalCatalogUtils.scala    |   165 +-
 .../catalog/ExternalCatalogWithListener.scala      |    16 +
 .../catalog/FunctionExpressionBuilder.scala        |    32 +
 .../catalyst/catalog/GlobalTempViewManager.scala   |     4 +-
 .../sql/catalyst/catalog/InMemoryCatalog.scala     |    77 +-
 .../catalog/InvalidUDFClassException.scala         |    15 +-
 .../sql/catalyst/catalog/RoutineLanguage.scala     |    29 +
 .../spark/sql/catalyst/catalog/SQLFunction.scala   |   332 +
 .../sql/catalyst/catalog/SessionCatalog.scala      |  2024 +-
 .../spark/sql/catalyst/catalog/SqlPathFormat.scala |    90 +
 .../catalog/SqlScriptingContextManager.scala       |    33 +
 .../SqlScriptingExecutionContextExtension.scala    |    49 +
 .../sql/catalyst/catalog/UserDefinedFunction.scala |   188 +
 .../catalog/UserDefinedFunctionErrors.scala        |   124 +
 .../sql/catalyst/catalog/VariableManager.scala     |   170 +
 .../apache/spark/sql/catalyst/catalog/events.scala |    86 +
 .../sql/catalyst/catalog/functionResources.scala   |     3 +-
 .../spark/sql/catalyst/catalog/interface.scala     |   619 +-
 .../spark/sql/catalyst/csv/CSVExprUtils.scala      |    30 +-
 .../spark/sql/catalyst/csv/CSVHeaderChecker.scala  |    53 +-
 .../spark/sql/catalyst/csv/CSVInferSchema.scala    |   102 +-
 .../apache/spark/sql/catalyst/csv/CSVOptions.scala |   263 +-
 .../sql/catalyst/csv/UnivocityGenerator.scala      |   143 +-
 .../spark/sql/catalyst/csv/UnivocityParser.scala   |   327 +-
 .../apache/spark/sql/catalyst/dsl/package.scala    |   219 +-
 .../spark/sql/catalyst/encoders/EncoderUtils.scala |   180 +
 .../sql/catalyst/encoders/ExpressionEncoder.scala  |   181 +-
 .../encoders/KryoSerializationCodecImpl.scala      |    33 +
 .../spark/sql/catalyst/encoders/OuterScopes.scala  |   109 -
 .../spark/sql/catalyst/encoders/RowEncoder.scala   |   354 -
 .../spark/sql/catalyst/encoders/package.scala      |     3 +-
 .../sql/catalyst/expressions/AliasHelper.scala     |   146 +-
 .../expressions/AnalysisAwareExpression.scala      |    23 +
 .../expressions/ApplyFunctionExpression.scala      |     8 +-
 .../expressions/ApproxTopKExpressions.scala        |   118 +
 .../sql/catalyst/expressions/AttributeMap.scala    |    61 +
 .../sql/catalyst/expressions/AttributeSet.scala    |    24 +-
 .../spark/sql/catalyst/expressions/Between.scala   |    63 +
 .../expressions/BloomFilterMightContain.scala      |   123 +
 .../sql/catalyst/expressions/BoundAttribute.scala  |    18 +-
 .../expressions/CallMethodViaReflection.scala      |   119 +-
 .../sql/catalyst/expressions/Canonicalize.scala    |   121 -
 .../spark/sql/catalyst/expressions/Cast.scala      |  1999 +-
 .../CodeGeneratorWithInterpretedFallback.scala     |     8 +-
 .../sql/catalyst/expressions/CollationKey.scala    |   107 +
 .../sql/catalyst/expressions/CursorReference.scala |    82 +
 .../expressions/DirectShufflePartitionID.scala     |    46 +
 .../expressions/DistributedSequenceID.scala        |    47 +
 .../sql/catalyst/expressions/DynamicPruning.scala  |    33 +-
 .../expressions/EquivalentExpressions.scala        |   211 +-
 .../sql/catalyst/expressions/EvalHelper.scala      |    32 +
 .../spark/sql/catalyst/expressions/EvalMode.scala  |    43 +
 .../catalyst/expressions/ExpectsInputTypes.scala   |    21 +-
 .../spark/sql/catalyst/expressions/ExprUtils.scala |   155 +-
 .../sql/catalyst/expressions/Expression.scala      |   644 +-
 .../sql/catalyst/expressions/ExpressionSet.scala   |    99 +-
 .../expressions/ExpressionsEvaluator.scala         |    52 +
 .../FunctionTableSubqueryArgumentExpression.scala  |   187 +
 .../expressions/GroupingAnalyticsExtractor.scala   |    48 +
 .../expressions/InterpretedMutableProjection.scala |    22 +-
 .../expressions/InterpretedSafeProjection.scala    |    18 +-
 .../expressions/InterpretedUnsafeProjection.scala  |   211 +-
 .../spark/sql/catalyst/expressions/JoinedRow.scala |    11 +-
 .../expressions/MonotonicallyIncreasingID.scala    |    12 +-
 .../expressions/NamedArgumentExpression.scala      |    58 +
 .../catalyst/expressions/NumericEvalContext.scala  |    57 +
 .../catalyst/expressions/PartitionTransforms.scala |    31 +-
 .../sql/catalyst/expressions/Projection.scala      |    20 +-
 .../expressions/ProjectionOverSchema.scala         |    19 +-
 .../spark/sql/catalyst/expressions/PythonUDF.scala |   337 +-
 .../spark/sql/catalyst/expressions/ScalaUDF.scala  |    19 +-
 .../sql/catalyst/expressions/SchemaPruning.scala   |    47 +-
 .../sql/catalyst/expressions/SelectedField.scala   |    23 +-
 .../expressions/SemiStructuredExtract.scala        |    60 +
 .../sql/catalyst/expressions/SessionWindow.scala   |    36 +-
 .../spark/sql/catalyst/expressions/SortOrder.scala |    45 +-
 .../catalyst/expressions/SpecificInternalRow.scala |    14 +-
 .../expressions/SubExprEvaluationRuntime.scala     |     5 +-
 .../sql/catalyst/expressions/TimeWindow.scala      |    82 +-
 .../sql/catalyst/expressions/ToJavaArray.scala     |   105 +
 .../sql/catalyst/expressions/ToPrettyString.scala  |    78 +
 .../sql/catalyst/expressions/ToStringBase.scala    |   501 +
 .../catalyst/expressions/TransformExpression.scala |   138 +
 .../spark/sql/catalyst/expressions/TryCast.scala   |    96 -
 .../spark/sql/catalyst/expressions/TryEval.scala   |   220 +-
 .../spark/sql/catalyst/expressions/UnwrapUDT.scala |    55 +
 .../catalyst/expressions/V2ExpressionUtils.scala   |   345 +-
 .../catalyst/expressions/VariableReference.scala   |    63 +
 .../sql/catalyst/expressions/WindowTime.scala      |    65 +
 .../spark/sql/catalyst/expressions/With.scala      |   168 +
 .../catalyst/expressions/aggregate/AnyValue.scala  |    68 +
 .../ApproxCountDistinctForIntervals.scala          |    44 +-
 .../aggregate/ApproxTopKAggregates.scala           |   879 +
 .../aggregate/ApproximatePercentile.scala          |   105 +-
 .../catalyst/expressions/aggregate/Average.scala   |    82 +-
 .../aggregate/BloomFilterAggregate.scala           |   259 +
 .../expressions/aggregate/CentralMomentAgg.scala   |   132 +-
 .../sql/catalyst/expressions/aggregate/Corr.scala  |     3 +-
 .../sql/catalyst/expressions/aggregate/Count.scala |    14 +-
 .../catalyst/expressions/aggregate/CountIf.scala   |    36 +-
 .../expressions/aggregate/CountMinSketchAgg.scala  |   124 +-
 .../expressions/aggregate/Covariance.scala         |    36 +-
 .../sql/catalyst/expressions/aggregate/First.scala |     5 +-
 .../expressions/aggregate/HistogramNumeric.scala   |   268 +
 .../aggregate/HyperLogLogPlusPlus.scala            |     5 +-
 .../sql/catalyst/expressions/aggregate/Last.scala  |     3 -
 .../sql/catalyst/expressions/aggregate/Max.scala   |     2 +-
 .../expressions/aggregate/MaxByAndMinBy.scala      |    14 +-
 .../catalyst/expressions/aggregate/MaxMinByK.scala |   303 +
 .../expressions/aggregate/MaxMinByKHeap.scala      |   170 +
 .../catalyst/expressions/aggregate/Measure.scala   |    60 +
 .../sql/catalyst/expressions/aggregate/Min.scala   |     2 +-
 .../sql/catalyst/expressions/aggregate/Mode.scala  |   353 +
 .../expressions/aggregate/PandasAggregate.scala    |    35 +
 .../expressions/aggregate/Percentile.scala         |   314 -
 .../expressions/aggregate/PivotFirst.scala         |    22 +-
 .../catalyst/expressions/aggregate/Product.scala   |   118 +-
 .../sql/catalyst/expressions/aggregate/Sum.scala   |   193 +-
 .../aggregate/SupportsOrderingWithinGroup.scala    |    45 +
 .../expressions/aggregate/UnevaluableAggs.scala    |    83 -
 .../expressions/aggregate/V2Aggregator.scala       |    15 +-
 .../expressions/aggregate/boolAggregates.scala     |    66 +
 .../catalyst/expressions/aggregate/collect.scala   |   614 +-
 .../aggregate/datasketchesAggregates.scala         |   407 +
 .../expressions/aggregate/interfaces.scala         |    82 +-
 .../expressions/aggregate/kllAggregates.scala      |   856 +
 .../expressions/aggregate/linearRegression.scala   |   384 +
 .../expressions/aggregate/percentiles.scala        |   530 +
 .../aggregate/thetasketchesAggregates.scala        |   650 +
 .../aggregate/tupleIntersectionAgg.scala           |   353 +
 .../expressions/aggregate/tupleSketchAgg.scala     |   534 +
 .../expressions/aggregate/tupleSketchState.scala   |    55 +
 .../expressions/aggregate/tupleUnionAgg.scala      |   409 +
 .../sql/catalyst/expressions/arithmetic.scala      |   724 +-
 .../catalyst/expressions/avroSqlFunctions.scala    |   295 +
 .../catalyst/expressions/bitmapExpressions.scala   |   417 +
 .../catalyst/expressions/bitwiseExpressions.scala  |    69 +-
 .../expressions/codegen/CodeFormatter.scala        |     4 +-
 .../expressions/codegen/CodeGenerator.scala        |   323 +-
 .../expressions/codegen/GenerateOrdering.scala     |     6 +-
 .../codegen/GenerateSafeProjection.scala           |     2 +-
 .../catalyst/expressions/codegen/javaCode.scala    |    21 +-
 .../expressions/collationExpressions.scala         |   175 +
 .../expressions/collectionOperations.scala         |  2403 +-
 .../catalyst/expressions/complexTypeCreator.scala  |   167 +-
 .../expressions/complexTypeExtractors.scala        |   576 +-
 .../expressions/conditionalExpressions.scala       |   145 +-
 .../expressions/constraintExpressions.scala        |    24 +-
 .../sql/catalyst/expressions/constraints.scala     |   380 +
 .../expressions/csv/CsvExpressionEvalUtils.scala   |   123 +
 .../sql/catalyst/expressions/csvExpressions.scala  |   212 +-
 .../expressions/datasketchesExpressions.scala      |   133 +
 .../catalyst/expressions/datetimeExpressions.scala |  1804 +-
 .../catalyst/expressions/decimalExpressions.scala  |   172 +-
 .../sql/catalyst/expressions/generators.scala      |   393 +-
 .../spark/sql/catalyst/expressions/grouping.scala  |    48 +-
 .../spark/sql/catalyst/expressions/hash.scala      |   372 +-
 .../expressions/higherOrderFunctions.scala         |   257 +-
 .../sql/catalyst/expressions/inputFileBlock.scala  |     9 +-
 .../catalyst/expressions/intervalExpressions.scala |   326 +-
 .../expressions/json/JsonExpressionEvalUtils.scala |   575 +
 .../sql/catalyst/expressions/jsonExpressions.scala |   885 +-
 .../sql/catalyst/expressions/kllExpressions.scala  |   701 +
 .../spark/sql/catalyst/expressions/literals.scala  |   231 +-
 .../sql/catalyst/expressions/maskExpressions.scala |   353 +
 .../sql/catalyst/expressions/mathExpressions.scala |   845 +-
 .../spark/sql/catalyst/expressions/misc.scala      |   419 +-
 .../catalyst/expressions/namedExpressions.scala    |   491 +-
 .../sql/catalyst/expressions/nullExpressions.scala |   146 +-
 .../expressions/numberFormatExpressions.scala      |   341 +
 .../sql/catalyst/expressions/objects/objects.scala |   443 +-
 .../spark/sql/catalyst/expressions/ordering.scala  |    46 +-
 .../spark/sql/catalyst/expressions/package.scala   |   103 +-
 .../sql/catalyst/expressions/pipeOperators.scala   |   138 +
 .../sql/catalyst/expressions/predicates.scala      |   568 +-
 .../catalyst/expressions/randomExpressions.scala   |   296 +-
 .../catalyst/expressions/regexpExpressions.scala   |   643 +-
 .../spark/sql/catalyst/expressions/rows.scala      |    38 +-
 .../expressions/st/STExpressionUtils.scala         |    85 +
 .../catalyst/expressions/st/stExpressions.scala    |   312 +
 .../catalyst/expressions/stringExpressions.scala   |  2305 +-
 .../spark/sql/catalyst/expressions/subquery.scala  |   297 +-
 .../expressions/thetasketchesExpressions.scala     |   216 +
 .../sql/catalyst/expressions/timeExpressions.scala |   980 +
 .../expressions/toFromProtobufSqlFunctions.scala   |   320 +
 .../sql/catalyst/expressions/tupleDifference.scala |   200 +
 .../catalyst/expressions/tupleIntersection.scala   |   277 +
 .../catalyst/expressions/tupleSketchEstimate.scala |   170 +
 .../catalyst/expressions/tupleSketchSummary.scala  |   157 +
 .../sql/catalyst/expressions/tupleUnion.scala      |   497 +
 .../expressions/url/UrlExpressionEvalUtils.scala   |   148 +
 .../sql/catalyst/expressions/urlExpressions.scala  |   273 +
 .../variant/VariantExpressionEvalUtils.scala       |   156 +
 .../expressions/variant/variantExpressions.scala   |   986 +
 .../catalyst/expressions/vectorExpressions.scala   |   704 +
 .../catalyst/expressions/windowExpressions.scala   |   306 +-
 .../expressions/xml/XmlExpressionEvalUtils.scala   |   211 +
 .../spark/sql/catalyst/expressions/xml/xpath.scala |   125 +-
 .../sql/catalyst/expressions/xmlExpressions.scala  |   285 +
 .../apache/spark/sql/catalyst/identifiers.scala    |    83 +-
 .../sql/catalyst/json/CreateJacksonParser.scala    |    14 +-
 .../spark/sql/catalyst/json/JSONOptions.scala      |   211 +-
 .../spark/sql/catalyst/json/JacksonGenerator.scala |    64 +-
 .../spark/sql/catalyst/json/JacksonParser.scala    |   308 +-
 .../spark/sql/catalyst/json/JacksonUtils.scala     |    28 +-
 .../spark/sql/catalyst/json/JsonFilters.scala      |     5 +-
 .../spark/sql/catalyst/json/JsonInferSchema.scala  |   130 +-
 .../sql/catalyst/normalizer/NormalizeCTEIds.scala  |    67 +
 .../sql/catalyst/optimizer/ComplexTypes.scala      |     2 +-
 .../catalyst/optimizer/CostBasedJoinReorder.scala  |     2 +-
 .../catalyst/optimizer/DecorrelateInnerQuery.scala |   544 +-
 .../catalyst/optimizer/EliminateResolvedHint.scala |    22 +-
 .../optimizer/EliminateSQLFunctionNode.scala       |    47 +
 .../optimizer/EliminateWindowPartitions.scala      |    41 +
 .../catalyst/optimizer/InferWindowGroupLimit.scala |   138 +
 .../catalyst/optimizer/InjectRuntimeFilter.scala   |   339 +
 .../spark/sql/catalyst/optimizer/InlineCTE.scala   |   246 +-
 .../optimizer/InsertMapSortExpression.scala        |   160 +
 .../optimizer/LimitPushDownThroughWindow.scala     |    11 +-
 .../sql/catalyst/optimizer/MergeSubplans.scala     |   379 +
 .../catalyst/optimizer/NestedColumnAliasing.scala  |   142 +-
 .../optimizer/NormalizeFloatingNumbers.scala       |    34 +-
 .../catalyst/optimizer/OptimizeCsvJsonExprs.scala  |    11 +-
 .../sql/catalyst/optimizer/OptimizeExpand.scala    |   127 +
 .../catalyst/optimizer/OptimizeJoinCondition.scala |    45 +
 .../catalyst/optimizer/OptimizeOneRowPlan.scala    |    68 +
 .../sql/catalyst/optimizer/OptimizeRand.scala      |    63 +
 .../spark/sql/catalyst/optimizer/Optimizer.scala   |  1555 +-
 .../spark/sql/catalyst/optimizer/PlanMerger.scala  |   647 +
 .../optimizer/PropagateEmptyRelation.scala         |   161 +-
 .../optimizer/PullOutGroupingExpressions.scala     |     6 +-
 .../PullOutNestedDataOuterRefExpressions.scala     |   136 +
 .../optimizer/PushDownJoinThroughUnion.scala       |   103 +
 .../optimizer/PushDownLeftSemiAntiJoin.scala       |    28 +-
 .../optimizer/PushExtraPredicateThroughJoin.scala  |     3 +-
 .../PushProjectionThroughLimitAndOffset.scala      |    43 +
 ...ushdownPredicatesAndPruneColumnsForCTEDef.scala |   176 +
 .../optimizer/RemoveRedundantAggregates.scala      |    41 +-
 .../catalyst/optimizer/RemoveRedundantSorts.scala  |    62 +
 .../optimizer/ReplaceCTERefWithRepartition.scala   |    95 +
 .../optimizer/ReplaceExceptWithFilter.scala        |     7 +-
 .../ReplaceNullWithFalseInPredicate.scala          |    37 +-
 .../sql/catalyst/optimizer/RewriteAsOfJoin.scala   |    90 +
 .../optimizer/RewriteDistinctAggregates.scala      |    41 +-
 .../catalyst/optimizer/RewriteNearestByJoin.scala  |   154 +
 .../catalyst/optimizer/RewriteWithExpression.scala |   201 +
 .../SimplifyConditionalsInPredicate.scala          |    78 -
 .../catalyst/optimizer/StarSchemaDetection.scala   |    13 +-
 .../optimizer/UnwrapCastInBinaryComparison.scala   |   293 +-
 .../spark/sql/catalyst/optimizer/expressions.scala |   805 +-
 .../sql/catalyst/optimizer/finishAnalysis.scala    |   142 +-
 .../spark/sql/catalyst/optimizer/joins.scala       |   258 +-
 .../spark/sql/catalyst/optimizer/objects.scala     |    21 +-
 .../spark/sql/catalyst/optimizer/subquery.scala    |   640 +-
 .../sql/catalyst/parser/AbstractSqlParser.scala    |   123 +
 .../spark/sql/catalyst/parser/AstBuilder.scala     |  5131 ++-
 .../sql/catalyst/parser/CatalystSqlParser.scala    |    27 +
 .../catalyst/parser/LegacyTypeStringParser.scala   |    93 -
 .../sql/catalyst/parser/ParameterContext.scala     |    52 +
 .../sql/catalyst/parser/ParameterHandler.scala     |   256 +
 .../spark/sql/catalyst/parser/ParseDriver.scala    |   310 -
 .../sql/catalyst/parser/ParserInterface.scala      |    28 +-
 .../spark/sql/catalyst/parser/ParserUtils.scala    |   530 +-
 .../catalyst/parser/SubstituteParamsParser.scala   |   227 +
 .../apache/spark/sql/catalyst/parser/package.scala |    48 +
 .../spark/sql/catalyst/planning/patterns.scala     |   339 +-
 .../plans/AliasAwareOutputExpression.scala         |   133 +
 .../sql/catalyst/plans/DescribeCommandSchema.scala |    20 +-
 .../spark/sql/catalyst/plans/NormalizePlan.scala   |   314 +
 .../spark/sql/catalyst/plans/QueryPlan.scala       |   371 +-
 .../sql/catalyst/plans/ReferenceAllColumns.scala   |    34 +
 .../spark/sql/catalyst/plans/joinTypes.scala       |   146 +-
 .../catalyst/plans/logical/AnalysisHelper.scala    |    97 +-
 .../catalyst/plans/logical/ColumnDefinition.scala  |   277 +
 .../spark/sql/catalyst/plans/logical/Command.scala |    26 +-
 .../plans/logical/DistinctKeyVisitor.scala         |   160 +
 .../sql/catalyst/plans/logical/EmptyRelation.scala |    35 +
 .../plans/logical/EventTimeWatermark.scala         |    85 +-
 .../plans/logical/ExecutableDuringAnalysis.scala   |    28 +
 .../plans/logical/FunctionBuilderBase.scala        |   234 +
 .../catalyst/plans/logical/IgnoreCachedData.scala  |    23 -
 .../sql/catalyst/plans/logical/LocalRelation.scala |    42 +-
 .../sql/catalyst/plans/logical/LogicalPlan.scala   |   333 +-
 .../plans/logical/LogicalPlanDistinctKeys.scala    |    34 +
 .../plans/logical/LogicalPlanVisitor.scala         |     6 +
 .../sql/catalyst/plans/logical/MergeRows.scala     |   135 +
 .../sql/catalyst/plans/logical/MultiResult.scala   |    30 +
 .../sql/catalyst/plans/logical/NearestByJoin.scala |   102 +
 .../sql/catalyst/plans/logical/PlanHelper.scala    |     3 +-
 .../plans/logical/QueryPlanConstraints.scala       |    30 +-
 .../plans/logical/ScriptTransformation.scala       |    59 +-
 .../plans/logical/SequentialStreamingUnion.scala   |   138 +
 .../plans/logical/SqlScriptingLogicalPlans.scala   |   456 +
 .../sql/catalyst/plans/logical/Statistics.scala    |    12 +-
 .../logical/StreamSourceAwareLogicalPlan.scala     |    35 +
 .../plans/logical/basicLogicalOperators.scala      |  1421 +-
 .../sql/catalyst/plans/logical/cteOperators.scala  |   296 +
 .../spark/sql/catalyst/plans/logical/hints.scala   |    24 +-
 .../spark/sql/catalyst/plans/logical/object.scala  |   281 +-
 .../plans/logical/pythonLogicalOperators.scala     |   327 +-
 .../sql/catalyst/plans/logical/statements.scala    |   201 +-
 .../statsEstimation/AggregateEstimation.scala      |     3 +-
 .../statsEstimation/BasicStatsPlanVisitor.scala    |     8 +-
 .../logical/statsEstimation/EstimationUtils.scala  |    18 +-
 .../logical/statsEstimation/FilterEstimation.scala |    88 +-
 .../logical/statsEstimation/JoinEstimation.scala   |    39 +-
 .../statsEstimation/ProjectEstimation.scala        |     3 +-
 .../SizeInBytesOnlyStatsPlanVisitor.scala          |    28 +-
 .../logical/statsEstimation/UnionEstimation.scala  |   167 +-
 .../plans/logical/v2AlterTableCommands.scala       |   207 +-
 .../sql/catalyst/plans/logical/v2Commands.scala    |  1514 +-
 .../sql/catalyst/plans/physical/partitioning.scala |   864 +-
 .../catalyst/rules/QueryExecutionMetering.scala    |    10 +-
 .../spark/sql/catalyst/rules/RuleExecutor.scala    |   173 +-
 .../sql/catalyst/rules/RuleIdCollection.scala      |    66 +-
 .../catalyst/streaming/InternalOutputModes.scala   |    64 -
 .../catalyst/streaming/StreamingRelationV2.scala   |    39 +-
 .../streaming/StreamingSourceIdentifyingName.scala |    94 +
 .../sql/catalyst/streaming/WriteToStream.scala     |     4 +-
 .../streaming/WriteToStreamStatement.scala         |    12 +-
 .../catalyst/transactions/TransactionUtils.scala   |    55 +
 .../apache/spark/sql/catalyst/trees/TreeNode.scala |   574 +-
 .../spark/sql/catalyst/trees/TreePatternBits.scala |    21 +-
 .../spark/sql/catalyst/trees/TreePatterns.scala    |   122 +-
 .../apache/spark/sql/catalyst/trees/package.scala  |     2 +-
 .../spark/sql/catalyst/types/DataTypeUtils.scala   |   334 +
 .../sql/catalyst/types/PhysicalDataType.scala      |   435 +
 .../spark/sql/catalyst/types/ops/TimeTypeOps.scala |   114 +
 .../spark/sql/catalyst/types/ops/TypeOps.scala     |   238 +
 .../sql/catalyst/util/ArrayBasedMapBuilder.scala   |    47 +-
 .../sql/catalyst/util/ArrayBasedMapData.scala      |    11 +-
 .../apache/spark/sql/catalyst/util/ArrayData.scala |     4 +-
 .../sql/catalyst/util/BadRecordException.scala     |   103 +-
 .../spark/sql/catalyst/util/CharVarcharUtils.scala |   198 +-
 .../spark/sql/catalyst/util/CharsetProvider.scala  |    82 +
 .../sql/catalyst/util/CompressionCodecs.scala      |    19 +-
 .../sql/catalyst/util/DataTypeJsonUtils.scala      |    55 -
 .../spark/sql/catalyst/util/DateFormatter.scala    |   224 -
 .../catalyst/util/DateTimeFormatterHelper.scala    |   347 -
 .../spark/sql/catalyst/util/DateTimeUtils.scala    |  1124 +-
 .../util/EvaluateUnresolvedInlineTable.scala       |   148 +
 .../sql/catalyst/util/FailureSafeParser.scala      |    28 +-
 .../spark/sql/catalyst/util/GeneratedColumn.scala  |   209 +
 .../spark/sql/catalyst/util/GenericArrayData.scala |    25 +-
 .../catalyst/util/HyperLogLogPlusPlusHelper.scala  |     8 +-
 .../spark/sql/catalyst/util/IdentityColumn.scala   |    78 +
 .../util/InternalRowComparableWrapper.scala        |   109 +
 .../sql/catalyst/util/IntervalMathUtils.scala      |    49 +
 .../spark/sql/catalyst/util/IntervalUtils.scala    |   817 +-
 .../spark/sql/catalyst/util/NumberConverter.scala  |    39 +-
 .../apache/spark/sql/catalyst/util/ParseMode.scala |    22 +-
 .../sql/catalyst/util/QuantileSummaries.scala      |     9 +-
 .../spark/sql/catalyst/util/RebaseDateTime.scala   |   493 -
 .../sql/catalyst/util/ReplaceDataProjections.scala |    24 +
 .../catalyst/util/ResolveDefaultColumnsUtil.scala  |   631 +
 .../catalyst/util/ResolveTableConstraints.scala    |    55 +
 .../spark/sql/catalyst/util/RowDeltaUtils.scala    |    31 +
 .../spark/sql/catalyst/util/SQLKeywordUtils.scala  |    53 +
 .../spark/sql/catalyst/util/StringKeyHashMap.scala |     2 +-
 .../spark/sql/catalyst/util/StringUtils.scala      |   359 +-
 .../spark/sql/catalyst/util/ThetaSketchUtils.scala |   112 +
 .../sql/catalyst/util/TimestampFormatter.scala     |   461 -
 .../spark/sql/catalyst/util/ToNumberParser.scala   |   884 +
 .../spark/sql/catalyst/util/TupleSketchUtils.scala |   357 +
 .../apache/spark/sql/catalyst/util/TypeUtils.scala |   109 +-
 .../spark/sql/catalyst/util/UDTUtilsImpl.scala     |    26 +
 .../spark/sql/catalyst/util/UTF8StringUtils.scala  |    54 +
 .../spark/sql/catalyst/util/UnsafeRowUtils.scala   |   162 +-
 .../sql/catalyst/util/V2ExpressionBuilder.scala    |   486 +
 .../sql/catalyst/util/WriteDeltaProjections.scala  |    25 +
 .../apache/spark/sql/catalyst/util/package.scala   |   247 +-
 .../catalyst/xml/IndentingXMLStreamWriter.scala    |   194 +
 .../sql/catalyst/xml/StaxXMLRecordReader.scala     |   132 +
 .../spark/sql/catalyst/xml/StaxXmlGenerator.scala  |   430 +
 .../spark/sql/catalyst/xml/StaxXmlParser.scala     |  1316 +
 .../sql/catalyst/xml/StaxXmlParserUtils.scala      |   283 +
 .../spark/sql/catalyst/xml/ValidatorUtil.scala     |    78 +
 .../spark/sql/catalyst/xml/XmlInferSchema.scala    |   716 +
 .../apache/spark/sql/catalyst/xml/XmlOptions.scala |   258 +
 .../sql/connector/catalog/CatalogManager.scala     |   373 +-
 .../catalog/CatalogNotFoundException.scala         |    10 +-
 .../sql/connector/catalog/CatalogV2Implicits.scala |   181 +-
 .../sql/connector/catalog/CatalogV2Util.scala      |   540 +-
 .../spark/sql/connector/catalog/Catalogs.scala     |    45 +-
 .../sql/connector/catalog/LookupCatalog.scala      |    67 +-
 .../catalog/TransactionAwareCatalogManager.scala   |    64 +
 .../spark/sql/connector/catalog/V1Table.scala      |   176 +-
 .../spark/sql/connector/catalog/V1ViewInfo.scala   |    73 +
 .../spark/sql/connector/catalog/V2TableUtil.scala  |   216 +
 .../catalog/transactions/TransactionInfoImpl.scala |    20 +
 .../connector/distributions/distributions.scala    |     5 +-
 .../sql/connector/expressions/expressions.scala    |   137 +-
 .../sql/connector/write/DeleteSummaryImpl.scala    |    27 +
 .../sql/connector/write/LogicalWriteInfoImpl.scala |    25 +-
 .../sql/connector/write/MergeSummaryImpl.scala     |    33 +
 .../write/RowLevelOperationInfoImpl.scala          |    25 +
 .../connector/write/RowLevelOperationTable.scala   |    52 +
 .../sql/connector/write/UpdateSummaryImpl.scala    |    27 +
 .../spark/sql/errors/QueryCompilationErrors.scala  |  4607 ++-
 .../apache/spark/sql/errors/QueryErrorsBase.scala  |    77 +
 .../spark/sql/errors/QueryExecutionErrors.scala    |  3707 ++-
 .../spark/sql/errors/QueryParsingErrors.scala      |   394 -
 .../spark/sql/errors/SqlScriptingErrors.scala      |   272 +
 .../sql/exceptions/SqlScriptingException.scala     |    53 +
 .../spark/sql/execution/arrow/ArrowWriter.scala    |   582 +
 .../sql/execution/arrow/ArrowWriterWrapper.scala   |    86 +
 .../execution/datasources/v2/ChangelogTable.scala  |   103 +
 .../datasources/v2/DataSourceV2Implicits.scala     |    38 +-
 .../datasources/v2/DataSourceV2Relation.scala      |   358 +-
 .../org/apache/spark/sql/internal/SQLConf.scala    |  5025 ++-
 .../apache/spark/sql/internal/StaticSQLConf.scala  |    46 +-
 .../spark/sql/internal/connector/ColumnImpl.scala  |    67 +
 .../connector/ExpressionWithToString.scala         |    25 +
 .../connector/PartitionFieldReferenceImpl.scala    |    31 +
 .../connector/PartitionOffsetWithIndex.scala       |    24 +
 .../connector/PartitionPredicateField.scala        |    33 +
 .../connector/PartitionPredicateImpl.scala         |   121 +
 .../sql/internal/connector/PredicateUtils.scala    |   141 +
 .../connector/ProcedureParameterImpl.scala         |    30 +
 .../internal/connector/SimpleTableProvider.scala   |     3 +-
 .../SupportsPushDownCatalystFilters.scala          |    42 +
 .../internal/connector/ToStringSQLBuilder.scala    |    43 +
 .../spark/sql/internal/connector/V1Function.scala  |    95 +
 .../internal/connector/VariantExtractionImpl.scala |    45 +
 .../sql/metricview/logical/metricViewNodes.scala   |    53 +
 .../spark/sql/metricview/serde/JsonUtils.scala     |    34 +
 .../sql/metricview/serde/MetricViewCanonical.scala |   208 +
 .../sql/metricview/serde/MetricViewFactory.scala   |    69 +
 .../sql/metricview/serde/MetricViewSerDeBase.scala |   189 +
 .../sql/metricview/serde/MetricViewSerDeV01.scala  |    83 +
 .../sql/metricview/util/MetricViewPlanner.scala    |    86 +
 .../org/apache/spark/sql/sources/filters.scala     |   159 +-
 .../apache/spark/sql/types/AbstractDataType.scala  |   229 -
 .../org/apache/spark/sql/types/ArrayType.scala     |   150 -
 .../org/apache/spark/sql/types/BinaryType.scala    |    54 -
 .../org/apache/spark/sql/types/BooleanType.scala   |    51 -
 .../org/apache/spark/sql/types/ByteType.scala      |    57 -
 .../spark/sql/types/CalendarIntervalType.scala     |    49 -
 .../org/apache/spark/sql/types/CharType.scala      |    38 -
 .../org/apache/spark/sql/types/DataType.scala      |   565 -
 .../spark/sql/types/DataTypeExpression.scala       |    90 +
 .../org/apache/spark/sql/types/DateType.scala      |    61 -
 .../spark/sql/types/DayTimeIntervalType.scala      |   113 -
 .../scala/org/apache/spark/sql/types/Decimal.scala |   694 -
 .../org/apache/spark/sql/types/DecimalType.scala   |   255 -
 .../org/apache/spark/sql/types/DoubleType.scala    |    89 -
 .../org/apache/spark/sql/types/FloatType.scala     |    93 -
 .../org/apache/spark/sql/types/IntegerType.scala   |    56 -
 .../org/apache/spark/sql/types/LongType.scala      |    56 -
 .../scala/org/apache/spark/sql/types/MapType.scala |   104 -
 .../org/apache/spark/sql/types/Metadata.scala      |   297 -
 .../org/apache/spark/sql/types/NullType.scala      |    43 -
 .../org/apache/spark/sql/types/ObjectType.scala    |    48 -
 .../org/apache/spark/sql/types/ShortType.scala     |    56 -
 .../org/apache/spark/sql/types/StringType.scala    |    53 -
 .../org/apache/spark/sql/types/StructField.scala   |   108 -
 .../org/apache/spark/sql/types/StructType.scala    |   676 -
 .../apache/spark/sql/types/TimestampNTZType.scala  |    65 -
 .../org/apache/spark/sql/types/TimestampType.scala |    63 -
 .../apache/spark/sql/types/UDTRegistration.scala   |    88 -
 .../apache/spark/sql/types/UserDefinedType.scala   |   145 -
 .../org/apache/spark/sql/types/VarcharType.scala   |    37 -
 .../spark/sql/types/YearMonthIntervalType.scala    |   107 -
 .../org/apache/spark/sql/types/numerics.scala      |    67 +-
 .../org/apache/spark/sql/util/ArrowUtils.scala     |   162 -
 .../apache/spark/sql/util/PartitioningUtils.scala  |    60 +-
 .../org/apache/spark/sql/util/SQLOpenHashSet.scala |   164 +
 .../org/apache/spark/sql/util/SchemaUtils.scala    |   317 +-
 .../spark/sql/catalyst/JavaTypeInferenceBeans.java |   138 +
 .../sql/catalyst/expressions/HiveHasherSuite.java  |    32 +-
 .../expressions/RowBasedKeyValueBatchSuite.java    |   192 +-
 .../expressions/TestThrowExceptionMethod.java      |    39 +
 .../spark/sql/catalyst/expressions/XXH64Suite.java |    86 +-
 .../catalyst/util/geo/WkbErrorHandlingTest.java    |   387 +
 .../sql/catalyst/util/geo/WkbGeographyTest.java    |   635 +
 .../sql/catalyst/util/geo/WkbLineStringTest.java   |   142 +
 .../catalyst/util/geo/WkbMultiGeometryTest.java    |   208 +
 .../spark/sql/catalyst/util/geo/WkbPointTest.java  |   190 +
 .../sql/catalyst/util/geo/WkbPolygonTest.java      |   175 +
 .../util/geo/WkbReaderWriterAdvancedTest.java      |  1247 +
 .../util/geo/WkbReaderWriterGeometryModelTest.java |   400 +
 .../spark/sql/catalyst/util/geo/WkbTestBase.java   |    54 +
 .../spark/sql/catalyst/util/geo/WktWriterTest.java |   871 +
 .../sql/connector/catalog/CatalogLoadingSuite.java |   166 +-
 .../sql/streaming/JavaGroupStateTimeoutSuite.java  |    11 +-
 .../spark/sql/streaming/JavaOutputModeSuite.java   |     8 +-
 .../resources/ansi-sql-2016-reserved-keywords.txt  |     2 +
 sql/catalyst/src/test/resources/log4j.properties   |    27 -
 sql/catalyst/src/test/resources/log4j2.properties  |    31 +
 .../catalyst/analysis/ExtractGeneratorSuite.scala  |    41 -
 .../spark/sql/CalendarIntervalBenchmark.scala      |    75 +
 .../scala/org/apache/spark/sql/HashBenchmark.scala |    11 +-
 .../apache/spark/sql/HashByteArrayBenchmark.scala  |     4 +-
 .../org/apache/spark/sql/RandomDataGenerator.scala |    74 +-
 .../spark/sql/RandomDataGeneratorSuite.scala       |    16 +
 .../scala/org/apache/spark/sql/RowJsonSuite.scala  |    27 +-
 .../test/scala/org/apache/spark/sql/RowTest.scala  |    26 +-
 .../spark/sql/SparkBloomFilterBenchmark.scala      |   318 +
 .../spark/sql/UnsafeProjectionBenchmark.scala      |    17 +-
 .../sql/catalyst/CatalystTypeConvertersSuite.scala |   331 +-
 .../spark/sql/catalyst/DistributionSuite.scala     |   243 +-
 .../spark/sql/catalyst/EscapePathBenchmark.scala   |   124 +
 .../sql/catalyst/JavaTypeInferenceSuite.scala      |   304 +
 .../sql/catalyst/QueryPlanningTrackerSuite.scala   |    25 +
 .../spark/sql/catalyst/SQLKeywordSuite.scala       |    35 +-
 .../spark/sql/catalyst/ScalaReflectionSuite.scala  |   107 +-
 .../spark/sql/catalyst/SchemaPruningTest.scala     |     4 +-
 .../spark/sql/catalyst/ShuffleSpecSuite.scala      |   545 +
 .../sql/catalyst/analysis/AnalysisErrorSuite.scala |  1139 +-
 .../analysis/AnalysisExceptionPositionSuite.scala  |    31 +-
 .../analysis/AnalysisExternalCatalogSuite.scala    |     4 +-
 .../sql/catalyst/analysis/AnalysisSuite.scala      |  1184 +-
 .../spark/sql/catalyst/analysis/AnalysisTest.scala |    74 +-
 .../AnalyzerExtensionPropagationSuite.scala        |   105 +
 .../catalyst/analysis/AnsiTypeCoercionSuite.scala  |   606 +-
 .../spark/sql/catalyst/analysis/CatalogSuite.scala |    17 +
 .../analysis/ChangelogInfoUtilsSuite.scala         |   197 +
 .../CreateTablePartitioningValidationSuite.scala   |    76 +-
 .../spark/sql/catalyst/analysis/DSLHintSuite.scala |    11 +-
 .../analysis/ExpressionTypeCheckingSuite.scala     |   892 +-
 .../catalyst/analysis/ExtractGeneratorSuite.scala  |     0
 .../GroupByOrdinalsRepeatedAnalysisSuite.scala     |    86 +
 .../catalyst/analysis/LookupFunctionsSuite.scala   |    99 +-
 .../analysis/NameStreamingSourcesSuite.scala       |   394 +
 .../analysis/NamedParameterFunctionSuite.scala     |   275 +
 .../analysis/NamedStreamingRelationSuite.scala     |   120 +
 .../analysis/PullOutNondeterministicSuite.scala    |     6 +-
 .../catalyst/analysis/ResolveAliasesSuite.scala    |   145 +
 .../analysis/ResolveEncodersInUDFSuite.scala       |    96 +
 .../analysis/ResolveEventTimeWatermarkSuite.scala  |   120 +
 .../analysis/ResolveGroupingAnalyticsSuite.scala   |   194 +-
 .../sql/catalyst/analysis/ResolveHintsSuite.scala  |    83 +-
 .../analysis/ResolveInlineTablesSuite.scala        |    79 +-
 .../analysis/ResolveLambdaVariablesSuite.scala     |    34 +-
 .../analysis/ResolveNaturalJoinSuite.scala         |    50 +-
 .../analysis/ResolveRecursiveCTESuite.scala        |   119 +
 .../catalyst/analysis/ResolveSubquerySuite.scala   |   135 +-
 .../sql/catalyst/analysis/ResolveUnionSuite.scala  |     2 +-
 .../analysis/ResolvedUuidExpressionsSuite.scala    |     8 +-
 .../SequentialStreamingUnionAnalysisSuite.scala    |   245 +
 .../analysis/StreamingJoinHelperSuite.scala        |     4 +-
 .../SubstituteUnresolvedOrdinalsSuite.scala        |    70 -
 .../catalyst/analysis/TableLookupCacheSuite.scala  |    31 +-
 .../sql/catalyst/analysis/TestRelations.scala      |    16 +
 .../sql/catalyst/analysis/TypeCoercionSuite.scala  |   640 +-
 .../analysis/UnsupportedOperationsSuite.scala      |   482 +-
 .../catalyst/analysis/V2WriteAnalysisSuite.scala   |  1028 +-
 .../resolver/CoercesExpressionTypesSuite.scala     |    87 +
 .../LimitLikeExpressionValidatorSuite.scala        |   109 +
 .../resolver/LogicalPlanDifferenceSuite.scala      |   478 +
 .../resolver/ResolutionValidatorSuite.scala        |   260 +
 .../TimezoneAwareExpressionResolverSuite.scala     |    96 +
 .../catalog/ExternalCatalogEventSuite.scala        |   114 +-
 .../catalyst/catalog/ExternalCatalogSuite.scala    |   210 +-
 .../catalog/ExternalCatalogUtilsSuite.scala        |    68 +
 .../sql/catalyst/catalog/SessionCatalogSuite.scala |  1249 +-
 .../spark/sql/catalyst/csv/CSVExprUtilsSuite.scala |    64 +-
 .../sql/catalyst/csv/CSVInferSchemaSuite.scala     |    85 +-
 .../sql/catalyst/csv/UnivocityParserSuite.scala    |    92 +-
 .../encoders/EncoderErrorMessageSuite.scala        |   105 +-
 .../catalyst/encoders/EncoderResolutionSuite.scala |   249 +-
 .../catalyst/encoders/ExpressionEncoderSuite.scala |   533 +-
 .../sql/catalyst/encoders/RowEncoderSuite.scala    |   196 +-
 .../catalyst/expressions/AnsiCastSuiteBase.scala   |   500 -
 .../expressions/ArithmeticExpressionSuite.scala    |   582 +-
 .../catalyst/expressions/AttributeMapSuite.scala   |   278 +
 .../expressions/AttributeResolutionSuite.scala     |    40 +-
 .../catalyst/expressions/BindReferencesSuite.scala |    43 +
 .../expressions/BitmapExpressionUtilsSuite.scala   |    95 +
 .../expressions/BitwiseExpressionsSuite.scala      |    93 +-
 .../expressions/CallMethodViaReflectionSuite.scala |    91 +-
 .../catalyst/expressions/CanonicalizeSuite.scala   |   351 +-
 .../spark/sql/catalyst/expressions/CastSuite.scala |   583 -
 .../sql/catalyst/expressions/CastSuiteBase.scala   |  1024 +-
 .../expressions/CastWithAnsiOffSuite.scala         |   933 +
 .../catalyst/expressions/CastWithAnsiOnSuite.scala |   903 +
 .../catalyst/expressions/CodeGenerationSuite.scala |    37 +-
 ...CodeGeneratorWithInterpretedFallbackSuite.scala |     2 +-
 .../expressions/CollationExpressionSuite.scala     |   267 +
 .../CollationRegexpExpressionsSuite.scala          |   216 +
 .../expressions/CollectionExpressionsSuite.scala   |  1198 +-
 .../catalyst/expressions/ComplexTypeSuite.scala    |   558 +-
 .../expressions/ConditionalExpressionSuite.scala   |    74 +-
 .../expressions/ConstraintExpressionSuite.scala    |    45 +
 .../catalyst/expressions/CsvExpressionsSuite.scala |    91 +-
 .../expressions/DateExpressionsSuite.scala         |   945 +-
 .../expressions/DecimalExpressionSuite.scala       |    27 +-
 .../expressions/DynamicPruningSubquerySuite.scala  |   114 +
 .../expressions/ExpressionEvalHelper.scala         |    92 +-
 .../expressions/ExpressionImplUtilsSuite.scala     |   424 +
 .../expressions/ExpressionSQLBuilderSuite.scala    |    69 +-
 .../catalyst/expressions/ExpressionSetSuite.scala  |    54 +-
 .../ExtractPredicatesWithinOutputSetSuite.scala    |     6 +-
 .../expressions/GeneratorExpressionSuite.scala     |    43 +-
 .../expressions/HashExpressionsSuite.scala         |   184 +-
 .../sql/catalyst/expressions/HexBenchmark.scala    |    90 +
 .../spark/sql/catalyst/expressions/HexSuite.scala  |    40 +
 .../expressions/HigherOrderFunctionsSuite.scala    |    63 +-
 .../expressions/IntervalExpressionsSuite.scala     |    38 +-
 .../expressions/JsonExpressionsSuite.scala         |   187 +-
 .../expressions/LiteralExpressionSuite.scala       |   250 +-
 .../catalyst/expressions/LiteralGenerator.scala    |    14 +-
 .../expressions/MathExpressionsSuite.scala         |   483 +-
 .../expressions/MiscExpressionsSuite.scala         |    37 +-
 .../expressions/MutableProjectionSuite.scala       |   102 +-
 .../expressions/NamedExpressionSuite.scala         |    15 +-
 .../expressions/NullExpressionsSuite.scala         |    34 +-
 .../expressions/ObjectExpressionsSuite.scala       |   232 +-
 .../sql/catalyst/expressions/OrderingSuite.scala   |    27 +-
 .../sql/catalyst/expressions/PredicateSuite.scala  |   168 +-
 .../sql/catalyst/expressions/RandomSuite.scala     |    23 +
 .../expressions/RegexpExpressionsSuite.scala       |   309 +-
 .../sql/catalyst/expressions/ScalaUDFSuite.scala   |    38 +-
 .../catalyst/expressions/SchemaPruningSuite.scala  |     4 +-
 .../catalyst/expressions/SelectedFieldSuite.scala  |    41 +-
 .../expressions/SortOrderExpressionsSuite.scala    |    18 +
 .../expressions/StringExpressionsSuite.scala       |  1251 +-
 .../SubExprEvaluationRuntimeSuite.scala            |    12 +
 .../SubexpressionEliminationSuite.scala            |   137 +-
 .../expressions/TimeExpressionsSuite.scala         |   698 +
 .../sql/catalyst/expressions/TimeWindowSuite.scala |    38 +-
 .../catalyst/expressions/ToPrettyStringSuite.scala |   144 +
 .../sql/catalyst/expressions/TryCastSuite.scala    |    98 +-
 .../sql/catalyst/expressions/TryEvalSuite.scala    |    59 +-
 .../expressions/UnsafeRowConverterSuite.scala      |    91 +-
 .../expressions/UnwrapUDTExpressionSuite.scala     |    40 +
 .../expressions/V2ExpressionUtilsSuite.scala       |    40 +
 .../catalyst/expressions/XmlExpressionsSuite.scala |   522 +
 .../aggregate/AggregateExpressionSuite.scala       |   121 +-
 .../ApproxCountDistinctForIntervalsSuite.scala     |    66 +-
 .../expressions/aggregate/ApproxTopKSuite.scala    |   408 +
 .../aggregate/ApproximatePercentileSuite.scala     |   107 +-
 .../aggregate/CentralMomentAggSuite.scala          |    77 +
 .../aggregate/CountMinSketchAggSuite.scala         |    77 +-
 .../expressions/aggregate/CovarianceAggSuite.scala |    39 +
 .../aggregate/DatasketchesHllSketchSuite.scala     |   156 +
 .../aggregate/DeclarativeAggregateEvaluator.scala  |    10 +-
 .../expressions/aggregate/FirstLastTestSuite.scala |     4 +
 .../aggregate/HistogramNumericSuite.scala          |   292 +
 .../expressions/aggregate/PercentileSuite.scala    |   153 +-
 .../aggregate/TestWithAndWithoutCodegen.scala      |    35 +
 .../aggregate/ThetasketchesAggSuite.scala          |   175 +
 .../expressions/aggregate/VectorAggSuite.scala     |   497 +
 .../codegen/BufferHolderSparkSubmitSuite.scala     |    26 +-
 .../expressions/codegen/BufferHolderSuite.scala    |    22 +-
 .../expressions/codegen/CodeBlockSuite.scala       |    13 +-
 .../codegen/GenerateUnsafeProjectionSuite.scala    |     8 +-
 .../codegen/GenerateUnsafeRowJoinerSuite.scala     |     2 +-
 .../codegen/GeneratedProjectionSuite.scala         |     7 +-
 .../codegen/UnsafeArrayWriterSuite.scala           |    40 +
 .../expressions/codegen/UnsafeRowWriterSuite.scala |    35 +-
 .../variant/VariantExpressionEvalUtilsSuite.scala  |   180 +
 .../variant/VariantExpressionSuite.scala           |  1203 +
 .../xml/ReusableStringReaderSuite.scala            |     6 +-
 .../expressions/xml/UDFXPathUtilSuite.scala        |     7 +-
 .../expressions/xml/XPathExpressionSuite.scala     |    14 +-
 .../sql/catalyst/json/JacksonGeneratorSuite.scala  |     6 +-
 .../sql/catalyst/json/JacksonParserSuite.scala     |    13 +
 .../sql/catalyst/json/JsonInferSchemaSuite.scala   |     8 +
 .../optimizer/AggregateOptimizeSuite.scala         |   131 +-
 .../BinaryComparisonSimplificationSuite.scala      |   118 +-
 .../optimizer/BooleanSimplificationSuite.scala     |   247 +-
 .../optimizer/CheckCartesianProductsSuite.scala    |     8 +-
 .../catalyst/optimizer/CollapseProjectSuite.scala  |   267 +-
 .../optimizer/CollapseRepartitionSuite.scala       |    61 +-
 .../catalyst/optimizer/CollapseWindowSuite.scala   |    52 +-
 .../catalyst/optimizer/ColumnPruningSuite.scala    |   263 +-
 .../catalyst/optimizer/CombiningLimitsSuite.scala  |    74 +-
 .../optimizer/ComputeCurrentTimeSuite.scala        |   227 +-
 .../catalyst/optimizer/ConstantFoldingSuite.scala  |   328 +-
 .../optimizer/ConstantPropagationSuite.scala       |    42 +-
 .../optimizer/ConvertToLocalRelationSuite.scala    |    12 +-
 .../optimizer/DecimalAggregatesSuite.scala         |    51 +-
 .../optimizer/DecorrelateInnerQuerySuite.scala     |   470 +-
 .../optimizer/EliminateAggregateFilterSuite.scala  |    32 +-
 .../optimizer/EliminateDistinctSuite.scala         |    45 +-
 .../optimizer/EliminateMapObjectsSuite.scala       |    10 +-
 .../catalyst/optimizer/EliminateOffsetsSuite.scala |   100 +
 .../optimizer/EliminateSerializationSuite.scala    |    15 +-
 .../EliminateSortsBeforeRepartitionSuite.scala     |   102 +-
 .../catalyst/optimizer/EliminateSortsSuite.scala   |   273 +-
 .../optimizer/EliminateSubqueryAliasesSuite.scala  |    20 +-
 .../optimizer/EliminateWindowPartitionsSuite.scala |    80 +
 .../ExtractPythonUDFFromJoinConditionSuite.scala   |    29 +-
 .../optimizer/FilterPushdownOnePassSuite.scala     |    84 +-
 .../catalyst/optimizer/FilterPushdownSuite.scala   |   956 +-
 .../optimizer/FoldablePropagationSuite.scala       |   149 +-
 .../optimizer/GenerateOptimizationSuite.scala      |   133 +
 .../InferFiltersFromConstraintsSuite.scala         |   263 +-
 .../optimizer/InferFiltersFromGenerateSuite.scala  |    92 +-
 .../optimizer/InferWindowGroupLimitSuite.scala     |   357 +
 .../sql/catalyst/optimizer/InlineCTESuite.scala    |    53 +
 .../catalyst/optimizer/JoinOptimizationSuite.scala |    16 +-
 .../optimizer/JoinSelectionHelperSuite.scala       |   100 +-
 .../optimizer/LeftSemiAntiJoinPushDownSuite.scala  |   254 +-
 .../optimizer/LikeSimplificationSuite.scala        |   202 +-
 .../catalyst/optimizer/LimitPushdownSuite.scala    |   205 +-
 .../LimitPushdownThroughWindowSuite.scala          |    24 +-
 .../catalyst/optimizer/MergeSubplansSuite.scala    |  1969 ++
 .../optimizer/NestedColumnAliasingSuite.scala      |   335 +-
 .../NormalizeFloatingPointNumbersSuite.scala       |    12 +-
 .../optimizer/NullDownPropagationSuite.scala       |    58 +
 .../optimizer/ObjectSerializerPruningSuite.scala   |    22 +-
 .../catalyst/optimizer/OptimizeCsvExprsSuite.scala |    14 +-
 .../catalyst/optimizer/OptimizeExpandSuite.scala   |   179 +
 .../sql/catalyst/optimizer/OptimizeInSuite.scala   |   176 +-
 .../optimizer/OptimizeJoinConditionSuite.scala     |    49 +
 .../optimizer/OptimizeJsonExprsSuite.scala         |    77 +-
 .../optimizer/OptimizeLimitZeroSuite.scala         |    34 +-
 .../optimizer/OptimizeOneRowPlanSuite.scala        |   104 +
 .../OptimizeOneRowRelationSubquerySuite.scala      |    91 +-
 .../sql/catalyst/optimizer/OptimizeRandSuite.scala |   176 +
 .../optimizer/OptimizeWindowFunctionsSuite.scala   |     2 +-
 .../optimizer/OptimizeWithFieldsSuite.scala        |    34 +-
 .../catalyst/optimizer/OptimizerLoggingSuite.scala |    44 +-
 .../optimizer/OptimizerRuleExclusionSuite.scala    |     8 +-
 .../OptimizerStructuralIntegrityCheckerSuite.scala |    77 +-
 .../sql/catalyst/optimizer/OptimizerSuite.scala    |   267 +-
 .../optimizer/OuterJoinEliminationSuite.scala      |   142 +-
 .../optimizer/PropagateEmptyRelationSuite.scala    |   266 +-
 .../sql/catalyst/optimizer/PruneFiltersSuite.scala |   100 +-
 .../PullupCorrelatedPredicatesSuite.scala          |    60 +-
 .../optimizer/PushDownJoinThroughUnionSuite.scala  |   241 +
 .../optimizer/PushFoldableIntoBranchesSuite.scala  |    52 +-
 .../optimizer/PushProjectThroughUnionSuite.scala   |    16 +-
 .../PushProjectionThroughLimitAndOffsetSuite.scala |   162 +
 .../optimizer/ReassignLambdaVariableIDSuite.scala  |    14 +-
 .../optimizer/RemoveNoopOperatorsSuite.scala       |    10 +-
 .../catalyst/optimizer/RemoveNoopUnionSuite.scala  |     4 +-
 .../optimizer/RemoveRedundantAggregatesSuite.scala |   238 +-
 .../RemoveRedundantAliasAndProjectSuite.scala      |   209 +-
 .../ReorderAssociativeOperatorSuite.scala          |    77 +-
 .../ReplaceNullWithFalseInPredicateSuite.scala     |    81 +-
 .../catalyst/optimizer/ReplaceOperatorSuite.scala  |   111 +-
 .../catalyst/optimizer/RewriteAsOfJoinSuite.scala  |   289 +
 .../optimizer/RewriteDistinctAggregatesSuite.scala |    76 +-
 .../optimizer/RewriteNearestByJoinSuite.scala      |   297 +
 .../catalyst/optimizer/RewriteSubquerySuite.scala  |    50 +-
 .../optimizer/RewriteWithExpressionSuite.scala     |   503 +
 .../sql/catalyst/optimizer/SetOperationSuite.scala |   332 +-
 .../catalyst/optimizer/SimpleTestOptimizer.scala   |    34 +
 .../catalyst/optimizer/SimplifyCastsSuite.scala    |    86 +-
 .../optimizer/SimplifyConditionalSuite.scala       |    54 +-
 .../SimplifyConditionalsInPredicateSuite.scala     |   246 -
 .../SimplifyDateTimeConversionsSuite.scala         |    69 +
 .../SimplifyStringCaseConversionSuite.scala        |    18 +-
 .../catalyst/optimizer/TransposeWindowSuite.scala  |    67 +-
 .../optimizer/TypedFilterOptimizationSuite.scala   |     9 +-
 .../UnwrapCastInBinaryComparisonSuite.scala        |   319 +-
 .../sql/catalyst/optimizer/complexTypesSuite.scala |   506 +-
 .../optimizer/joinReorder/JoinReorderSuite.scala   |    12 +-
 .../joinReorder/StarJoinReorderSuite.scala         |     6 +-
 .../sql/catalyst/parser/CastingSyntaxSuite.scala   |   103 +
 .../spark/sql/catalyst/parser/DDLParserSuite.scala |  3417 +-
 .../sql/catalyst/parser/DataTypeParserSuite.scala  |    85 +-
 .../sql/catalyst/parser/ErrorParserSuite.scala     |   453 +-
 .../catalyst/parser/ExpressionParserSuite.scala    |   679 +-
 .../parser/ParameterSubstitutionSuite.scala        |   173 +
 .../sql/catalyst/parser/ParserUtilsSuite.scala     |   135 +-
 .../sql/catalyst/parser/PivotParserSuite.scala     |   111 +
 .../sql/catalyst/parser/PlanParserSuite.scala      |  1814 +-
 .../catalyst/parser/SqlScriptingParserSuite.scala  |  3488 +++
 .../parser/StreamRelationParserSuite.scala         |   713 +
 .../parser/TableIdentifierParserSuite.scala        |    38 +-
 .../catalyst/parser/TableSchemaParserSuite.scala   |    37 +-
 .../sql/catalyst/parser/UnpivotParserSuite.scala   |   340 +
 .../catalyst/planning/PhysicalOperationSuite.scala |   113 +
 .../sql/catalyst/planning/ScanOperationSuite.scala |   109 -
 .../plans/ConstraintPropagationSuite.scala         |   171 +-
 .../spark/sql/catalyst/plans/JoinTypesTest.scala   |    15 +
 .../sql/catalyst/plans/LogicalPlanSuite.scala      |    79 +-
 .../sql/catalyst/plans/NormalizePlanSuite.scala    |   475 +
 .../apache/spark/sql/catalyst/plans/PlanTest.scala |    93 +-
 .../spark/sql/catalyst/plans/QueryPlanSuite.scala  |    90 +-
 .../spark/sql/catalyst/plans/SQLHelper.scala       |    32 +-
 .../spark/sql/catalyst/plans/SameResultSuite.scala |    21 +-
 .../plans/logical/AnalysisHelperSuite.scala        |     2 +-
 .../plans/logical/DistinctKeyVisitorSuite.scala    |   206 +
 .../plans/logical/LogicalPlanIntegritySuite.scala  |    22 +-
 .../BasicStatsEstimationSuite.scala                |    88 +-
 .../statsEstimation/FilterEstimationSuite.scala    |    74 +
 .../statsEstimation/ProjectEstimationSuite.scala   |    34 +-
 .../UnionAggregateEstimationSuite.scala            |   127 +
 .../statsEstimation/UnionEstimationSuite.scala     |   140 +-
 .../streaming/InternalOutputModesSuite.scala       |    19 +-
 .../transactions/TransactionUtilsSuite.scala       |   124 +
 .../sql/catalyst/trees/RuleExecutorSuite.scala     |   109 +-
 .../sql/catalyst/trees/SQLQueryContextSuite.scala  |    49 +
 .../spark/sql/catalyst/trees/TreeNodeSuite.scala   |   376 +-
 .../catalyst/util/ArrayBasedMapBuilderSuite.scala  |    77 +-
 .../catalyst/util/ArrayDataIndexedSeqSuite.scala   |    28 +-
 .../catalyst/util/CaseInsensitiveMapSuite.scala    |     2 +-
 .../sql/catalyst/util/DateFormatterSuite.scala     |     7 +-
 .../util/DateTimeFormatterHelperSuite.scala        |    63 +-
 .../sql/catalyst/util/DateTimeTestUtils.scala      |    19 +-
 .../sql/catalyst/util/DateTimeUtilsSuite.scala     |   939 +-
 .../sql/catalyst/util/DatetimeFormatterSuite.scala |    67 +-
 .../catalyst/util/GenericArrayDataBenchmark.scala  |     7 +-
 .../sql/catalyst/util/GeographyExecutionSuite.java |   253 +
 .../sql/catalyst/util/GeometryExecutionSuite.java  |   273 +
 .../InternalRowComparableWrapperBenchmark.scala    |    75 +
 .../sql/catalyst/util/IntervalUtilsSuite.scala     |   340 +-
 .../sql/catalyst/util/NumberConverterSuite.scala   |    21 +-
 .../catalyst/util/PhysicalAggregationSuite.scala   |     9 +-
 .../sql/catalyst/util/QuotingUtilsSuite.scala      |    43 +
 .../sql/catalyst/util/RebaseDateTimeSuite.scala    |    30 +-
 .../spark/sql/catalyst/util/StUtilsSuite.java      |   243 +
 .../spark/sql/catalyst/util/StringUtilsSuite.scala |   166 +-
 .../sql/catalyst/util/ThetaSketchUtilsSuite.scala  |   133 +
 .../sql/catalyst/util/TimeFormatterSuite.scala     |   182 +
 .../catalyst/util/TimestampFormatterSuite.scala    |   189 +-
 .../sql/catalyst/util/TupleSketchUtilsSuite.scala  |   550 +
 .../spark/sql/catalyst/util/TypeUtilsSuite.scala   |    23 +-
 .../spark/sql/catalyst/util/UnsafeArraySuite.scala |    18 +-
 .../sql/catalyst/util/UnsafeRowUtilsSuite.scala    |   115 +-
 .../connector/catalog/CatalogManagerSuite.scala    |    25 +-
 .../spark/sql/connector/catalog/CatalogSuite.scala |   936 +-
 .../sql/connector/catalog/CatalogV2UtilSuite.scala |     4 +-
 .../catalog/ComposedColumnIdTableCatalog.scala     |   288 +
 .../sql/connector/catalog/ConstraintSuite.scala    |   146 +
 .../connector/catalog/EnumTypeSetBenchmark.scala   |   179 +
 .../catalog/InMemoryAtomicPartitionTable.scala     |    18 +-
 .../sql/connector/catalog/InMemoryBaseTable.scala  |  1250 +
 .../sql/connector/catalog/InMemoryCatalog.scala    |    28 +-
 .../catalog/InMemoryChangelogCatalog.scala         |   305 +
 .../InMemoryEnhancedPartitionFilterTable.scala     |   169 +
 ...MemoryEnhancedRuntimePartitionFilterTable.scala |   127 +
 .../InMemoryPartitionPredicateDeleteTable.scala    |   146 +
 .../connector/catalog/InMemoryPartitionTable.scala |    23 +-
 .../catalog/InMemoryPartitionTableCatalog.scala    |    32 +-
 .../catalog/InMemoryRowLevelOperationTable.scala   |   297 +
 .../InMemoryRowLevelOperationTableCatalog.scala    |   134 +
 .../sql/connector/catalog/InMemoryTable.scala      |   688 +-
 .../connector/catalog/InMemoryTableCatalog.scala   |   249 +-
 ...MemoryTableEnhancedPartitionFilterCatalog.scala |    49 +
 ...ableEnhancedRuntimePartitionFilterCatalog.scala |    50 +
 .../catalog/InMemoryTableViewCatalog.scala         |   244 +
 .../catalog/InMemoryTableWithV2Filter.scala        |   209 +
 .../catalog/InMemoryTableWithV2FilterCatalog.scala |    49 +
 .../sql/connector/catalog/LookupCatalogSuite.scala |    21 +-
 .../catalog/MixedColumnIdTableCatalog.scala        |   125 +
 .../catalog/NullColumnIdInMemoryTableCatalog.scala |   105 +
 .../catalog/NullTableIdInMemoryTableCatalog.scala  |    50 +
 .../catalog/SharedInMemoryTableCatalog.scala       |    46 +
 .../catalog/StagingInMemoryTableCatalog.scala      |    45 +-
 .../SupportsAtomicPartitionManagementSuite.scala   |    54 +-
 .../catalog/SupportsPartitionManagementSuite.scala |    88 +-
 .../TypeChangeResetsColIdTableCatalog.scala        |    69 +
 .../sql/connector/catalog/V2TableUtilSuite.scala   |   721 +
 .../apache/spark/sql/connector/catalog/txns.scala  |   254 +
 .../expressions/TransformExtractorSuite.scala      |   115 +-
 .../datasources/v2/DataSourceV2RelationSuite.scala |   110 +
 .../connector/PartitionPredicateImplSuite.scala    |   109 +
 .../metricview/serde/MetricViewFactorySuite.scala  |   373 +
 .../spark/sql/sources/FakeCommandRunner.scala      |    30 +
 .../org/apache/spark/sql/types/DataTypeSuite.scala |   938 +-
 .../apache/spark/sql/types/DataTypeTestUtils.scala |    29 +-
 .../types/DataTypeWriteCompatibilitySuite.scala    |   576 +-
 .../org/apache/spark/sql/types/DecimalSuite.scala  |   195 +-
 .../org/apache/spark/sql/types/MetadataSuite.scala |    91 +-
 .../apache/spark/sql/types/StructTypeSuite.scala   |   576 +-
 .../scala/org/apache/spark/sql/types/TestUDT.scala |    20 +
 .../apache/spark/sql/util/ArrowUtilsSuite.scala    |    98 +-
 .../sql/util/CaseInsensitiveStringMapSuite.scala   |    17 +-
 .../apache/spark/sql/util/SchemaUtilsSuite.scala   |    52 +-
 .../org/apache/spark/sql/util/UtilSuite.scala      |    31 -
 sql/connect/README.md                              |    68 +
 sql/connect/bin/spark-connect                      |    44 +
 sql/connect/bin/spark-connect-build                |    33 +
 sql/connect/bin/spark-connect-scala-client         |    79 +
 .../bin/spark-connect-scala-client-classpath       |    34 +
 sql/connect/bin/spark-connect-shell                |    39 +
 sql/connect/client/jdbc/pom.xml                    |   140 +
 .../connect/client/jdbc/SparkConnectDriver.java    |    31 +
 .../resources/META-INF/services/java.sql.Driver    |    18 +
 .../jdbc/NonRegisteringSparkConnectDriver.scala    |    49 +
 .../client/jdbc/SparkConnectConnection.scala       |   280 +
 .../client/jdbc/SparkConnectDatabaseMetaData.scala |   819 +
 .../client/jdbc/SparkConnectResultSet.scala        |   789 +
 .../jdbc/SparkConnectResultSetMetaData.scala       |    84 +
 .../client/jdbc/SparkConnectStatement.scala        |   267 +
 .../connect/client/jdbc/util/JdbcErrorUtils.scala  |    56 +
 .../connect/client/jdbc/util/JdbcTypeUtils.scala   |   152 +
 .../jdbc/src/test/resources/log4j2.properties      |    42 +
 .../jdbc/SparkConnectDatabaseMetaDataSuite.scala   |   805 +
 .../client/jdbc/SparkConnectDriverSuite.scala      |    43 +
 .../jdbc/SparkConnectJdbcDataTypeSuite.scala       |   792 +
 .../client/jdbc/SparkConnectResultSetSuite.scala   |   189 +
 .../client/jdbc/SparkConnectStatementSuite.scala   |   119 +
 .../sql/connect/client/jdbc/test/JdbcHelper.scala  |    48 +
 sql/connect/client/jvm/pom.xml                     |   289 +
 .../apache/spark/sql/application/ConnectRepl.scala |   193 +
 .../sql/connect/client/AmmoniteClassFinder.scala   |    47 +
 .../org/apache/spark/sql/JavaEncoderSuite.java     |   114 +
 .../jvm/src/test/resources/StubClassDummyUdf.scala |    31 +
 .../test/resources/StubClassDummyUdfPacker.scala   |    41 +
 .../jvm/src/test/resources/log4j2.properties       |    42 +
 .../apache/spark/sql/DataFrameSubquerySuite.scala  |   862 +
 .../sql/DataFrameTableValuedFunctionsSuite.scala   |   544 +
 .../spark/sql/GeographyConnectDataFrameSuite.scala |   127 +
 .../spark/sql/GeometryConnectDataFrameSuite.scala  |   133 +
 .../apache/spark/sql/PlanGenerationTestSuite.scala |  3666 +++
 .../org/apache/spark/sql/SQLExpressionsSuite.scala |    47 +
 .../spark/sql/application/ReplE2ESuite.scala       |   572 +
 .../apache/spark/sql/connect/CatalogSuite.scala    |   478 +
 .../apache/spark/sql/connect/CheckpointSuite.scala |   126 +
 .../sql/connect/ClientDataFrameStatSuite.scala     |   276 +
 .../spark/sql/connect/ClientDatasetSuite.scala     |   187 +
 .../spark/sql/connect/ClientE2ETestSuite.scala     |  2100 ++
 .../connect/ColumnNodeToProtoConverterSuite.scala  |   482 +
 .../apache/spark/sql/connect/ColumnTestSuite.scala |   266 +
 .../sql/connect/DataFrameNaFunctionSuite.scala     |   408 +
 .../apache/spark/sql/connect/DataFrameSuite.scala  |    91 +
 .../spark/sql/connect/FunctionTestSuite.scala      |   285 +
 .../KeyValueGroupedDatasetE2ETestSuite.scala       |   821 +
 .../spark/sql/connect/MergeIntoE2ETestSuite.scala  |   188 +
 .../spark/sql/connect/SQLImplicitsTestSuite.scala  |   190 +
 ...kSessionBuilderImplementationBindingSuite.scala |    36 +
 .../spark/sql/connect/SparkSessionCloneSuite.scala |    64 +
 .../sql/connect/SparkSessionCloseE2ESuite.scala    |    58 +
 .../spark/sql/connect/SparkSessionE2ESuite.scala   |   502 +
 .../spark/sql/connect/SparkSessionSuite.scala      |   278 +
 .../spark/sql/connect/StaticProcedureSuite.scala   |    31 +
 .../spark/sql/connect/StubbingTestSuite.scala      |    37 +
 .../sql/connect/UDFClassLoadingE2ESuite.scala      |   109 +
 .../sql/connect/UnsupportedFeaturesSuite.scala     |   112 +
 .../connect/UserDefinedFunctionE2ETestSuite.scala  |   593 +
 .../sql/connect/UserDefinedFunctionSuite.scala     |    91 +
 .../spark/sql/connect/client/ArtifactSuite.scala   |   380 +
 .../CheckConnectJvmClientCompatibility.scala       |   427 +
 .../sql/connect/client/ClassFinderSuite.scala      |    53 +
 .../SparkConnectClientBuilderParseTestSuite.scala  |   135 +
 .../client/SparkConnectClientRetriesSuite.scala    |   278 +
 .../connect/client/SparkConnectClientSuite.scala   |  1094 +
 .../connect/client/arrow/ArrowEncoderSuite.scala   |  1368 +
 .../streaming/ClientStreamingQuerySuite.scala      |   888 +
 .../FlatMapGroupsWithStateStreamingSuite.scala     |   233 +
 .../streaming/StreamingQueryProgressSuite.scala    |   229 +
 .../streaming/TransformWithStateConnectSuite.scala |   740 +
 .../spark/sql/connect/test/ConnectFunSuite.scala   |    61 +
 .../sql/connect/test/IntegrationTestUtils.scala    |   165 +
 .../apache/spark/sql/connect/test/QueryTest.scala  |   465 +
 .../sql/connect/test/RemoteSparkSession.scala      |   265 +
 .../apache/spark/sql/connect/test/SQLHelper.scala  |   157 +
 sql/connect/common/README.md                       |     5 +
 sql/connect/common/pom.xml                         |   215 +
 sql/connect/common/src/main/buf.gen.yaml           |    36 +
 sql/connect/common/src/main/buf.work.yaml          |    19 +
 sql/connect/common/src/main/protobuf/README.md     |    71 +
 sql/connect/common/src/main/protobuf/buf.yaml      |    25 +
 .../src/main/protobuf/spark/connect/base.proto     |  1367 +
 .../src/main/protobuf/spark/connect/catalog.proto  |   324 +
 .../src/main/protobuf/spark/connect/commands.proto |   560 +
 .../src/main/protobuf/spark/connect/common.proto   |   179 +
 .../protobuf/spark/connect/example_plugins.proto   |    42 +
 .../main/protobuf/spark/connect/expressions.proto  |   557 +
 .../src/main/protobuf/spark/connect/ml.proto       |   147 +
 .../main/protobuf/spark/connect/ml_common.proto    |    64 +
 .../main/protobuf/spark/connect/pipelines.proto    |   343 +
 .../main/protobuf/spark/connect/relations.proto    |  1278 +
 .../src/main/protobuf/spark/connect/types.proto    |   227 +
 .../org/apache/spark/sql/connect/Catalog.scala     |   929 +
 .../connect/ConnectClientUnsupportedErrors.scala   |    53 +
 .../spark/sql/connect/ConnectConversions.scala     |    88 +
 .../spark/sql/connect/ConnectProtoUtils.scala      |    56 +
 .../spark/sql/connect/DataFrameNaFunctions.scala   |   214 +
 .../apache/spark/sql/connect/DataFrameReader.scala |   236 +
 .../spark/sql/connect/DataFrameStatFunctions.scala |   145 +
 .../apache/spark/sql/connect/DataFrameWriter.scala |   160 +
 .../spark/sql/connect/DataFrameWriterV2.scala      |   126 +
 .../spark/sql/connect/DataStreamReader.scala       |   166 +
 .../spark/sql/connect/DataStreamWriter.scala       |   201 +
 .../org/apache/spark/sql/connect/Dataset.scala     |  1571 +
 .../spark/sql/connect/KeyValueGroupedDataset.scala |   800 +
 .../apache/spark/sql/connect/MergeIntoWriter.scala |   136 +
 .../sql/connect/RelationalGroupedDataset.scala     |   182 +
 .../apache/spark/sql/connect/RuntimeConfig.scala   |   165 +
 .../org/apache/spark/sql/connect/SQLContext.scala  |   317 +
 .../apache/spark/sql/connect/SQLImplicits.scala    |    40 +
 .../apache/spark/sql/connect/SessionCleaner.scala  |    48 +
 .../apache/spark/sql/connect/SparkSession.scala    |  1150 +
 .../apache/spark/sql/connect/StreamingQuery.scala  |   173 +
 .../sql/connect/StreamingQueryListenerBus.scala    |   155 +
 .../spark/sql/connect/StreamingQueryManager.scala  |   120 +
 .../spark/sql/connect/TableValuedFunction.scala    |   109 +
 .../apache/spark/sql/connect/UDFRegistration.scala |    68 +
 .../apache/spark/sql/connect/UdfToProtoUtils.scala |   112 +
 .../UserDefinedAggregateFunctionWrapper.scala      |    94 +
 .../spark/sql/connect/client/ArtifactManager.scala |   482 +
 .../spark/sql/connect/client/ClassFinder.scala     |    55 +
 .../client/CustomSparkConnectBlockingStub.scala    |   161 +
 .../connect/client/CustomSparkConnectStub.scala    |    35 +
 .../ExecutePlanResponseReattachableIterator.scala  |   345 +
 .../connect/client/GrpcExceptionConverter.scala    |   522 +
 .../sql/connect/client/GrpcRetryHandler.scala      |   238 +
 .../sql/connect/client/ResponseValidator.scala     |   142 +
 .../spark/sql/connect/client/RetryPolicy.scala     |   195 +
 .../sql/connect/client/SparkConnectClient.scala    |  1170 +
 .../connect/client/SparkConnectClientParser.scala  |   104 +
 .../sql/connect/client/SparkConnectStubState.scala |    41 +
 .../spark/sql/connect/client/SparkResult.scala     |   463 +
 .../apache/spark/sql/connect/client/ToStub.scala   |    23 +
 .../connect/client/arrow/ArrowDeserializer.scala   |   635 +
 .../connect/client/arrow/ArrowEncoderUtils.scala   |    68 +
 .../sql/connect/client/arrow/ArrowSerializer.scala |   629 +
 .../connect/client/arrow/ArrowVectorReader.scala   |   298 +
 .../client/arrow/GeospatialArrowSerDe.scala        |   101 +
 .../client/arrow/ScalaCollectionUtils.scala        |    37 +
 .../arrow/types/ops/TimeTypeConnectOps.scala       |   122 +
 .../apache/spark/sql/connect/client/package.scala  |    28 +
 .../spark/sql/connect/columnNodeSupport.scala      |   319 +
 .../spark/sql/connect/common/Abbreviator.scala     |   112 +
 .../connect/common/DataTypeProtoConverter.scala    |   407 +
 .../sql/connect/common/ForeachWriterPacket.scala   |    34 +
 .../sql/connect/common/InvalidPlanInput.scala      |    56 +
 .../common/LiteralValueProtoConverter.scala        |   709 +
 .../spark/sql/connect/common/ProtoDataTypes.scala  |   136 +
 .../sql/connect/common/ProtoSpecializedArray.scala |   172 +
 .../spark/sql/connect/common/ProtoUtils.scala      |    79 +
 .../common/StorageLevelProtoConverter.scala        |    46 +
 .../connect/common/StreamingListenerPacket.scala   |    32 +
 .../spark/sql/connect/common/UdfPacket.scala       |    71 +
 .../apache/spark/sql/connect/common/UdfUtils.scala |   589 +
 .../sql/connect/common/config/ConnectCommon.scala  |    24 +
 .../connect/common/types/ops/ConnectTypeOps.scala  |   175 +
 .../org/apache/spark/sql/connect/package.scala     |    22 +
 .../src/test/resources/protobuf-tests/common.desc  |    12 +
 .../explain-results/alias_string.explain           |     2 +
 .../explain-results/alias_symbol.explain           |     2 +
 .../query-tests/explain-results/apply.explain      |     2 +
 .../query-tests/explain-results/as_string.explain  |     2 +
 .../query-tests/explain-results/as_symbol.explain  |     2 +
 .../explain-results/between_expr.explain           |     3 +
 .../query-tests/explain-results/coalesce.explain   |     2 +
 .../query-tests/explain-results/col.explain        |     2 +
 .../query-tests/explain-results/colRegex.explain   |     2 +
 .../query-tests/explain-results/column_add.explain |     2 +
 .../explain-results/column_alias.explain           |     2 +
 .../query-tests/explain-results/column_and.explain |     2 +
 .../explain-results/column_apply.explain           |     2 +
 .../explain-results/column_as_multi.explain        |     3 +
 .../column_as_with_metadata.explain                |     2 +
 .../query-tests/explain-results/column_asc.explain |     2 +
 .../explain-results/column_asc_nulls_first.explain |     2 +
 .../explain-results/column_asc_nulls_last.explain  |     2 +
 .../explain-results/column_between.explain         |     2 +
 .../explain-results/column_bitwiseAND.explain      |     2 +
 .../explain-results/column_bitwiseOR.explain       |     2 +
 .../explain-results/column_bitwiseXOR.explain      |     2 +
 .../explain-results/column_cast.explain            |     2 +
 .../explain-results/column_contains.explain        |     2 +
 .../explain-results/column_desc.explain            |     2 +
 .../column_desc_nulls_first.explain                |     2 +
 .../explain-results/column_desc_nulls_last.explain |     2 +
 .../explain-results/column_divide.explain          |     2 +
 .../explain-results/column_dropFields.explain      |     2 +
 .../explain-results/column_endsWith.explain        |     2 +
 .../explain-results/column_eqNullSafe.explain      |     2 +
 .../explain-results/column_equals.explain          |     2 +
 .../query-tests/explain-results/column_geq.explain |     2 +
 .../explain-results/column_getField.explain        |     2 +
 .../explain-results/column_getItem.explain         |     2 +
 .../query-tests/explain-results/column_gt.explain  |     2 +
 .../explain-results/column_ilike.explain           |     2 +
 .../explain-results/column_isNaN.explain           |     2 +
 .../explain-results/column_isNotNull.explain       |     2 +
 .../explain-results/column_isNull.explain          |     2 +
 .../explain-results/column_isin.explain            |     2 +
 .../query-tests/explain-results/column_leq.explain |     2 +
 .../explain-results/column_like.explain            |     2 +
 .../query-tests/explain-results/column_lt.explain  |     2 +
 .../explain-results/column_modulo.explain          |     2 +
 .../explain-results/column_multiply.explain        |     2 +
 .../query-tests/explain-results/column_not.explain |     2 +
 .../explain-results/column_not_equals.explain      |     2 +
 .../query-tests/explain-results/column_or.explain  |     2 +
 .../explain-results/column_rlike.explain           |     2 +
 .../explain-results/column_star.explain            |     2 +
 .../column_star_with_target.explain                |     2 +
 .../explain-results/column_startsWith.explain      |     2 +
 .../explain-results/column_substr.explain          |     2 +
 .../explain-results/column_subtract.explain        |     2 +
 .../explain-results/column_try_cast.explain        |     2 +
 .../explain-results/column_unary_minus.explain     |     2 +
 .../explain-results/column_when_otherwise.explain  |     2 +
 .../explain-results/column_withField.explain       |     2 +
 .../query-tests/explain-results/crossJoin.explain  |     3 +
 .../query-tests/explain-results/crosstab.explain   |     5 +
 .../explain-results/csv_from_dataset.explain       |     1 +
 .../explain-results/cube_column.explain            |     4 +
 .../explain-results/cube_string.explain            |     4 +
 .../query-tests/explain-results/describe.explain   |     6 +
 .../query-tests/explain-results/distinct.explain   |     2 +
 .../query-tests/explain-results/drop.explain       |     2 +
 .../explain-results/dropDuplicates.explain         |     2 +
 .../dropDuplicates_names_array.explain             |     2 +
 .../dropDuplicates_names_seq.explain               |     2 +
 .../explain-results/dropDuplicates_varargs.explain |     2 +
 .../explain-results/drop_multiple_column.explain   |     2 +
 .../explain-results/drop_multiple_strings.explain  |     2 +
 .../explain-results/drop_single_column.explain     |     2 +
 .../explain-results/drop_single_string.explain     |     2 +
 .../query-tests/explain-results/except.explain     |     3 +
 .../query-tests/explain-results/exceptAll.explain  |     3 +
 .../explain-results/expression_extension.explain   |     2 +
 .../expression_extension_deprecated.explain        |     2 +
 .../query-tests/explain-results/fill.explain       |     2 +
 .../query-tests/explain-results/filter.explain     |     2 +
 .../explain-results/filter_expr.explain            |     2 +
 .../query-tests/explain-results/freqItems.explain  |     2 +
 .../explain-results/from_avro_with_options.explain |     2 +
 .../from_avro_without_options.explain              |     2 +
 ..._protobuf_messageClassName_descFilePath.explain |     2 +
 ...f_messageClassName_descFilePath_options.explain |     2 +
 .../explain-results/function_abs.explain           |     2 +
 .../explain-results/function_acos.explain          |     2 +
 .../explain-results/function_acosh.explain         |     2 +
 .../explain-results/function_add_months.explain    |     2 +
 .../explain-results/function_aes_decrypt.explain   |     2 +
 .../function_aes_decrypt_with_mode.explain         |     2 +
 .../function_aes_decrypt_with_mode_padding.explain |     2 +
 ...ction_aes_decrypt_with_mode_padding_aad.explain |     2 +
 .../explain-results/function_aes_encrypt.explain   |     2 +
 .../function_aes_encrypt_with_mode.explain         |     2 +
 .../function_aes_encrypt_with_mode_padding.explain |     2 +
 ...nction_aes_encrypt_with_mode_padding_iv.explain |     2 +
 ...on_aes_encrypt_with_mode_padding_iv_aad.explain |     2 +
 .../explain-results/function_aggregate.explain     |     2 +
 .../function_aggregate_with_finish_lambda.explain  |     2 +
 .../explain-results/function_any.explain           |     2 +
 .../function_any_value_with_ignore_nulls.explain   |     2 +
 .../function_any_value_with_respect_nulls.explain  |     2 +
 .../function_approx_count_distinct.explain         |     2 +
 .../function_approx_count_distinct_rsd.explain     |     2 +
 .../function_approx_percentile.explain             |     2 +
 .../explain-results/function_array.explain         |     2 +
 .../explain-results/function_array_agg.explain     |     2 +
 .../explain-results/function_array_append.explain  |     2 +
 .../explain-results/function_array_compact.explain |     2 +
 .../function_array_contains.explain                |     2 +
 .../function_array_distinct.explain                |     2 +
 .../explain-results/function_array_except.explain  |     2 +
 .../explain-results/function_array_insert.explain  |     2 +
 .../function_array_intersect.explain               |     2 +
 .../explain-results/function_array_join.explain    |     2 +
 ...nction_array_join_with_null_replacement.explain |     2 +
 .../explain-results/function_array_max.explain     |     2 +
 .../explain-results/function_array_min.explain     |     2 +
 .../function_array_position.explain                |     2 +
 .../explain-results/function_array_prepend.explain |     2 +
 .../explain-results/function_array_remove.explain  |     2 +
 .../explain-results/function_array_repeat.explain  |     2 +
 .../explain-results/function_array_size.explain    |     2 +
 .../explain-results/function_array_sort.explain    |     2 +
 .../function_array_sort_with_comparator.explain    |     2 +
 .../explain-results/function_array_union.explain   |     2 +
 .../function_arrays_overlap.explain                |     2 +
 .../explain-results/function_arrays_zip.explain    |     2 +
 .../explain-results/function_asc.explain           |     2 +
 .../function_asc_nulls_first.explain               |     2 +
 .../function_asc_nulls_last.explain                |     2 +
 .../explain-results/function_ascii.explain         |     2 +
 .../explain-results/function_asin.explain          |     2 +
 .../explain-results/function_asinh.explain         |     2 +
 .../function_assert_true_with_message.explain      |     2 +
 .../explain-results/function_atan.explain          |     2 +
 .../explain-results/function_atan2.explain         |     2 +
 .../explain-results/function_atanh.explain         |     2 +
 .../explain-results/function_avg.explain           |     2 +
 .../explain-results/function_base64.explain        |     2 +
 .../explain-results/function_bin.explain           |     2 +
 .../explain-results/function_bit_and.explain       |     2 +
 .../explain-results/function_bit_count.explain     |     2 +
 .../explain-results/function_bit_get.explain       |     2 +
 .../explain-results/function_bit_length.explain    |     2 +
 .../explain-results/function_bit_or.explain        |     2 +
 .../explain-results/function_bit_xor.explain       |     2 +
 .../function_bitmap_bit_position.explain           |     2 +
 .../function_bitmap_bucket_number.explain          |     2 +
 .../function_bitmap_construct_agg.explain          |     2 +
 .../explain-results/function_bitmap_count.explain  |     2 +
 .../explain-results/function_bitmap_or_agg.explain |     2 +
 .../explain-results/function_bitwise_not.explain   |     2 +
 .../explain-results/function_bool_and.explain      |     2 +
 .../explain-results/function_bool_or.explain       |     2 +
 .../explain-results/function_bround.explain        |     2 +
 .../explain-results/function_btrim.explain         |     2 +
 ...nction_btrim_with_specified_trim_string.explain |     2 +
 .../explain-results/function_bucket.explain        |     2 +
 .../explain-results/function_call_function.explain |     2 +
 .../function_call_function_orphaned.explain        |     2 +
 .../explain-results/function_cardinality.explain   |     2 +
 .../explain-results/function_ceil.explain          |     2 +
 .../explain-results/function_ceil_scale.explain    |     2 +
 .../explain-results/function_ceiling.explain       |     2 +
 .../explain-results/function_ceiling_scale.explain |     2 +
 .../explain-results/function_char.explain          |     2 +
 .../explain-results/function_char_length.explain   |     2 +
 .../function_character_length.explain              |     2 +
 .../explain-results/function_chr.explain           |     2 +
 .../explain-results/function_coalesce.explain      |     2 +
 .../explain-results/function_col.explain           |     2 +
 .../explain-results/function_collate.explain       |     2 +
 .../explain-results/function_collation.explain     |     2 +
 .../explain-results/function_collect_list.explain  |     2 +
 .../explain-results/function_collect_set.explain   |     2 +
 .../explain-results/function_concat.explain        |     2 +
 .../explain-results/function_concat_ws.explain     |     2 +
 .../explain-results/function_contains.explain      |     2 +
 .../explain-results/function_conv.explain          |     2 +
 ..._convert_timezone_with_source_time_zone.explain |     2 +
 ...nvert_timezone_without_source_time_zone.explain |     2 +
 .../explain-results/function_corr.explain          |     2 +
 .../explain-results/function_cos.explain           |     2 +
 .../explain-results/function_cosh.explain          |     2 +
 .../explain-results/function_cot.explain           |     2 +
 .../explain-results/function_count.explain         |     2 +
 .../explain-results/function_countDistinct.explain |     2 +
 .../explain-results/function_count_if.explain      |     4 +
 .../function_count_min_sketch.explain              |     2 +
 .../explain-results/function_count_typed.explain   |     2 +
 .../explain-results/function_covar_pop.explain     |     2 +
 .../explain-results/function_covar_samp.explain    |     2 +
 .../explain-results/function_crc32.explain         |     2 +
 .../explain-results/function_csc.explain           |     2 +
 .../explain-results/function_cume_dist.explain     |     5 +
 .../explain-results/function_curdate.explain       |     2 +
 .../function_current_catalog.explain               |     2 +
 .../function_current_database.explain              |     2 +
 .../explain-results/function_current_date.explain  |     2 +
 .../function_current_schema.explain                |     2 +
 .../function_current_timestamp.explain             |     2 +
 .../function_current_timezone.explain              |     2 +
 .../explain-results/function_current_user.explain  |     2 +
 .../explain-results/function_date_add.explain      |     2 +
 .../explain-results/function_date_diff.explain     |     2 +
 .../explain-results/function_date_format.explain   |     2 +
 .../function_date_from_unix_date.explain           |     2 +
 .../explain-results/function_date_part.explain     |     2 +
 .../explain-results/function_date_sub.explain      |     2 +
 .../explain-results/function_date_trunc.explain    |     2 +
 .../explain-results/function_dateadd.explain       |     2 +
 .../explain-results/function_datediff.explain      |     2 +
 .../explain-results/function_datepart.explain      |     2 +
 .../explain-results/function_day.explain           |     2 +
 .../explain-results/function_dayname.explain       |     2 +
 .../explain-results/function_dayofmonth.explain    |     2 +
 .../explain-results/function_dayofweek.explain     |     2 +
 .../explain-results/function_dayofyear.explain     |     2 +
 .../explain-results/function_days.explain          |     2 +
 .../explain-results/function_decode.explain        |     2 +
 .../explain-results/function_degrees.explain       |     2 +
 .../explain-results/function_dense_rank.explain    |     5 +
 .../explain-results/function_desc.explain          |     2 +
 .../function_desc_nulls_first.explain              |     2 +
 .../function_desc_nulls_last.explain               |     2 +
 .../query-tests/explain-results/function_e.explain |     2 +
 .../explain-results/function_element_at.explain    |     2 +
 .../explain-results/function_elt.explain           |     2 +
 .../explain-results/function_encode.explain        |     2 +
 .../explain-results/function_endswith.explain      |     2 +
 .../explain-results/function_equal_null.explain    |     2 +
 .../explain-results/function_every.explain         |     2 +
 .../explain-results/function_exists.explain        |     2 +
 .../explain-results/function_exp.explain           |     2 +
 .../explain-results/function_explode.explain       |     3 +
 .../explain-results/function_explode_outer.explain |     3 +
 .../explain-results/function_expm1.explain         |     2 +
 .../explain-results/function_expr.explain          |     2 +
 .../explain-results/function_extract.explain       |     2 +
 .../explain-results/function_factorial.explain     |     2 +
 .../explain-results/function_filter.explain        |     2 +
 .../function_filter_with_pair_input.explain        |     2 +
 .../explain-results/function_find_in_set.explain   |     2 +
 .../function_first_value_with_ignore_nulls.explain |     2 +
 ...function_first_value_with_respect_nulls.explain |     2 +
 .../function_first_with_ignore_nulls.explain       |     2 +
 .../function_first_with_respect_nulls.explain      |     2 +
 .../explain-results/function_flatten.explain       |     2 +
 .../explain-results/function_floor.explain         |     2 +
 .../explain-results/function_floor_scale.explain   |     2 +
 .../explain-results/function_forall.explain        |     2 +
 .../explain-results/function_format_number.explain |     2 +
 .../explain-results/function_from_csv.explain      |     2 +
 .../explain-results/function_from_json.explain     |     2 +
 .../function_from_json_orphaned.explain            |     2 +
 .../function_from_json_with_json_schema.explain    |     2 +
 .../explain-results/function_from_unixtime.explain |     2 +
 .../function_from_utc_timestamp.explain            |     2 +
 .../explain-results/function_from_xml.explain      |     2 +
 .../function_from_xml_with_json_schema.explain     |     2 +
 .../explain-results/function_get.explain           |     2 +
 .../function_get_json_object.explain               |     2 +
 .../explain-results/function_getbit.explain        |     2 +
 .../explain-results/function_greatest.explain      |     2 +
 .../explain-results/function_hash.explain          |     2 +
 .../explain-results/function_hex.explain           |     2 +
 .../function_histogram_numeric.explain             |     2 +
 .../explain-results/function_hour.explain          |     2 +
 .../explain-results/function_hours.explain         |     2 +
 .../explain-results/function_hypot.explain         |     2 +
 .../explain-results/function_ifnull.explain        |     2 +
 .../explain-results/function_ilike.explain         |     2 +
 .../function_ilike_with_escape.explain             |     2 +
 .../explain-results/function_initcap.explain       |     2 +
 .../explain-results/function_inline.explain        |     3 +
 .../explain-results/function_inline_outer.explain  |     3 +
 .../function_input_file_block_length.explain       |     2 +
 .../function_input_file_block_start.explain        |     2 +
 .../function_input_file_name.explain               |     2 +
 .../function_is_valid_variant.explain              |     2 +
 .../function_is_variant_null.explain               |     2 +
 .../explain-results/function_isnan.explain         |     2 +
 .../explain-results/function_isnotnull.explain     |     2 +
 .../explain-results/function_isnull.explain        |     2 +
 .../explain-results/function_java_method.explain   |     2 +
 .../function_json_array_length.explain             |     2 +
 .../function_json_object_keys.explain              |     2 +
 .../explain-results/function_json_tuple.explain    |     3 +
 .../explain-results/function_kurtosis.explain      |     2 +
 .../explain-results/function_lag.explain           |     5 +
 .../explain-results/function_last_day.explain      |     2 +
 .../function_last_value_with_ignore_nulls.explain  |     2 +
 .../function_last_value_with_respect_nulls.explain |     2 +
 .../function_last_with_ignore_nulls.explain        |     2 +
 .../function_last_with_respect_nulls.explain       |     2 +
 .../explain-results/function_lcase.explain         |     2 +
 .../explain-results/function_lead.explain          |     5 +
 .../explain-results/function_least.explain         |     2 +
 .../explain-results/function_left.explain          |     2 +
 .../explain-results/function_len.explain           |     2 +
 .../explain-results/function_length.explain        |     2 +
 .../explain-results/function_levenshtein.explain   |     2 +
 .../function_levenshtein_with_threshold.explain    |     2 +
 .../explain-results/function_like.explain          |     2 +
 .../function_like_with_escape.explain              |     2 +
 .../explain-results/function_lit.explain           |     2 +
 .../explain-results/function_lit_array.explain     |     2 +
 .../explain-results/function_ln.explain            |     2 +
 .../function_localtimestamp.explain                |     2 +
 .../explain-results/function_locate.explain        |     2 +
 .../function_locate_with_pos.explain               |     2 +
 .../explain-results/function_log.explain           |     2 +
 .../explain-results/function_log10.explain         |     2 +
 .../explain-results/function_log1p.explain         |     2 +
 .../explain-results/function_log2.explain          |     2 +
 .../explain-results/function_log_with_base.explain |     2 +
 .../explain-results/function_lower.explain         |     2 +
 .../explain-results/function_lpad.explain          |     2 +
 .../explain-results/function_lpad_binary.explain   |     2 +
 .../explain-results/function_ltrim.explain         |     2 +
 .../function_ltrim_with_pattern.explain            |     2 +
 .../explain-results/function_make_date.explain     |     2 +
 .../function_make_dt_interval.explain              |     2 +
 .../function_make_dt_interval_days.explain         |     2 +
 .../function_make_dt_interval_days_hours.explain   |     2 +
 ...nction_make_dt_interval_days_hours_mins.explain |     2 +
 ...n_make_dt_interval_days_hours_mins_secs.explain |     2 +
 .../explain-results/function_make_interval.explain |     2 +
 .../function_make_interval_years.explain           |     2 +
 .../function_make_interval_years_months.explain    |     2 +
 ...nction_make_interval_years_months_weeks.explain |     2 +
 ...n_make_interval_years_months_weeks_days.explain |     2 +
 ..._interval_years_months_weeks_days_hours.explain |     2 +
 ...rval_years_months_weeks_days_hours_mins.explain |     2 +
 ...years_months_weeks_days_hours_mins_secs.explain |     2 +
 ...nction_make_timestamp_ltz_with_timezone.explain |     2 +
 ...ion_make_timestamp_ltz_without_timezone.explain |     2 +
 .../function_make_timestamp_ntz.explain            |     2 +
 .../function_make_timestamp_with_timezone.explain  |     2 +
 ...unction_make_timestamp_without_timezone.explain |     2 +
 .../function_make_ym_interval.explain              |     2 +
 .../function_make_ym_interval_years.explain        |     2 +
 .../function_make_ym_interval_years_months.explain |     2 +
 .../explain-results/function_map.explain           |     2 +
 .../explain-results/function_map_concat.explain    |     2 +
 .../function_map_contains_key.explain              |     2 +
 .../explain-results/function_map_entries.explain   |     2 +
 .../explain-results/function_map_filter.explain    |     2 +
 .../function_map_from_arrays.explain               |     2 +
 .../function_map_from_entries.explain              |     2 +
 .../explain-results/function_map_keys.explain      |     2 +
 .../explain-results/function_map_values.explain    |     2 +
 .../explain-results/function_map_zip_with.explain  |     2 +
 .../explain-results/function_mask.explain          |     2 +
 .../function_mask_with_specific_upperChar.explain  |     2 +
 ..._mask_with_specific_upperChar_lowerChar.explain |     2 +
 ..._specific_upperChar_lowerChar_digitChar.explain |     2 +
 ...upperChar_lowerChar_digitChar_otherChar.explain |     2 +
 .../explain-results/function_max.explain           |     2 +
 .../explain-results/function_max_by.explain        |     2 +
 .../explain-results/function_md5.explain           |     2 +
 .../explain-results/function_median.explain        |     2 +
 .../explain-results/function_min.explain           |     2 +
 .../explain-results/function_min_by.explain        |     2 +
 .../explain-results/function_minute.explain        |     2 +
 .../explain-results/function_mode.explain          |     2 +
 .../explain-results/function_mode_orphaned.explain |     2 +
 .../function_monotonically_increasing_id.explain   |     2 +
 .../explain-results/function_month.explain         |     2 +
 .../explain-results/function_monthname.explain     |     2 +
 .../explain-results/function_months.explain        |     2 +
 .../function_months_between.explain                |     2 +
 .../function_months_between_with_roundoff.explain  |     2 +
 .../explain-results/function_named_struct.explain  |     2 +
 .../explain-results/function_nanvl.explain         |     2 +
 .../explain-results/function_negate.explain        |     2 +
 .../explain-results/function_negative.explain      |     2 +
 .../explain-results/function_next_day.explain      |     2 +
 .../explain-results/function_now.explain           |     2 +
 .../explain-results/function_nth_value.explain     |     5 +
 .../explain-results/function_ntile.explain         |     5 +
 .../explain-results/function_nullif.explain        |     2 +
 .../explain-results/function_nvl.explain           |     2 +
 .../explain-results/function_nvl2.explain          |     2 +
 .../explain-results/function_octet_length.explain  |     2 +
 .../explain-results/function_overlay.explain       |     2 +
 .../function_overlay_with_len.explain              |     2 +
 .../explain-results/function_parse_json.explain    |     2 +
 .../explain-results/function_parse_url.explain     |     2 +
 .../function_parse_url_with_key.explain            |     2 +
 .../explain-results/function_percent_rank.explain  |     5 +
 .../function_percentile_approx.explain             |     2 +
 .../function_percentile_with_frequency.explain     |     2 +
 .../function_percentile_without_frequency.explain  |     2 +
 .../explain-results/function_pi.explain            |     2 +
 .../explain-results/function_pmod.explain          |     2 +
 .../explain-results/function_posexplode.explain    |     3 +
 .../function_posexplode_outer.explain              |     3 +
 .../explain-results/function_position.explain      |     2 +
 .../function_position_with_start.explain           |     2 +
 .../explain-results/function_positive.explain      |     2 +
 .../explain-results/function_pow.explain           |     2 +
 .../explain-results/function_power.explain         |     2 +
 .../explain-results/function_printf.explain        |     2 +
 .../explain-results/function_product.explain       |     2 +
 .../explain-results/function_quarter.explain       |     2 +
 .../explain-results/function_radians.explain       |     2 +
 .../explain-results/function_raise_error.explain   |     2 +
 .../function_rand_with_seed.explain                |     2 +
 .../function_randn_with_seed.explain               |     2 +
 .../function_random_with_seed.explain              |     2 +
 .../explain-results/function_rank.explain          |     5 +
 .../explain-results/function_reduce.explain        |     2 +
 .../explain-results/function_reflect.explain       |     2 +
 .../explain-results/function_regexp.explain        |     2 +
 .../explain-results/function_regexp_count.explain  |     2 +
 .../function_regexp_extract.explain                |     2 +
 ...gexp_extract_all_with_regex_group_index.explain |     2 +
 ...p_extract_all_without_regex_group_index.explain |     2 +
 ...ion_regexp_instr_with_regex_group_index.explain |     2 +
 ..._regexp_instr_without_regex_group_index.explain |     2 +
 .../explain-results/function_regexp_like.explain   |     2 +
 .../function_regexp_replace.explain                |     2 +
 .../explain-results/function_regexp_substr.explain |     3 +
 .../explain-results/function_regr_avgx.explain     |     2 +
 .../explain-results/function_regr_avgy.explain     |     2 +
 .../explain-results/function_regr_count.explain    |     2 +
 .../function_regr_intercept.explain                |     2 +
 .../explain-results/function_regr_r2.explain       |     2 +
 .../explain-results/function_regr_slope.explain    |     2 +
 .../explain-results/function_regr_sxx.explain      |     2 +
 .../explain-results/function_regr_sxy.explain      |     2 +
 .../explain-results/function_regr_syy.explain      |     2 +
 .../explain-results/function_replace.explain       |     2 +
 .../function_replace_with_specified_string.explain |     2 +
 .../explain-results/function_reverse.explain       |     2 +
 .../explain-results/function_right.explain         |     2 +
 .../explain-results/function_rint.explain          |     2 +
 .../explain-results/function_rlike.explain         |     2 +
 .../explain-results/function_round.explain         |     2 +
 .../explain-results/function_row_number.explain    |     5 +
 .../explain-results/function_rpad.explain          |     2 +
 .../explain-results/function_rpad_binary.explain   |     2 +
 .../explain-results/function_rtrim.explain         |     2 +
 .../function_rtrim_with_pattern.explain            |     2 +
 .../explain-results/function_schema_of_csv.explain |     2 +
 .../function_schema_of_json.explain                |     2 +
 .../function_schema_of_json_with_options.explain   |     2 +
 .../function_schema_of_variant.explain             |     2 +
 .../function_schema_of_variant_agg.explain         |     2 +
 .../explain-results/function_sec.explain           |     2 +
 .../explain-results/function_second.explain        |     2 +
 .../explain-results/function_sentences.explain     |     2 +
 .../function_sentences_with_language.explain       |     2 +
 ...ion_sentences_with_language_and_country.explain |     2 +
 .../explain-results/function_sequence.explain      |     2 +
 .../explain-results/function_session_user.explain  |     2 +
 .../function_session_window.explain                |     4 +
 .../function_session_window_orphaned.explain       |     4 +
 .../explain-results/function_sha.explain           |     2 +
 .../explain-results/function_sha1.explain          |     2 +
 .../explain-results/function_sha2.explain          |     2 +
 .../explain-results/function_shiftleft.explain     |     2 +
 .../explain-results/function_shiftright.explain    |     2 +
 .../function_shiftrightunsigned.explain            |     2 +
 .../explain-results/function_sign.explain          |     2 +
 .../explain-results/function_signum.explain        |     2 +
 .../explain-results/function_sin.explain           |     2 +
 .../explain-results/function_sinh.explain          |     2 +
 .../explain-results/function_size.explain          |     2 +
 .../explain-results/function_skewness.explain      |     2 +
 .../explain-results/function_slice.explain         |     2 +
 .../explain-results/function_some.explain          |     2 +
 .../explain-results/function_sort_array.explain    |     2 +
 .../function_spark_partition_id.explain            |     2 +
 .../explain-results/function_split.explain         |     2 +
 .../explain-results/function_split_part.explain    |     2 +
 .../function_split_using_columns.explain           |     2 +
 .../function_split_with_limit.explain              |     2 +
 ...function_split_with_limit_using_columns.explain |     2 +
 .../explain-results/function_sqrt.explain          |     2 +
 .../explain-results/function_stack.explain         |     3 +
 .../explain-results/function_startswith.explain    |     2 +
 .../explain-results/function_std.explain           |     2 +
 .../explain-results/function_stddev.explain        |     2 +
 .../explain-results/function_stddev_pop.explain    |     2 +
 .../explain-results/function_stddev_samp.explain   |     2 +
 .../explain-results/function_str_to_map.explain    |     2 +
 ...to_map_with_pair_and_keyValue_delimiter.explain |     2 +
 ...function_str_to_map_with_pair_delimiter.explain |     2 +
 .../explain-results/function_struct.explain        |     2 +
 .../explain-results/function_substr.explain        |     2 +
 .../function_substr_with_len.explain               |     2 +
 .../explain-results/function_substring.explain     |     2 +
 .../function_substring_index.explain               |     2 +
 .../function_substring_using_columns.explain       |     2 +
 .../function_substring_with_columns.explain        |     2 +
 .../explain-results/function_sum.explain           |     2 +
 .../explain-results/function_sum_distinct.explain  |     2 +
 .../explain-results/function_tan.explain           |     2 +
 .../explain-results/function_tanh.explain          |     2 +
 .../explain-results/function_timestamp_add.explain |     2 +
 .../function_timestamp_diff.explain                |     2 +
 .../function_timestamp_micros.explain              |     2 +
 .../function_timestamp_millis.explain              |     2 +
 .../function_timestamp_seconds.explain             |     2 +
 .../explain-results/function_to_binary.explain     |     2 +
 .../function_to_binary_with_format.explain         |     2 +
 .../explain-results/function_to_char.explain       |     2 +
 .../explain-results/function_to_csv.explain        |     2 +
 .../explain-results/function_to_date.explain       |     2 +
 .../function_to_date_with_format.explain           |     2 +
 .../explain-results/function_to_json.explain       |     2 +
 .../explain-results/function_to_number.explain     |     2 +
 .../explain-results/function_to_timestamp.explain  |     2 +
 .../function_to_timestamp_ltz.explain              |     2 +
 .../function_to_timestamp_ltz_with_format.explain  |     2 +
 .../function_to_timestamp_ntz.explain              |     2 +
 .../function_to_timestamp_ntz_with_format.explain  |     2 +
 .../function_to_timestamp_with_format.explain      |     2 +
 .../function_to_unix_timestamp.explain             |     2 +
 .../function_to_unix_timestamp_with_format.explain |     2 +
 .../function_to_utc_timestamp.explain              |     2 +
 .../explain-results/function_to_varchar.explain    |     2 +
 .../explain-results/function_transform.explain     |     2 +
 .../function_transform_keys.explain                |     2 +
 .../function_transform_values.explain              |     2 +
 .../function_transform_with_index.explain          |     2 +
 .../explain-results/function_translate.explain     |     2 +
 .../explain-results/function_trim.explain          |     2 +
 .../function_trim_with_pattern.explain             |     2 +
 .../explain-results/function_trunc.explain         |     2 +
 .../explain-results/function_try_add.explain       |     2 +
 .../function_try_aes_decrypt.explain               |     2 +
 .../function_try_aes_decrypt_with_mode.explain     |     2 +
 ...ction_try_aes_decrypt_with_mode_padding.explain |     2 +
 ...n_try_aes_decrypt_with_mode_padding_aad.explain |     2 +
 .../explain-results/function_try_avg.explain       |     2 +
 .../explain-results/function_try_divide.explain    |     2 +
 .../function_try_element_at_array.explain          |     2 +
 .../function_try_element_at_map.explain            |     2 +
 .../function_try_make_interval_years.explain       |     2 +
 ...function_try_make_interval_years_months.explain |     2 +
 ...on_try_make_interval_years_months_weeks.explain |     2 +
 ...y_make_interval_years_months_weeks_days.explain |     2 +
 ..._interval_years_months_weeks_days_hours.explain |     2 +
 ...rval_years_months_weeks_days_hours_mins.explain |     2 +
 ...years_months_weeks_days_hours_mins_secs.explain |     2 +
 ...on_try_make_timestamp_ltz_with_timezone.explain |     2 +
 ...try_make_timestamp_ltz_without_timezone.explain |     2 +
 .../function_try_make_timestamp_ntz.explain        |     2 +
 ...nction_try_make_timestamp_with_timezone.explain |     2 +
 ...ion_try_make_timestamp_without_timezone.explain |     2 +
 .../explain-results/function_try_multiply.explain  |     2 +
 .../function_try_parse_json.explain                |     2 +
 .../explain-results/function_try_parse_url.explain |     2 +
 .../function_try_parse_url_with_key.explain        |     2 +
 .../explain-results/function_try_reflect.explain   |     2 +
 .../explain-results/function_try_subtract.explain  |     2 +
 .../explain-results/function_try_sum.explain       |     2 +
 .../explain-results/function_try_to_binary.explain |     2 +
 .../function_try_to_binary_without_format.explain  |     2 +
 .../explain-results/function_try_to_date.explain   |     2 +
 .../function_try_to_date_with_format.explain       |     2 +
 .../explain-results/function_try_to_number.explain |     2 +
 .../function_try_to_timestamp.explain              |     2 +
 ...unction_try_to_timestamp_without_format.explain |     2 +
 .../function_try_url_decode.explain                |     2 +
 .../function_try_variant_get.explain               |     2 +
 .../explain-results/function_typedLit.explain      |     2 +
 .../explain-results/function_typeof.explain        |     2 +
 .../explain-results/function_ucase.explain         |     2 +
 .../explain-results/function_unbase64.explain      |     2 +
 .../explain-results/function_unhex.explain         |     2 +
 .../explain-results/function_unix_date.explain     |     2 +
 .../explain-results/function_unix_micros.explain   |     2 +
 .../explain-results/function_unix_millis.explain   |     2 +
 .../explain-results/function_unix_seconds.explain  |     2 +
 .../function_unix_timestamp.explain                |     2 +
 .../function_unix_timestamp_with_format.explain    |     2 +
 .../explain-results/function_upper.explain         |     2 +
 .../explain-results/function_url_decode.explain    |     2 +
 .../explain-results/function_url_encode.explain    |     2 +
 .../explain-results/function_user.explain          |     2 +
 .../explain-results/function_var_pop.explain       |     2 +
 .../explain-results/function_var_samp.explain      |     2 +
 .../explain-results/function_variance.explain      |     2 +
 .../explain-results/function_variant_get.explain   |     2 +
 .../explain-results/function_weekday.explain       |     2 +
 .../explain-results/function_weekofyear.explain    |     2 +
 .../explain-results/function_window.explain        |     4 +
 .../explain-results/function_window_time.explain   |     4 +
 .../explain-results/function_xpath.explain         |     2 +
 .../explain-results/function_xpath_boolean.explain |     2 +
 .../explain-results/function_xpath_double.explain  |     2 +
 .../explain-results/function_xpath_float.explain   |     2 +
 .../explain-results/function_xpath_int.explain     |     2 +
 .../explain-results/function_xpath_long.explain    |     2 +
 .../explain-results/function_xpath_number.explain  |     2 +
 .../explain-results/function_xpath_short.explain   |     2 +
 .../explain-results/function_xpath_string.explain  |     2 +
 .../explain-results/function_xxhash64.explain      |     2 +
 .../explain-results/function_year.explain          |     2 +
 .../explain-results/function_years.explain         |     2 +
 .../explain-results/function_zip_with.explain      |     2 +
 .../explain-results/groupby_agg.explain            |     2 +
 .../explain-results/groupby_agg_columns.explain    |     2 +
 .../explain-results/groupby_agg_orphaned.explain   |     2 +
 .../explain-results/groupby_agg_string.explain     |     2 +
 .../explain-results/groupby_avg.explain            |     2 +
 .../explain-results/groupby_count.explain          |     2 +
 .../explain-results/groupby_max.explain            |     2 +
 .../explain-results/groupby_mean.explain           |     2 +
 .../explain-results/groupby_min.explain            |     2 +
 .../explain-results/groupby_sum.explain            |     2 +
 .../explain-results/groupingSets.explain           |     4 +
 .../grouping_and_grouping_id.explain               |     4 +
 .../query-tests/explain-results/hint.explain       |     2 +
 .../explain-results/hll_sketch_agg.explain         |     2 +
 .../hll_sketch_agg_with_columnName.explain         |     2 +
 ...ketch_agg_with_columnName_lgConfigK_int.explain |     2 +
 .../hll_sketch_agg_with_column_lgConfigK.explain   |     2 +
 ...ll_sketch_agg_with_column_lgConfigK_int.explain |     2 +
 .../explain-results/hll_union_agg.explain          |     2 +
 .../hll_union_agg_with_columnName.explain          |     2 +
 ...umnName_allowDifferentLgConfigK_boolean.explain |     2 +
 ...agg_with_column_allowDifferentLgConfigK.explain |     2 +
 ..._column_allowDifferentLgConfigK_boolean.explain |     2 +
 .../query-tests/explain-results/intersect.explain  |     3 +
 .../explain-results/intersectAll.explain           |     3 +
 .../explain-results/join_condition.explain         |     5 +
 .../explain-results/join_inner_condition.explain   |     5 +
 .../join_inner_no_condition.explain                |     3 +
 .../join_inner_using_multiple_col_array.explain    |     4 +
 .../join_inner_using_multiple_col_seq.explain      |     4 +
 .../join_inner_using_single_col.explain            |     4 +
 .../join_using_multiple_col_array.explain          |     4 +
 .../join_using_multiple_col_seq.explain            |     4 +
 .../explain-results/join_using_single_col.explain  |     4 +
 .../explain-results/json_from_dataset.explain      |     1 +
 .../query-tests/explain-results/limit.explain      |     3 +
 .../explain-results/melt_no_values.explain         |     2 +
 .../explain-results/melt_values.explain            |     2 +
 .../query-tests/explain-results/offset.explain     |     2 +
 .../explain-results/orderBy_columns.explain        |     2 +
 .../explain-results/orderBy_strings.explain        |     2 +
 .../query-tests/explain-results/pivot.explain      |     4 +
 .../pivot_without_column_values.explain            |     4 +
 .../query-tests/explain-results/range.explain      |     1 +
 .../query-tests/explain-results/read.explain       |     1 +
 .../explain-results/read_changes.explain           |     2 +
 .../read_changes_with_options.explain              |     2 +
 .../query-tests/explain-results/read_csv.explain   |     1 +
 .../query-tests/explain-results/read_jdbc.explain  |     1 +
 .../read_jdbc_with_partition.explain               |     1 +
 .../read_jdbc_with_predicates.explain              |     1 +
 .../query-tests/explain-results/read_json.explain  |     1 +
 .../query-tests/explain-results/read_orc.explain   |     1 +
 .../explain-results/read_parquet.explain           |     1 +
 .../query-tests/explain-results/read_table.explain |     2 +
 .../query-tests/explain-results/read_text.explain  |     1 +
 .../explain-results/relation_extension.explain     |     1 +
 .../relation_extension_deprecated.explain          |     1 +
 .../explain-results/repartition.explain            |     2 +
 .../explain-results/repartitionById.explain        |     2 +
 .../repartitionByRange_expressions.explain         |     2 +
 ...itionByRange_num_partitions_expressions.explain |     2 +
 .../repartition_expressions.explain                |     2 +
 .../repartition_num_partitions_expressions.explain |     2 +
 .../query-tests/explain-results/replace.explain    |     2 +
 .../explain-results/rollup_column.explain          |     4 +
 .../explain-results/rollup_string.explain          |     4 +
 .../query-tests/explain-results/sampleBy.explain   |     2 +
 .../explain-results/sample_fraction_seed.explain   |     2 +
 .../sample_withReplacement_fraction_seed.explain   |     2 +
 .../query-tests/explain-results/select.explain     |     2 +
 .../query-tests/explain-results/selectExpr.explain |     2 +
 .../explain-results/select_collated_string.explain |     2 +
 .../explain-results/select_strings.explain         |     2 +
 .../explain-results/select_typed_1-arg.explain     |     3 +
 .../explain-results/select_typed_2-arg.explain     |     2 +
 .../explain-results/select_typed_3-arg.explain     |     2 +
 .../explain-results/select_typed_4-arg.explain     |     2 +
 .../explain-results/select_typed_5-arg.explain     |     2 +
 .../sortWithinPartitions_columns.explain           |     2 +
 .../sortWithinPartitions_strings.explain           |     2 +
 .../explain-results/sort_columns.explain           |     2 +
 .../explain-results/sort_strings.explain           |     2 +
 .../streaming_changes_API_with_options.explain     |     2 +
 .../streaming_table_API_with_options.explain       |     2 +
 .../query-tests/explain-results/summary.explain    |     5 +
 .../query-tests/explain-results/table.explain      |     2 +
 .../explain-results/table_API_with_options.explain |     2 +
 .../explain-results/test_broadcast.explain         |     5 +
 .../query-tests/explain-results/to.explain         |     2 +
 .../query-tests/explain-results/toDF.explain       |     2 +
 .../query-tests/explain-results/toJSON.explain     |     2 +
 .../explain-results/to_avro_with_schema.explain    |     2 +
 .../explain-results/to_avro_without_schema.explain |     2 +
 .../to_protobuf_messageClassName.explain           |     3 +
 ..._protobuf_messageClassName_descFilePath.explain |     3 +
 ...f_messageClassName_descFilePath_options.explain |     3 +
 .../to_protobuf_messageClassName_options.explain   |     3 +
 .../query-tests/explain-results/union.explain      |     3 +
 .../query-tests/explain-results/unionAll.explain   |     3 +
 .../explain-results/unionByName.explain            |     6 +
 .../unionByName_allowMissingColumns.explain        |     5 +
 .../explain-results/unpivot_no_values.explain      |     2 +
 .../explain-results/unpivot_values.explain         |     2 +
 .../explain-results/where_column.explain           |     2 +
 .../query-tests/explain-results/where_expr.explain |     2 +
 .../explain-results/width_bucket.explain           |     2 +
 .../query-tests/explain-results/window.explain     |     8 +
 .../withColumnRenamed_java_map.explain             |     2 +
 .../withColumnRenamed_scala_map.explain            |     2 +
 .../withColumnRenamed_single.explain               |     2 +
 .../explain-results/withColumn_single.explain      |     2 +
 .../explain-results/withColumns_java_map.explain   |     2 +
 .../explain-results/withColumns_scala_map.explain  |     2 +
 .../explain-results/withMetadata.explain           |     2 +
 .../explain-results/zipWithIndex.explain           |     4 +
 .../zipWithIndex_custom_column.explain             |     4 +
 .../query-tests/queries/alias_string.json          |    16 +
 .../query-tests/queries/alias_string.proto.bin     |   Bin 0 -> 55 bytes
 .../query-tests/queries/alias_symbol.json          |    16 +
 .../query-tests/queries/alias_symbol.proto.bin     |   Bin 0 -> 54 bytes
 .../test/resources/query-tests/queries/apply.json  |    38 +
 .../resources/query-tests/queries/apply.proto.bin  |   Bin 0 -> 233 bytes
 .../resources/query-tests/queries/as_string.json   |    16 +
 .../query-tests/queries/as_string.proto.bin        |   Bin 0 -> 54 bytes
 .../resources/query-tests/queries/as_symbol.json   |    16 +
 .../query-tests/queries/as_symbol.proto.bin        |   Bin 0 -> 54 bytes
 .../query-tests/queries/between_expr.json          |    37 +
 .../query-tests/queries/between_expr.proto.bin     |   Bin 0 -> 272 bytes
 .../resources/query-tests/queries/coalesce.json    |    17 +
 .../query-tests/queries/coalesce.proto.bin         |   Bin 0 -> 53 bytes
 .../test/resources/query-tests/queries/col.json    |    60 +
 .../resources/query-tests/queries/col.proto.bin    |   Bin 0 -> 430 bytes
 .../resources/query-tests/queries/colRegex.json    |    38 +
 .../query-tests/queries/colRegex.proto.bin         |   Bin 0 -> 249 bytes
 .../resources/query-tests/queries/column_add.json  |    81 +
 .../query-tests/queries/column_add.proto.bin       |   Bin 0 -> 709 bytes
 .../query-tests/queries/column_alias.json          |    59 +
 .../query-tests/queries/column_alias.proto.bin     |   Bin 0 -> 521 bytes
 .../resources/query-tests/queries/column_and.json  |   169 +
 .../query-tests/queries/column_and.proto.bin       |   Bin 0 -> 1451 bytes
 .../query-tests/queries/column_apply.json          |    80 +
 .../query-tests/queries/column_apply.proto.bin     |   Bin 0 -> 713 bytes
 .../query-tests/queries/column_as_multi.json       |    59 +
 .../query-tests/queries/column_as_multi.proto.bin  |   Bin 0 -> 549 bytes
 .../queries/column_as_with_metadata.json           |    60 +
 .../queries/column_as_with_metadata.proto.bin      |   Bin 0 -> 555 bytes
 .../resources/query-tests/queries/column_asc.json  |    42 +
 .../query-tests/queries/column_asc.proto.bin       |   Bin 0 -> 350 bytes
 .../queries/column_asc_nulls_first.json            |    42 +
 .../queries/column_asc_nulls_first.proto.bin       |   Bin 0 -> 350 bytes
 .../query-tests/queries/column_asc_nulls_last.json |    42 +
 .../queries/column_asc_nulls_last.proto.bin        |   Bin 0 -> 350 bytes
 .../query-tests/queries/column_between.json        |   169 +
 .../query-tests/queries/column_between.proto.bin   |   Bin 0 -> 1447 bytes
 .../query-tests/queries/column_bitwiseAND.json     |    81 +
 .../queries/column_bitwiseAND.proto.bin            |   Bin 0 -> 714 bytes
 .../query-tests/queries/column_bitwiseOR.json      |    81 +
 .../query-tests/queries/column_bitwiseOR.proto.bin |   Bin 0 -> 711 bytes
 .../query-tests/queries/column_bitwiseXOR.json     |    81 +
 .../queries/column_bitwiseXOR.proto.bin            |   Bin 0 -> 713 bytes
 .../resources/query-tests/queries/column_cast.json |    62 +
 .../query-tests/queries/column_cast.proto.bin      |   Bin 0 -> 522 bytes
 .../query-tests/queries/column_contains.json       |    81 +
 .../query-tests/queries/column_contains.proto.bin  |   Bin 0 -> 719 bytes
 .../resources/query-tests/queries/column_desc.json |    42 +
 .../query-tests/queries/column_desc.proto.bin      |   Bin 0 -> 350 bytes
 .../queries/column_desc_nulls_first.json           |    42 +
 .../queries/column_desc_nulls_first.proto.bin      |   Bin 0 -> 350 bytes
 .../queries/column_desc_nulls_last.json            |    42 +
 .../queries/column_desc_nulls_last.proto.bin       |   Bin 0 -> 350 bytes
 .../query-tests/queries/column_divide.json         |    81 +
 .../query-tests/queries/column_divide.proto.bin    |   Bin 0 -> 708 bytes
 .../query-tests/queries/column_dropFields.json     |    81 +
 .../queries/column_dropFields.proto.bin            |   Bin 0 -> 713 bytes
 .../query-tests/queries/column_endsWith.json       |    81 +
 .../query-tests/queries/column_endsWith.proto.bin  |   Bin 0 -> 723 bytes
 .../query-tests/queries/column_eqNullSafe.json     |    81 +
 .../queries/column_eqNullSafe.proto.bin            |   Bin 0 -> 722 bytes
 .../query-tests/queries/column_equals.json         |    81 +
 .../query-tests/queries/column_equals.proto.bin    |   Bin 0 -> 713 bytes
 .../resources/query-tests/queries/column_geq.json  |    81 +
 .../query-tests/queries/column_geq.proto.bin       |   Bin 0 -> 716 bytes
 .../query-tests/queries/column_getField.json       |    80 +
 .../query-tests/queries/column_getField.proto.bin  |   Bin 0 -> 703 bytes
 .../query-tests/queries/column_getItem.json        |    80 +
 .../query-tests/queries/column_getItem.proto.bin   |   Bin 0 -> 702 bytes
 .../resources/query-tests/queries/column_gt.json   |    81 +
 .../query-tests/queries/column_gt.proto.bin        |   Bin 0 -> 712 bytes
 .../query-tests/queries/column_ilike.json          |    81 +
 .../query-tests/queries/column_ilike.proto.bin     |   Bin 0 -> 709 bytes
 .../query-tests/queries/column_isNaN.json          |    60 +
 .../query-tests/queries/column_isNaN.proto.bin     |   Bin 0 -> 528 bytes
 .../query-tests/queries/column_isNotNull.json      |    60 +
 .../query-tests/queries/column_isNotNull.proto.bin |   Bin 0 -> 536 bytes
 .../query-tests/queries/column_isNull.json         |    60 +
 .../query-tests/queries/column_isNull.proto.bin    |   Bin 0 -> 530 bytes
 .../resources/query-tests/queries/column_isin.json |   123 +
 .../query-tests/queries/column_isin.proto.bin      |   Bin 0 -> 1071 bytes
 .../resources/query-tests/queries/column_leq.json  |    81 +
 .../query-tests/queries/column_leq.proto.bin       |   Bin 0 -> 713 bytes
 .../resources/query-tests/queries/column_like.json |    81 +
 .../query-tests/queries/column_like.proto.bin      |   Bin 0 -> 709 bytes
 .../resources/query-tests/queries/column_lt.json   |    81 +
 .../query-tests/queries/column_lt.proto.bin        |   Bin 0 -> 709 bytes
 .../query-tests/queries/column_modulo.json         |    81 +
 .../query-tests/queries/column_modulo.proto.bin    |   Bin 0 -> 709 bytes
 .../query-tests/queries/column_multiply.json       |    81 +
 .../query-tests/queries/column_multiply.proto.bin  |   Bin 0 -> 710 bytes
 .../resources/query-tests/queries/column_not.json  |    60 +
 .../query-tests/queries/column_not.proto.bin       |   Bin 0 -> 529 bytes
 .../query-tests/queries/column_not_equals.json     |   104 +
 .../queries/column_not_equals.proto.bin            |   Bin 0 -> 904 bytes
 .../resources/query-tests/queries/column_or.json   |   169 +
 .../query-tests/queries/column_or.proto.bin        |   Bin 0 -> 1450 bytes
 .../query-tests/queries/column_rlike.json          |    81 +
 .../query-tests/queries/column_rlike.proto.bin     |   Bin 0 -> 712 bytes
 .../resources/query-tests/queries/column_star.json |    36 +
 .../query-tests/queries/column_star.proto.bin      |   Bin 0 -> 338 bytes
 .../queries/column_star_with_target.json           |    37 +
 .../queries/column_star_with_target.proto.bin      |   Bin 0 -> 343 bytes
 .../query-tests/queries/column_startsWith.json     |    81 +
 .../queries/column_startsWith.proto.bin            |   Bin 0 -> 729 bytes
 .../query-tests/queries/column_substr.json         |   102 +
 .../query-tests/queries/column_substr.proto.bin    |   Bin 0 -> 890 bytes
 .../query-tests/queries/column_subtract.json       |    81 +
 .../query-tests/queries/column_subtract.proto.bin  |   Bin 0 -> 710 bytes
 .../query-tests/queries/column_try_cast.json       |    63 +
 .../query-tests/queries/column_try_cast.proto.bin  |   Bin 0 -> 528 bytes
 .../query-tests/queries/column_unary_minus.json    |    60 +
 .../queries/column_unary_minus.proto.bin           |   Bin 0 -> 537 bytes
 .../query-tests/queries/column_when_otherwise.json |   232 +
 .../queries/column_when_otherwise.proto.bin        |   Bin 0 -> 1999 bytes
 .../query-tests/queries/column_withField.json      |    81 +
 .../query-tests/queries/column_withField.proto.bin |   Bin 0 -> 712 bytes
 .../resources/query-tests/queries/crossJoin.json   |    24 +
 .../query-tests/queries/crossJoin.proto.bin        |   Bin 0 -> 98 bytes
 .../resources/query-tests/queries/crosstab.json    |    17 +
 .../query-tests/queries/crosstab.proto.bin         |   Bin 0 -> 55 bytes
 .../query-tests/queries/csv_from_dataset.json      |    39 +
 .../query-tests/queries/csv_from_dataset.proto.bin |   Bin 0 -> 169 bytes
 .../resources/query-tests/queries/cube_column.json |   126 +
 .../query-tests/queries/cube_column.proto.bin      |   Bin 0 -> 937 bytes
 .../resources/query-tests/queries/cube_string.json |   128 +
 .../query-tests/queries/cube_string.proto.bin      |   Bin 0 -> 957 bytes
 .../resources/query-tests/queries/describe.json    |    16 +
 .../query-tests/queries/describe.proto.bin         |   Bin 0 -> 56 bytes
 .../resources/query-tests/queries/distinct.json    |    17 +
 .../query-tests/queries/distinct.proto.bin         |   Bin 0 -> 52 bytes
 .../test/resources/query-tests/queries/drop.json   |    17 +
 .../resources/query-tests/queries/drop.proto.bin   |   Bin 0 -> 58 bytes
 .../query-tests/queries/dropDuplicates.json        |    17 +
 .../query-tests/queries/dropDuplicates.proto.bin   |   Bin 0 -> 52 bytes
 .../queries/dropDuplicates_names_array.json        |    17 +
 .../queries/dropDuplicates_names_array.proto.bin   |   Bin 0 -> 57 bytes
 .../queries/dropDuplicates_names_seq.json          |    17 +
 .../queries/dropDuplicates_names_seq.proto.bin     |   Bin 0 -> 56 bytes
 .../queries/dropDuplicates_varargs.json            |    17 +
 .../queries/dropDuplicates_varargs.proto.bin       |   Bin 0 -> 60 bytes
 .../query-tests/queries/drop_multiple_column.json  |    58 +
 .../queries/drop_multiple_column.proto.bin         |   Bin 0 -> 421 bytes
 .../query-tests/queries/drop_multiple_strings.json |    16 +
 .../queries/drop_multiple_strings.proto.bin        |   Bin 0 -> 59 bytes
 .../query-tests/queries/drop_single_column.json    |    37 +
 .../queries/drop_single_column.proto.bin           |   Bin 0 -> 235 bytes
 .../query-tests/queries/drop_single_string.json    |    16 +
 .../queries/drop_single_string.proto.bin           |   Bin 0 -> 52 bytes
 .../test/resources/query-tests/queries/except.json |    25 +
 .../resources/query-tests/queries/except.proto.bin |   Bin 0 -> 94 bytes
 .../resources/query-tests/queries/exceptAll.json   |    25 +
 .../query-tests/queries/exceptAll.proto.bin        |   Bin 0 -> 94 bytes
 .../query-tests/queries/expression_extension.json  |    26 +
 .../queries/expression_extension.proto.bin         |   Bin 0 -> 127 bytes
 .../queries/expression_extension_deprecated.json   |    26 +
 .../expression_extension_deprecated.proto.bin      |   Bin 0 -> 127 bytes
 .../test/resources/query-tests/queries/fill.json   |    19 +
 .../resources/query-tests/queries/fill.proto.bin   |   Bin 0 -> 57 bytes
 .../test/resources/query-tests/queries/filter.json |    81 +
 .../resources/query-tests/queries/filter.proto.bin |   Bin 0 -> 606 bytes
 .../resources/query-tests/queries/filter_expr.json |    37 +
 .../query-tests/queries/filter_expr.proto.bin      |   Bin 0 -> 252 bytes
 .../resources/query-tests/queries/freqItems.json   |    17 +
 .../query-tests/queries/freqItems.proto.bin        |   Bin 0 -> 65 bytes
 .../queries/from_avro_with_options.json            |   188 +
 .../queries/from_avro_with_options.proto.bin       |   Bin 0 -> 1403 bytes
 .../queries/from_avro_without_options.json         |    81 +
 .../queries/from_avro_without_options.proto.bin    |   Bin 0 -> 579 bytes
 ...rom_protobuf_messageClassName_descFilePath.json |   102 +
 ...rotobuf_messageClassName_descFilePath.proto.bin |   Bin 0 -> 994 bytes
 ...obuf_messageClassName_descFilePath_options.json |   167 +
 ...messageClassName_descFilePath_options.proto.bin |   Bin 0 -> 1530 bytes
 .../query-tests/queries/function_abs.json          |    60 +
 .../query-tests/queries/function_abs.proto.bin     |   Bin 0 -> 531 bytes
 .../query-tests/queries/function_acos.json         |    60 +
 .../query-tests/queries/function_acos.proto.bin    |   Bin 0 -> 534 bytes
 .../query-tests/queries/function_acosh.json        |    60 +
 .../query-tests/queries/function_acosh.proto.bin   |   Bin 0 -> 537 bytes
 .../query-tests/queries/function_add_months.json   |    81 +
 .../queries/function_add_months.proto.bin          |   Bin 0 -> 682 bytes
 .../query-tests/queries/function_aes_decrypt.json  |    81 +
 .../queries/function_aes_decrypt.proto.bin         |   Bin 0 -> 732 bytes
 .../queries/function_aes_decrypt_with_mode.json    |   102 +
 .../function_aes_decrypt_with_mode.proto.bin       |   Bin 0 -> 917 bytes
 .../function_aes_decrypt_with_mode_padding.json    |   123 +
 ...unction_aes_decrypt_with_mode_padding.proto.bin |   Bin 0 -> 1102 bytes
 ...function_aes_decrypt_with_mode_padding_aad.json |   144 +
 ...ion_aes_decrypt_with_mode_padding_aad.proto.bin |   Bin 0 -> 1287 bytes
 .../query-tests/queries/function_aes_encrypt.json  |    81 +
 .../queries/function_aes_encrypt.proto.bin         |   Bin 0 -> 732 bytes
 .../queries/function_aes_encrypt_with_mode.json    |   102 +
 .../function_aes_encrypt_with_mode.proto.bin       |   Bin 0 -> 917 bytes
 .../function_aes_encrypt_with_mode_padding.json    |   123 +
 ...unction_aes_encrypt_with_mode_padding.proto.bin |   Bin 0 -> 1102 bytes
 .../function_aes_encrypt_with_mode_padding_iv.json |   144 +
 ...tion_aes_encrypt_with_mode_padding_iv.proto.bin |   Bin 0 -> 1289 bytes
 ...ction_aes_encrypt_with_mode_padding_iv_aad.json |   165 +
 ..._aes_encrypt_with_mode_padding_iv_aad.proto.bin |   Bin 0 -> 1474 bytes
 .../query-tests/queries/function_aggregate.json    |   166 +
 .../queries/function_aggregate.proto.bin           |   Bin 0 -> 1334 bytes
 .../function_aggregate_with_finish_lambda.json     |   210 +
 ...function_aggregate_with_finish_lambda.proto.bin |   Bin 0 -> 1702 bytes
 .../query-tests/queries/function_any.json          |    60 +
 .../query-tests/queries/function_any.proto.bin     |   Bin 0 -> 425 bytes
 .../function_any_value_with_ignore_nulls.json      |    81 +
 .../function_any_value_with_ignore_nulls.proto.bin |   Bin 0 -> 727 bytes
 .../function_any_value_with_respect_nulls.json     |    60 +
 ...function_any_value_with_respect_nulls.proto.bin |   Bin 0 -> 543 bytes
 .../queries/function_approx_count_distinct.json    |    60 +
 .../function_approx_count_distinct.proto.bin       |   Bin 0 -> 585 bytes
 .../function_approx_count_distinct_rsd.json        |    81 +
 .../function_approx_count_distinct_rsd.proto.bin   |   Bin 0 -> 794 bytes
 .../queries/function_approx_percentile.json        |   102 +
 .../queries/function_approx_percentile.proto.bin   |   Bin 0 -> 934 bytes
 .../query-tests/queries/function_array.json        |    81 +
 .../query-tests/queries/function_array.proto.bin   |   Bin 0 -> 724 bytes
 .../query-tests/queries/function_array_agg.json    |    60 +
 .../queries/function_array_agg.proto.bin           |   Bin 0 -> 543 bytes
 .../query-tests/queries/function_array_append.json |    81 +
 .../queries/function_array_append.proto.bin        |   Bin 0 -> 733 bytes
 .../queries/function_array_compact.json            |    60 +
 .../queries/function_array_compact.proto.bin       |   Bin 0 -> 551 bytes
 .../queries/function_array_contains.json           |    81 +
 .../queries/function_array_contains.proto.bin      |   Bin 0 -> 737 bytes
 .../queries/function_array_distinct.json           |    60 +
 .../queries/function_array_distinct.proto.bin      |   Bin 0 -> 553 bytes
 .../query-tests/queries/function_array_except.json |   146 +
 .../queries/function_array_except.proto.bin        |   Bin 0 -> 1295 bytes
 .../query-tests/queries/function_array_insert.json |   102 +
 .../queries/function_array_insert.proto.bin        |   Bin 0 -> 917 bytes
 .../queries/function_array_intersect.json          |   125 +
 .../queries/function_array_intersect.proto.bin     |   Bin 0 -> 1117 bytes
 .../query-tests/queries/function_array_join.json   |    81 +
 .../queries/function_array_join.proto.bin          |   Bin 0 -> 737 bytes
 .../function_array_join_with_null_replacement.json |   102 +
 ...tion_array_join_with_null_replacement.proto.bin |   Bin 0 -> 932 bytes
 .../query-tests/queries/function_array_max.json    |    60 +
 .../queries/function_array_max.proto.bin           |   Bin 0 -> 543 bytes
 .../query-tests/queries/function_array_min.json    |    60 +
 .../queries/function_array_min.proto.bin           |   Bin 0 -> 543 bytes
 .../queries/function_array_position.json           |    81 +
 .../queries/function_array_position.proto.bin      |   Bin 0 -> 748 bytes
 .../queries/function_array_prepend.json            |    81 +
 .../queries/function_array_prepend.proto.bin       |   Bin 0 -> 735 bytes
 .../query-tests/queries/function_array_remove.json |    81 +
 .../queries/function_array_remove.proto.bin        |   Bin 0 -> 743 bytes
 .../query-tests/queries/function_array_repeat.json |    81 +
 .../queries/function_array_repeat.proto.bin        |   Bin 0 -> 742 bytes
 .../query-tests/queries/function_array_size.json   |    60 +
 .../queries/function_array_size.proto.bin          |   Bin 0 -> 545 bytes
 .../query-tests/queries/function_array_sort.json   |    60 +
 .../queries/function_array_sort.proto.bin          |   Bin 0 -> 545 bytes
 .../function_array_sort_with_comparator.json       |   117 +
 .../function_array_sort_with_comparator.proto.bin  |   Bin 0 -> 950 bytes
 .../query-tests/queries/function_array_union.json  |   146 +
 .../queries/function_array_union.proto.bin         |   Bin 0 -> 1293 bytes
 .../queries/function_arrays_overlap.json           |   125 +
 .../queries/function_arrays_overlap.proto.bin      |   Bin 0 -> 1115 bytes
 .../query-tests/queries/function_arrays_zip.json   |   125 +
 .../queries/function_arrays_zip.proto.bin          |   Bin 0 -> 1113 bytes
 .../query-tests/queries/function_asc.json          |    60 +
 .../query-tests/queries/function_asc.proto.bin     |   Bin 0 -> 528 bytes
 .../queries/function_asc_nulls_first.json          |    60 +
 .../queries/function_asc_nulls_first.proto.bin     |   Bin 0 -> 552 bytes
 .../queries/function_asc_nulls_last.json           |    60 +
 .../queries/function_asc_nulls_last.proto.bin      |   Bin 0 -> 550 bytes
 .../query-tests/queries/function_ascii.json        |    60 +
 .../query-tests/queries/function_ascii.proto.bin   |   Bin 0 -> 535 bytes
 .../query-tests/queries/function_asin.json         |    60 +
 .../query-tests/queries/function_asin.proto.bin    |   Bin 0 -> 534 bytes
 .../query-tests/queries/function_asinh.json        |    60 +
 .../query-tests/queries/function_asinh.proto.bin   |   Bin 0 -> 537 bytes
 .../queries/function_assert_true_with_message.json |   125 +
 .../function_assert_true_with_message.proto.bin    |   Bin 0 -> 1112 bytes
 .../query-tests/queries/function_atan.json         |    60 +
 .../query-tests/queries/function_atan.proto.bin    |   Bin 0 -> 534 bytes
 .../query-tests/queries/function_atan2.json        |   106 +
 .../query-tests/queries/function_atan2.proto.bin   |   Bin 0 -> 903 bytes
 .../query-tests/queries/function_atanh.json        |    60 +
 .../query-tests/queries/function_atanh.proto.bin   |   Bin 0 -> 537 bytes
 .../query-tests/queries/function_avg.json          |    60 +
 .../query-tests/queries/function_avg.proto.bin     |   Bin 0 -> 531 bytes
 .../query-tests/queries/function_base64.json       |    85 +
 .../query-tests/queries/function_base64.proto.bin  |   Bin 0 -> 718 bytes
 .../query-tests/queries/function_bin.json          |    60 +
 .../query-tests/queries/function_bin.proto.bin     |   Bin 0 -> 531 bytes
 .../query-tests/queries/function_bit_and.json      |    60 +
 .../query-tests/queries/function_bit_and.proto.bin |   Bin 0 -> 539 bytes
 .../query-tests/queries/function_bit_count.json    |    60 +
 .../queries/function_bit_count.proto.bin           |   Bin 0 -> 543 bytes
 .../query-tests/queries/function_bit_get.json      |    81 +
 .../query-tests/queries/function_bit_get.proto.bin |   Bin 0 -> 723 bytes
 .../query-tests/queries/function_bit_length.json   |    60 +
 .../queries/function_bit_length.proto.bin          |   Bin 0 -> 545 bytes
 .../query-tests/queries/function_bit_or.json       |    60 +
 .../query-tests/queries/function_bit_or.proto.bin  |   Bin 0 -> 537 bytes
 .../query-tests/queries/function_bit_xor.json      |    60 +
 .../query-tests/queries/function_bit_xor.proto.bin |   Bin 0 -> 539 bytes
 .../queries/function_bitmap_bit_position.json      |    60 +
 .../queries/function_bitmap_bit_position.proto.bin |   Bin 0 -> 564 bytes
 .../queries/function_bitmap_bucket_number.json     |    60 +
 .../function_bitmap_bucket_number.proto.bin        |   Bin 0 -> 564 bytes
 .../queries/function_bitmap_construct_agg.json     |    60 +
 .../function_bitmap_construct_agg.proto.bin        |   Bin 0 -> 566 bytes
 .../query-tests/queries/function_bitmap_count.json |    60 +
 .../queries/function_bitmap_count.proto.bin        |   Bin 0 -> 444 bytes
 .../queries/function_bitmap_or_agg.json            |    60 +
 .../queries/function_bitmap_or_agg.proto.bin       |   Bin 0 -> 446 bytes
 .../query-tests/queries/function_bitwise_not.json  |    60 +
 .../queries/function_bitwise_not.proto.bin         |   Bin 0 -> 537 bytes
 .../query-tests/queries/function_bool_and.json     |    60 +
 .../queries/function_bool_and.proto.bin            |   Bin 0 -> 435 bytes
 .../query-tests/queries/function_bool_or.json      |    60 +
 .../query-tests/queries/function_bool_or.proto.bin |   Bin 0 -> 433 bytes
 .../query-tests/queries/function_bround.json       |    81 +
 .../query-tests/queries/function_bround.proto.bin  |   Bin 0 -> 721 bytes
 .../query-tests/queries/function_btrim.json        |    60 +
 .../query-tests/queries/function_btrim.proto.bin   |   Bin 0 -> 535 bytes
 .../function_btrim_with_specified_trim_string.json |    81 +
 ...tion_btrim_with_specified_trim_string.proto.bin |   Bin 0 -> 720 bytes
 .../query-tests/queries/function_bucket.json       |    81 +
 .../query-tests/queries/function_bucket.proto.bin  |   Bin 0 -> 720 bytes
 .../queries/function_call_function.json            |    61 +
 .../queries/function_call_function.proto.bin       |   Bin 0 -> 545 bytes
 .../queries/function_call_function_orphaned.json   |    25 +
 .../function_call_function_orphaned.proto.bin      |   Bin 0 -> 175 bytes
 .../query-tests/queries/function_cardinality.json  |    60 +
 .../queries/function_cardinality.proto.bin         |   Bin 0 -> 547 bytes
 .../query-tests/queries/function_ceil.json         |    60 +
 .../query-tests/queries/function_ceil.proto.bin    |   Bin 0 -> 534 bytes
 .../query-tests/queries/function_ceil_scale.json   |    81 +
 .../queries/function_ceil_scale.proto.bin          |   Bin 0 -> 717 bytes
 .../query-tests/queries/function_ceiling.json      |    60 +
 .../query-tests/queries/function_ceiling.proto.bin |   Bin 0 -> 539 bytes
 .../queries/function_ceiling_scale.json            |    81 +
 .../queries/function_ceiling_scale.proto.bin       |   Bin 0 -> 723 bytes
 .../query-tests/queries/function_char.json         |    60 +
 .../query-tests/queries/function_char.proto.bin    |   Bin 0 -> 533 bytes
 .../query-tests/queries/function_char_length.json  |    60 +
 .../queries/function_char_length.proto.bin         |   Bin 0 -> 547 bytes
 .../queries/function_character_length.json         |    60 +
 .../queries/function_character_length.proto.bin    |   Bin 0 -> 557 bytes
 .../query-tests/queries/function_chr.json          |    60 +
 .../query-tests/queries/function_chr.proto.bin     |   Bin 0 -> 531 bytes
 .../query-tests/queries/function_coalesce.json     |    81 +
 .../queries/function_coalesce.proto.bin            |   Bin 0 -> 725 bytes
 .../query-tests/queries/function_col.json          |    37 +
 .../query-tests/queries/function_col.proto.bin     |   Bin 0 -> 342 bytes
 .../query-tests/queries/function_collate.json      |    81 +
 .../query-tests/queries/function_collate.proto.bin |   Bin 0 -> 734 bytes
 .../query-tests/queries/function_collation.json    |    60 +
 .../queries/function_collation.proto.bin           |   Bin 0 -> 543 bytes
 .../query-tests/queries/function_collect_list.json |    60 +
 .../queries/function_collect_list.proto.bin        |   Bin 0 -> 558 bytes
 .../query-tests/queries/function_collect_set.json  |    60 +
 .../queries/function_collect_set.proto.bin         |   Bin 0 -> 555 bytes
 .../query-tests/queries/function_concat.json       |   190 +
 .../query-tests/queries/function_concat.proto.bin  |   Bin 0 -> 1667 bytes
 .../query-tests/queries/function_concat_ws.json    |   123 +
 .../queries/function_concat_ws.proto.bin           |   Bin 0 -> 1109 bytes
 .../query-tests/queries/function_contains.json     |    81 +
 .../queries/function_contains.proto.bin            |   Bin 0 -> 726 bytes
 .../query-tests/queries/function_conv.json         |   102 +
 .../query-tests/queries/function_conv.proto.bin    |   Bin 0 -> 903 bytes
 ...ion_convert_timezone_with_source_time_zone.json |   102 +
 ...onvert_timezone_with_source_time_zone.proto.bin |   Bin 0 -> 898 bytes
 ..._convert_timezone_without_source_time_zone.json |    81 +
 ...ert_timezone_without_source_time_zone.proto.bin |   Bin 0 -> 700 bytes
 .../query-tests/queries/function_corr.json         |    81 +
 .../query-tests/queries/function_corr.proto.bin    |   Bin 0 -> 720 bytes
 .../query-tests/queries/function_cos.json          |    60 +
 .../query-tests/queries/function_cos.proto.bin     |   Bin 0 -> 531 bytes
 .../query-tests/queries/function_cosh.json         |    60 +
 .../query-tests/queries/function_cosh.proto.bin    |   Bin 0 -> 534 bytes
 .../query-tests/queries/function_cot.json          |    60 +
 .../query-tests/queries/function_cot.proto.bin     |   Bin 0 -> 531 bytes
 .../query-tests/queries/function_count.json        |    60 +
 .../query-tests/queries/function_count.proto.bin   |   Bin 0 -> 535 bytes
 .../queries/function_countDistinct.json            |    82 +
 .../queries/function_countDistinct.proto.bin       |   Bin 0 -> 750 bytes
 .../query-tests/queries/function_count_if.json     |   104 +
 .../queries/function_count_if.proto.bin            |   Bin 0 -> 897 bytes
 .../queries/function_count_min_sketch.json         |   123 +
 .../queries/function_count_min_sketch.proto.bin    |   Bin 0 -> 1123 bytes
 .../query-tests/queries/function_count_typed.json  |    60 +
 .../queries/function_count_typed.proto.bin         |   Bin 0 -> 430 bytes
 .../query-tests/queries/function_covar_pop.json    |    81 +
 .../queries/function_covar_pop.proto.bin           |   Bin 0 -> 740 bytes
 .../query-tests/queries/function_covar_samp.json   |    81 +
 .../queries/function_covar_samp.proto.bin          |   Bin 0 -> 744 bytes
 .../query-tests/queries/function_crc32.json        |    85 +
 .../query-tests/queries/function_crc32.proto.bin   |   Bin 0 -> 716 bytes
 .../query-tests/queries/function_csc.json          |    60 +
 .../query-tests/queries/function_csc.proto.bin     |   Bin 0 -> 531 bytes
 .../query-tests/queries/function_cume_dist.json    |   107 +
 .../queries/function_cume_dist.proto.bin           |   Bin 0 -> 904 bytes
 .../query-tests/queries/function_curdate.json      |    38 +
 .../query-tests/queries/function_curdate.proto.bin |   Bin 0 -> 299 bytes
 .../queries/function_current_catalog.json          |    38 +
 .../queries/function_current_catalog.proto.bin     |   Bin 0 -> 369 bytes
 .../queries/function_current_database.json         |    38 +
 .../queries/function_current_database.proto.bin    |   Bin 0 -> 371 bytes
 .../query-tests/queries/function_current_date.json |    38 +
 .../queries/function_current_date.proto.bin        |   Bin 0 -> 309 bytes
 .../queries/function_current_schema.json           |    38 +
 .../queries/function_current_schema.proto.bin      |   Bin 0 -> 367 bytes
 .../queries/function_current_timestamp.json        |    38 +
 .../queries/function_current_timestamp.proto.bin   |   Bin 0 -> 319 bytes
 .../queries/function_current_timezone.json         |    38 +
 .../queries/function_current_timezone.proto.bin    |   Bin 0 -> 317 bytes
 .../query-tests/queries/function_current_user.json |    38 +
 .../queries/function_current_user.proto.bin        |   Bin 0 -> 363 bytes
 .../query-tests/queries/function_date_add.json     |    81 +
 .../queries/function_date_add.proto.bin            |   Bin 0 -> 676 bytes
 .../query-tests/queries/function_date_diff.json    |   146 +
 .../queries/function_date_diff.proto.bin           |   Bin 0 -> 1244 bytes
 .../query-tests/queries/function_date_format.json  |    81 +
 .../queries/function_date_format.proto.bin         |   Bin 0 -> 695 bytes
 .../queries/function_date_from_unix_date.json      |    60 +
 .../queries/function_date_from_unix_date.proto.bin |   Bin 0 -> 508 bytes
 .../query-tests/queries/function_date_part.json    |    81 +
 .../queries/function_date_part.proto.bin           |   Bin 0 -> 677 bytes
 .../query-tests/queries/function_date_sub.json     |    81 +
 .../queries/function_date_sub.proto.bin            |   Bin 0 -> 676 bytes
 .../query-tests/queries/function_date_trunc.json   |    81 +
 .../queries/function_date_trunc.proto.bin          |   Bin 0 -> 673 bytes
 .../query-tests/queries/function_dateadd.json      |    81 +
 .../query-tests/queries/function_dateadd.proto.bin |   Bin 0 -> 669 bytes
 .../query-tests/queries/function_datediff.json     |   146 +
 .../queries/function_datediff.proto.bin            |   Bin 0 -> 1242 bytes
 .../query-tests/queries/function_datepart.json     |    81 +
 .../queries/function_datepart.proto.bin            |   Bin 0 -> 675 bytes
 .../query-tests/queries/function_day.json          |    60 +
 .../query-tests/queries/function_day.proto.bin     |   Bin 0 -> 477 bytes
 .../query-tests/queries/function_dayname.json      |    60 +
 .../query-tests/queries/function_dayname.proto.bin |   Bin 0 -> 485 bytes
 .../query-tests/queries/function_dayofmonth.json   |    60 +
 .../queries/function_dayofmonth.proto.bin          |   Bin 0 -> 491 bytes
 .../query-tests/queries/function_dayofweek.json    |    60 +
 .../queries/function_dayofweek.proto.bin           |   Bin 0 -> 489 bytes
 .../query-tests/queries/function_dayofyear.json    |    60 +
 .../queries/function_dayofyear.proto.bin           |   Bin 0 -> 489 bytes
 .../query-tests/queries/function_days.json         |    60 +
 .../query-tests/queries/function_days.proto.bin    |   Bin 0 -> 529 bytes
 .../query-tests/queries/function_decode.json       |   106 +
 .../query-tests/queries/function_decode.proto.bin  |   Bin 0 -> 910 bytes
 .../query-tests/queries/function_degrees.json      |    60 +
 .../query-tests/queries/function_degrees.proto.bin |   Bin 0 -> 543 bytes
 .../query-tests/queries/function_dense_rank.json   |   107 +
 .../queries/function_dense_rank.proto.bin          |   Bin 0 -> 906 bytes
 .../query-tests/queries/function_desc.json         |    60 +
 .../query-tests/queries/function_desc.proto.bin    |   Bin 0 -> 530 bytes
 .../queries/function_desc_nulls_first.json         |    60 +
 .../queries/function_desc_nulls_first.proto.bin    |   Bin 0 -> 554 bytes
 .../queries/function_desc_nulls_last.json          |    60 +
 .../queries/function_desc_nulls_last.proto.bin     |   Bin 0 -> 552 bytes
 .../resources/query-tests/queries/function_e.json  |    38 +
 .../query-tests/queries/function_e.proto.bin       |   Bin 0 -> 341 bytes
 .../query-tests/queries/function_element_at.json   |    81 +
 .../queries/function_element_at.proto.bin          |   Bin 0 -> 739 bytes
 .../query-tests/queries/function_elt.json          |   102 +
 .../query-tests/queries/function_elt.proto.bin     |   Bin 0 -> 901 bytes
 .../query-tests/queries/function_encode.json       |    81 +
 .../query-tests/queries/function_encode.proto.bin  |   Bin 0 -> 729 bytes
 .../query-tests/queries/function_endswith.json     |    81 +
 .../queries/function_endswith.proto.bin            |   Bin 0 -> 726 bytes
 .../query-tests/queries/function_equal_null.json   |    81 +
 .../queries/function_equal_null.proto.bin          |   Bin 0 -> 730 bytes
 .../query-tests/queries/function_every.json        |    60 +
 .../query-tests/queries/function_every.proto.bin   |   Bin 0 -> 429 bytes
 .../query-tests/queries/function_exists.json       |   132 +
 .../query-tests/queries/function_exists.proto.bin  |   Bin 0 -> 1107 bytes
 .../query-tests/queries/function_exp.json          |    60 +
 .../query-tests/queries/function_exp.proto.bin     |   Bin 0 -> 531 bytes
 .../query-tests/queries/function_explode.json      |    60 +
 .../query-tests/queries/function_explode.proto.bin |   Bin 0 -> 539 bytes
 .../queries/function_explode_outer.json            |    60 +
 .../queries/function_explode_outer.proto.bin       |   Bin 0 -> 551 bytes
 .../query-tests/queries/function_expm1.json        |    60 +
 .../query-tests/queries/function_expm1.proto.bin   |   Bin 0 -> 537 bytes
 .../query-tests/queries/function_expr.json         |    37 +
 .../query-tests/queries/function_expr.proto.bin    |   Bin 0 -> 346 bytes
 .../query-tests/queries/function_extract.json      |    81 +
 .../query-tests/queries/function_extract.proto.bin |   Bin 0 -> 673 bytes
 .../query-tests/queries/function_factorial.json    |   104 +
 .../queries/function_factorial.proto.bin           |   Bin 0 -> 911 bytes
 .../query-tests/queries/function_filter.json       |   132 +
 .../query-tests/queries/function_filter.proto.bin  |   Bin 0 -> 1107 bytes
 .../queries/function_filter_with_pair_input.json   |   205 +
 .../function_filter_with_pair_input.proto.bin      |   Bin 0 -> 1679 bytes
 .../query-tests/queries/function_find_in_set.json  |    81 +
 .../queries/function_find_in_set.proto.bin         |   Bin 0 -> 732 bytes
 .../function_first_value_with_ignore_nulls.json    |    81 +
 ...unction_first_value_with_ignore_nulls.proto.bin |   Bin 0 -> 731 bytes
 .../function_first_value_with_respect_nulls.json   |    60 +
 ...nction_first_value_with_respect_nulls.proto.bin |   Bin 0 -> 547 bytes
 .../queries/function_first_with_ignore_nulls.json  |    81 +
 .../function_first_with_ignore_nulls.proto.bin     |   Bin 0 -> 723 bytes
 .../queries/function_first_with_respect_nulls.json |    81 +
 .../function_first_with_respect_nulls.proto.bin    |   Bin 0 -> 723 bytes
 .../query-tests/queries/function_flatten.json      |   148 +
 .../query-tests/queries/function_flatten.proto.bin |   Bin 0 -> 1301 bytes
 .../query-tests/queries/function_floor.json        |    60 +
 .../query-tests/queries/function_floor.proto.bin   |   Bin 0 -> 537 bytes
 .../query-tests/queries/function_floor_scale.json  |    81 +
 .../queries/function_floor_scale.proto.bin         |   Bin 0 -> 719 bytes
 .../query-tests/queries/function_forall.json       |   132 +
 .../query-tests/queries/function_forall.proto.bin  |   Bin 0 -> 1107 bytes
 .../queries/function_format_number.json            |    81 +
 .../queries/function_format_number.proto.bin       |   Bin 0 -> 745 bytes
 .../query-tests/queries/function_from_csv.json     |   146 +
 .../queries/function_from_csv.proto.bin            |   Bin 0 -> 1339 bytes
 .../query-tests/queries/function_from_json.json    |    81 +
 .../queries/function_from_json.proto.bin           |   Bin 0 -> 770 bytes
 .../queries/function_from_json_orphaned.json       |    29 +
 .../queries/function_from_json_orphaned.proto.bin  |   Bin 0 -> 394 bytes
 .../function_from_json_with_json_schema.json       |    81 +
 .../function_from_json_with_json_schema.proto.bin  |   Bin 0 -> 934 bytes
 .../queries/function_from_unixtime.json            |    60 +
 .../queries/function_from_unixtime.proto.bin       |   Bin 0 -> 496 bytes
 .../queries/function_from_utc_timestamp.json       |    81 +
 .../queries/function_from_utc_timestamp.proto.bin  |   Bin 0 -> 712 bytes
 .../query-tests/queries/function_from_xml.json     |    81 +
 .../queries/function_from_xml.proto.bin            |   Bin 0 -> 767 bytes
 .../function_from_xml_with_json_schema.json        |    81 +
 .../function_from_xml_with_json_schema.proto.bin   |   Bin 0 -> 932 bytes
 .../query-tests/queries/function_get.json          |    81 +
 .../query-tests/queries/function_get.proto.bin     |   Bin 0 -> 715 bytes
 .../queries/function_get_json_object.json          |    81 +
 .../queries/function_get_json_object.proto.bin     |   Bin 0 -> 764 bytes
 .../query-tests/queries/function_getbit.json       |    81 +
 .../query-tests/queries/function_getbit.proto.bin  |   Bin 0 -> 721 bytes
 .../query-tests/queries/function_greatest.json     |   124 +
 .../queries/function_greatest.proto.bin            |   Bin 0 -> 1088 bytes
 .../query-tests/queries/function_hash.json         |    81 +
 .../query-tests/queries/function_hash.proto.bin    |   Bin 0 -> 719 bytes
 .../query-tests/queries/function_hex.json          |    60 +
 .../query-tests/queries/function_hex.proto.bin     |   Bin 0 -> 531 bytes
 .../queries/function_histogram_numeric.json        |    81 +
 .../queries/function_histogram_numeric.proto.bin   |   Bin 0 -> 743 bytes
 .../query-tests/queries/function_hour.json         |    60 +
 .../query-tests/queries/function_hour.proto.bin    |   Bin 0 -> 479 bytes
 .../query-tests/queries/function_hours.json        |    60 +
 .../query-tests/queries/function_hours.proto.bin   |   Bin 0 -> 531 bytes
 .../query-tests/queries/function_hypot.json        |    81 +
 .../query-tests/queries/function_hypot.proto.bin   |   Bin 0 -> 720 bytes
 .../query-tests/queries/function_ifnull.json       |    81 +
 .../query-tests/queries/function_ifnull.proto.bin  |   Bin 0 -> 722 bytes
 .../query-tests/queries/function_ilike.json        |    81 +
 .../query-tests/queries/function_ilike.proto.bin   |   Bin 0 -> 720 bytes
 .../queries/function_ilike_with_escape.json        |   102 +
 .../queries/function_ilike_with_escape.proto.bin   |   Bin 0 -> 905 bytes
 .../query-tests/queries/function_initcap.json      |    60 +
 .../query-tests/queries/function_initcap.proto.bin |   Bin 0 -> 539 bytes
 .../query-tests/queries/function_inline.json       |    83 +
 .../query-tests/queries/function_inline.proto.bin  |   Bin 0 -> 741 bytes
 .../query-tests/queries/function_inline_outer.json |    83 +
 .../queries/function_inline_outer.proto.bin        |   Bin 0 -> 753 bytes
 .../queries/function_input_file_block_length.json  |    38 +
 .../function_input_file_block_length.proto.bin     |   Bin 0 -> 385 bytes
 .../queries/function_input_file_block_start.json   |    38 +
 .../function_input_file_block_start.proto.bin      |   Bin 0 -> 383 bytes
 .../queries/function_input_file_name.json          |    38 +
 .../queries/function_input_file_name.proto.bin     |   Bin 0 -> 369 bytes
 .../queries/function_is_valid_variant.json         |    83 +
 .../queries/function_is_valid_variant.proto.bin    |   Bin 0 -> 761 bytes
 .../queries/function_is_variant_null.json          |    83 +
 .../queries/function_is_variant_null.proto.bin     |   Bin 0 -> 759 bytes
 .../query-tests/queries/function_isnan.json        |    60 +
 .../query-tests/queries/function_isnan.proto.bin   |   Bin 0 -> 535 bytes
 .../query-tests/queries/function_isnotnull.json    |    60 +
 .../queries/function_isnotnull.proto.bin           |   Bin 0 -> 543 bytes
 .../query-tests/queries/function_isnull.json       |    60 +
 .../query-tests/queries/function_isnull.proto.bin  |   Bin 0 -> 537 bytes
 .../query-tests/queries/function_java_method.json  |   102 +
 .../queries/function_java_method.proto.bin         |   Bin 0 -> 939 bytes
 .../queries/function_json_array_length.json        |    60 +
 .../queries/function_json_array_length.proto.bin   |   Bin 0 -> 559 bytes
 .../queries/function_json_object_keys.json         |    60 +
 .../queries/function_json_object_keys.proto.bin    |   Bin 0 -> 557 bytes
 .../query-tests/queries/function_json_tuple.json   |   123 +
 .../queries/function_json_tuple.proto.bin          |   Bin 0 -> 1122 bytes
 .../query-tests/queries/function_kurtosis.json     |    60 +
 .../queries/function_kurtosis.proto.bin            |   Bin 0 -> 546 bytes
 .../query-tests/queries/function_lag.json          |   199 +
 .../query-tests/queries/function_lag.proto.bin     |   Bin 0 -> 1633 bytes
 .../query-tests/queries/function_last_day.json     |    60 +
 .../queries/function_last_day.proto.bin            |   Bin 0 -> 487 bytes
 .../function_last_value_with_ignore_nulls.json     |    81 +
 ...function_last_value_with_ignore_nulls.proto.bin |   Bin 0 -> 729 bytes
 .../function_last_value_with_respect_nulls.json    |    60 +
 ...unction_last_value_with_respect_nulls.proto.bin |   Bin 0 -> 545 bytes
 .../queries/function_last_with_ignore_nulls.json   |    81 +
 .../function_last_with_ignore_nulls.proto.bin      |   Bin 0 -> 719 bytes
 .../queries/function_last_with_respect_nulls.json  |    81 +
 .../function_last_with_respect_nulls.proto.bin     |   Bin 0 -> 719 bytes
 .../query-tests/queries/function_lcase.json        |    60 +
 .../query-tests/queries/function_lcase.proto.bin   |   Bin 0 -> 535 bytes
 .../query-tests/queries/function_lead.json         |   192 +
 .../query-tests/queries/function_lead.proto.bin    |   Bin 0 -> 1633 bytes
 .../query-tests/queries/function_least.json        |   124 +
 .../query-tests/queries/function_least.proto.bin   |   Bin 0 -> 1082 bytes
 .../query-tests/queries/function_left.json         |    81 +
 .../query-tests/queries/function_left.proto.bin    |   Bin 0 -> 718 bytes
 .../query-tests/queries/function_len.json          |    60 +
 .../query-tests/queries/function_len.proto.bin     |   Bin 0 -> 531 bytes
 .../query-tests/queries/function_length.json       |    60 +
 .../query-tests/queries/function_length.proto.bin  |   Bin 0 -> 537 bytes
 .../query-tests/queries/function_levenshtein.json  |    81 +
 .../queries/function_levenshtein.proto.bin         |   Bin 0 -> 734 bytes
 .../function_levenshtein_with_threshold.json       |   102 +
 .../function_levenshtein_with_threshold.proto.bin  |   Bin 0 -> 926 bytes
 .../query-tests/queries/function_like.json         |    81 +
 .../query-tests/queries/function_like.proto.bin    |   Bin 0 -> 718 bytes
 .../queries/function_like_with_escape.json         |   102 +
 .../queries/function_like_with_escape.proto.bin    |   Bin 0 -> 903 bytes
 .../query-tests/queries/function_lit.json          |   645 +
 .../query-tests/queries/function_lit.proto.bin     |   Bin 0 -> 5400 bytes
 .../query-tests/queries/function_lit_array.json    |   908 +
 .../queries/function_lit_array.proto.bin           |   Bin 0 -> 5116 bytes
 .../resources/query-tests/queries/function_ln.json |    60 +
 .../query-tests/queries/function_ln.proto.bin      |   Bin 0 -> 529 bytes
 .../queries/function_localtimestamp.json           |    38 +
 .../queries/function_localtimestamp.proto.bin      |   Bin 0 -> 313 bytes
 .../query-tests/queries/function_locate.json       |    81 +
 .../query-tests/queries/function_locate.proto.bin  |   Bin 0 -> 727 bytes
 .../queries/function_locate_with_pos.json          |   102 +
 .../queries/function_locate_with_pos.proto.bin     |   Bin 0 -> 914 bytes
 .../query-tests/queries/function_log.json          |    60 +
 .../query-tests/queries/function_log.proto.bin     |   Bin 0 -> 530 bytes
 .../query-tests/queries/function_log10.json        |    60 +
 .../query-tests/queries/function_log10.proto.bin   |   Bin 0 -> 537 bytes
 .../query-tests/queries/function_log1p.json        |    60 +
 .../query-tests/queries/function_log1p.proto.bin   |   Bin 0 -> 537 bytes
 .../query-tests/queries/function_log2.json         |    60 +
 .../query-tests/queries/function_log2.proto.bin    |   Bin 0 -> 534 bytes
 .../queries/function_log_with_base.json            |    81 +
 .../queries/function_log_with_base.proto.bin       |   Bin 0 -> 722 bytes
 .../query-tests/queries/function_lower.json        |    60 +
 .../query-tests/queries/function_lower.proto.bin   |   Bin 0 -> 535 bytes
 .../query-tests/queries/function_lpad.json         |   102 +
 .../query-tests/queries/function_lpad.proto.bin    |   Bin 0 -> 904 bytes
 .../query-tests/queries/function_lpad_binary.json  |   102 +
 .../queries/function_lpad_binary.proto.bin         |   Bin 0 -> 802 bytes
 .../query-tests/queries/function_ltrim.json        |    60 +
 .../query-tests/queries/function_ltrim.proto.bin   |   Bin 0 -> 535 bytes
 .../queries/function_ltrim_with_pattern.json       |    81 +
 .../queries/function_ltrim_with_pattern.proto.bin  |   Bin 0 -> 724 bytes
 .../query-tests/queries/function_make_date.json    |   102 +
 .../queries/function_make_date.proto.bin           |   Bin 0 -> 857 bytes
 .../queries/function_make_dt_interval.json         |    38 +
 .../queries/function_make_dt_interval.proto.bin    |   Bin 0 -> 371 bytes
 .../queries/function_make_dt_interval_days.json    |    60 +
 .../function_make_dt_interval_days.proto.bin       |   Bin 0 -> 557 bytes
 .../function_make_dt_interval_days_hours.json      |    81 +
 .../function_make_dt_interval_days_hours.proto.bin |   Bin 0 -> 742 bytes
 .../function_make_dt_interval_days_hours_mins.json |   102 +
 ...tion_make_dt_interval_days_hours_mins.proto.bin |   Bin 0 -> 927 bytes
 ...tion_make_dt_interval_days_hours_mins_secs.json |   123 +
 ...make_dt_interval_days_hours_mins_secs.proto.bin |   Bin 0 -> 1112 bytes
 .../queries/function_make_interval.json            |    38 +
 .../queries/function_make_interval.proto.bin       |   Bin 0 -> 365 bytes
 .../queries/function_make_interval_years.json      |    60 +
 .../queries/function_make_interval_years.proto.bin |   Bin 0 -> 551 bytes
 .../function_make_interval_years_months.json       |    81 +
 .../function_make_interval_years_months.proto.bin  |   Bin 0 -> 736 bytes
 .../function_make_interval_years_months_weeks.json |   102 +
 ...tion_make_interval_years_months_weeks.proto.bin |   Bin 0 -> 921 bytes
 ...tion_make_interval_years_months_weeks_days.json |   123 +
 ...make_interval_years_months_weeks_days.proto.bin |   Bin 0 -> 1106 bytes
 ...ake_interval_years_months_weeks_days_hours.json |   144 +
 ...nterval_years_months_weeks_days_hours.proto.bin |   Bin 0 -> 1291 bytes
 ...nterval_years_months_weeks_days_hours_mins.json |   165 +
 ...al_years_months_weeks_days_hours_mins.proto.bin |   Bin 0 -> 1476 bytes
 ...al_years_months_weeks_days_hours_mins_secs.json |   186 +
 ...ars_months_weeks_days_hours_mins_secs.proto.bin |   Bin 0 -> 1661 bytes
 .../function_make_timestamp_ltz_with_timezone.json |   186 +
 ...tion_make_timestamp_ltz_with_timezone.proto.bin |   Bin 0 -> 1671 bytes
 ...nction_make_timestamp_ltz_without_timezone.json |   165 +
 ...n_make_timestamp_ltz_without_timezone.proto.bin |   Bin 0 -> 1486 bytes
 .../queries/function_make_timestamp_ntz.json       |   165 +
 .../queries/function_make_timestamp_ntz.proto.bin  |   Bin 0 -> 1486 bytes
 .../function_make_timestamp_with_timezone.json     |   186 +
 ...function_make_timestamp_with_timezone.proto.bin |   Bin 0 -> 1663 bytes
 .../function_make_timestamp_without_timezone.json  |   165 +
 ...ction_make_timestamp_without_timezone.proto.bin |   Bin 0 -> 1478 bytes
 .../queries/function_make_ym_interval.json         |    38 +
 .../queries/function_make_ym_interval.proto.bin    |   Bin 0 -> 371 bytes
 .../queries/function_make_ym_interval_years.json   |    60 +
 .../function_make_ym_interval_years.proto.bin      |   Bin 0 -> 557 bytes
 .../function_make_ym_interval_years_months.json    |    81 +
 ...unction_make_ym_interval_years_months.proto.bin |   Bin 0 -> 742 bytes
 .../query-tests/queries/function_map.json          |   123 +
 .../query-tests/queries/function_map.proto.bin     |   Bin 0 -> 1089 bytes
 .../query-tests/queries/function_map_concat.json   |   256 +
 .../queries/function_map_concat.proto.bin          |   Bin 0 -> 2212 bytes
 .../queries/function_map_contains_key.json         |    81 +
 .../queries/function_map_contains_key.proto.bin    |   Bin 0 -> 757 bytes
 .../query-tests/queries/function_map_entries.json  |    60 +
 .../queries/function_map_entries.proto.bin         |   Bin 0 -> 547 bytes
 .../query-tests/queries/function_map_filter.json   |   134 +
 .../queries/function_map_filter.proto.bin          |   Bin 0 -> 1138 bytes
 .../queries/function_map_from_arrays.json          |   169 +
 .../queries/function_map_from_arrays.proto.bin     |   Bin 0 -> 1500 bytes
 .../queries/function_map_from_entries.json         |   140 +
 .../queries/function_map_from_entries.proto.bin    |   Bin 0 -> 1175 bytes
 .../query-tests/queries/function_map_keys.json     |    60 +
 .../queries/function_map_keys.proto.bin            |   Bin 0 -> 541 bytes
 .../query-tests/queries/function_map_values.json   |    60 +
 .../queries/function_map_values.proto.bin          |   Bin 0 -> 545 bytes
 .../query-tests/queries/function_map_zip_with.json |   226 +
 .../queries/function_map_zip_with.proto.bin        |   Bin 0 -> 1874 bytes
 .../query-tests/queries/function_mask.json         |    60 +
 .../query-tests/queries/function_mask.proto.bin    |   Bin 0 -> 533 bytes
 .../function_mask_with_specific_upperChar.json     |    81 +
 ...function_mask_with_specific_upperChar.proto.bin |   Bin 0 -> 718 bytes
 ...ion_mask_with_specific_upperChar_lowerChar.json |   102 +
 ...ask_with_specific_upperChar_lowerChar.proto.bin |   Bin 0 -> 903 bytes
 ...ith_specific_upperChar_lowerChar_digitChar.json |   123 +
 ...pecific_upperChar_lowerChar_digitChar.proto.bin |   Bin 0 -> 1088 bytes
 ...ic_upperChar_lowerChar_digitChar_otherChar.json |   144 +
 ...perChar_lowerChar_digitChar_otherChar.proto.bin |   Bin 0 -> 1273 bytes
 .../query-tests/queries/function_max.json          |    60 +
 .../query-tests/queries/function_max.proto.bin     |   Bin 0 -> 425 bytes
 .../query-tests/queries/function_max_by.json       |    81 +
 .../query-tests/queries/function_max_by.proto.bin  |   Bin 0 -> 722 bytes
 .../query-tests/queries/function_md5.json          |    85 +
 .../query-tests/queries/function_md5.proto.bin     |   Bin 0 -> 712 bytes
 .../query-tests/queries/function_median.json       |    60 +
 .../query-tests/queries/function_median.proto.bin  |   Bin 0 -> 537 bytes
 .../query-tests/queries/function_min.json          |    60 +
 .../query-tests/queries/function_min.proto.bin     |   Bin 0 -> 531 bytes
 .../query-tests/queries/function_min_by.json       |    81 +
 .../query-tests/queries/function_min_by.proto.bin  |   Bin 0 -> 722 bytes
 .../query-tests/queries/function_minute.json       |    60 +
 .../query-tests/queries/function_minute.proto.bin  |   Bin 0 -> 483 bytes
 .../query-tests/queries/function_mode.json         |    60 +
 .../query-tests/queries/function_mode.proto.bin    |   Bin 0 -> 533 bytes
 .../queries/function_mode_orphaned.json            |    29 +
 .../queries/function_mode_orphaned.proto.bin       |   Bin 0 -> 179 bytes
 .../function_monotonically_increasing_id.json      |    38 +
 .../function_monotonically_increasing_id.proto.bin |   Bin 0 -> 393 bytes
 .../query-tests/queries/function_month.json        |    60 +
 .../query-tests/queries/function_month.proto.bin   |   Bin 0 -> 481 bytes
 .../query-tests/queries/function_monthname.json    |    60 +
 .../queries/function_monthname.proto.bin           |   Bin 0 -> 489 bytes
 .../query-tests/queries/function_months.json       |    60 +
 .../query-tests/queries/function_months.proto.bin  |   Bin 0 -> 533 bytes
 .../queries/function_months_between.json           |    82 +
 .../queries/function_months_between.proto.bin      |   Bin 0 -> 706 bytes
 .../function_months_between_with_roundoff.json     |   103 +
 ...function_months_between_with_roundoff.proto.bin |   Bin 0 -> 901 bytes
 .../query-tests/queries/function_named_struct.json |   123 +
 .../queries/function_named_struct.proto.bin        |   Bin 0 -> 1105 bytes
 .../query-tests/queries/function_nanvl.json        |    81 +
 .../query-tests/queries/function_nanvl.proto.bin   |   Bin 0 -> 726 bytes
 .../query-tests/queries/function_negate.json       |    60 +
 .../query-tests/queries/function_negate.proto.bin  |   Bin 0 -> 539 bytes
 .../query-tests/queries/function_negative.json     |    60 +
 .../queries/function_negative.proto.bin            |   Bin 0 -> 541 bytes
 .../query-tests/queries/function_next_day.json     |    81 +
 .../queries/function_next_day.proto.bin            |   Bin 0 -> 679 bytes
 .../query-tests/queries/function_now.json          |    38 +
 .../query-tests/queries/function_now.proto.bin     |   Bin 0 -> 291 bytes
 .../query-tests/queries/function_nth_value.json    |   171 +
 .../queries/function_nth_value.proto.bin           |   Bin 0 -> 1466 bytes
 .../query-tests/queries/function_ntile.json        |   129 +
 .../query-tests/queries/function_ntile.proto.bin   |   Bin 0 -> 1083 bytes
 .../query-tests/queries/function_nullif.json       |    81 +
 .../query-tests/queries/function_nullif.proto.bin  |   Bin 0 -> 722 bytes
 .../query-tests/queries/function_nvl.json          |    81 +
 .../query-tests/queries/function_nvl.proto.bin     |   Bin 0 -> 716 bytes
 .../query-tests/queries/function_nvl2.json         |   102 +
 .../query-tests/queries/function_nvl2.proto.bin    |   Bin 0 -> 903 bytes
 .../query-tests/queries/function_octet_length.json |    60 +
 .../queries/function_octet_length.proto.bin        |   Bin 0 -> 549 bytes
 .../query-tests/queries/function_overlay.json      |   102 +
 .../query-tests/queries/function_overlay.proto.bin |   Bin 0 -> 910 bytes
 .../queries/function_overlay_with_len.json         |   123 +
 .../queries/function_overlay_with_len.proto.bin    |   Bin 0 -> 1095 bytes
 .../query-tests/queries/function_parse_json.json   |    60 +
 .../queries/function_parse_json.proto.bin          |   Bin 0 -> 545 bytes
 .../query-tests/queries/function_parse_url.json    |    81 +
 .../queries/function_parse_url.proto.bin           |   Bin 0 -> 728 bytes
 .../queries/function_parse_url_with_key.json       |   102 +
 .../queries/function_parse_url_with_key.proto.bin  |   Bin 0 -> 913 bytes
 .../query-tests/queries/function_percent_rank.json |   107 +
 .../queries/function_percent_rank.proto.bin        |   Bin 0 -> 910 bytes
 .../queries/function_percentile_approx.json        |   102 +
 .../queries/function_percentile_approx.proto.bin   |   Bin 0 -> 934 bytes
 .../function_percentile_with_frequency.json        |   102 +
 .../function_percentile_with_frequency.proto.bin   |   Bin 0 -> 920 bytes
 .../function_percentile_without_frequency.json     |    81 +
 ...function_percentile_without_frequency.proto.bin |   Bin 0 -> 736 bytes
 .../resources/query-tests/queries/function_pi.json |    38 +
 .../query-tests/queries/function_pi.proto.bin      |   Bin 0 -> 343 bytes
 .../query-tests/queries/function_pmod.json         |    81 +
 .../query-tests/queries/function_pmod.proto.bin    |   Bin 0 -> 717 bytes
 .../query-tests/queries/function_posexplode.json   |    60 +
 .../queries/function_posexplode.proto.bin          |   Bin 0 -> 545 bytes
 .../queries/function_posexplode_outer.json         |    60 +
 .../queries/function_posexplode_outer.proto.bin    |   Bin 0 -> 557 bytes
 .../query-tests/queries/function_position.json     |    81 +
 .../queries/function_position.proto.bin            |   Bin 0 -> 726 bytes
 .../queries/function_position_with_start.json      |   102 +
 .../queries/function_position_with_start.proto.bin |   Bin 0 -> 911 bytes
 .../query-tests/queries/function_positive.json     |    60 +
 .../queries/function_positive.proto.bin            |   Bin 0 -> 541 bytes
 .../query-tests/queries/function_pow.json          |    81 +
 .../query-tests/queries/function_pow.proto.bin     |   Bin 0 -> 718 bytes
 .../query-tests/queries/function_power.json        |    81 +
 .../query-tests/queries/function_power.proto.bin   |   Bin 0 -> 720 bytes
 .../query-tests/queries/function_printf.json       |   102 +
 .../query-tests/queries/function_printf.proto.bin  |   Bin 0 -> 907 bytes
 .../query-tests/queries/function_product.json      |    60 +
 .../query-tests/queries/function_product.proto.bin |   Bin 0 -> 539 bytes
 .../query-tests/queries/function_quarter.json      |    60 +
 .../query-tests/queries/function_quarter.proto.bin |   Bin 0 -> 485 bytes
 .../query-tests/queries/function_radians.json      |    60 +
 .../query-tests/queries/function_radians.proto.bin |   Bin 0 -> 543 bytes
 .../query-tests/queries/function_raise_error.json  |    60 +
 .../queries/function_raise_error.proto.bin         |   Bin 0 -> 552 bytes
 .../queries/function_rand_with_seed.json           |    60 +
 .../queries/function_rand_with_seed.proto.bin      |   Bin 0 -> 534 bytes
 .../queries/function_randn_with_seed.json          |    60 +
 .../queries/function_randn_with_seed.proto.bin     |   Bin 0 -> 537 bytes
 .../queries/function_random_with_seed.json         |    60 +
 .../queries/function_random_with_seed.proto.bin    |   Bin 0 -> 536 bytes
 .../query-tests/queries/function_rank.json         |   107 +
 .../query-tests/queries/function_rank.proto.bin    |   Bin 0 -> 894 bytes
 .../query-tests/queries/function_reduce.json       |   166 +
 .../query-tests/queries/function_reduce.proto.bin  |   Bin 0 -> 1322 bytes
 .../query-tests/queries/function_reflect.json      |   102 +
 .../query-tests/queries/function_reflect.proto.bin |   Bin 0 -> 931 bytes
 .../query-tests/queries/function_regexp.json       |    81 +
 .../query-tests/queries/function_regexp.proto.bin  |   Bin 0 -> 728 bytes
 .../query-tests/queries/function_regexp_count.json |    81 +
 .../queries/function_regexp_count.proto.bin        |   Bin 0 -> 736 bytes
 .../queries/function_regexp_extract.json           |   102 +
 .../queries/function_regexp_extract.proto.bin      |   Bin 0 -> 954 bytes
 ..._regexp_extract_all_with_regex_group_index.json |   102 +
 ...xp_extract_all_with_regex_group_index.proto.bin |   Bin 0 -> 942 bytes
 ...gexp_extract_all_without_regex_group_index.json |    81 +
 ...extract_all_without_regex_group_index.proto.bin |   Bin 0 -> 758 bytes
 ...nction_regexp_instr_with_regex_group_index.json |   102 +
 ...n_regexp_instr_with_regex_group_index.proto.bin |   Bin 0 -> 927 bytes
 ...ion_regexp_instr_without_regex_group_index.json |    81 +
 ...egexp_instr_without_regex_group_index.proto.bin |   Bin 0 -> 743 bytes
 .../query-tests/queries/function_regexp_like.json  |    81 +
 .../queries/function_regexp_like.proto.bin         |   Bin 0 -> 738 bytes
 .../queries/function_regexp_replace.json           |   102 +
 .../queries/function_regexp_replace.proto.bin      |   Bin 0 -> 951 bytes
 .../queries/function_regexp_substr.json            |    81 +
 .../queries/function_regexp_substr.proto.bin       |   Bin 0 -> 747 bytes
 .../query-tests/queries/function_regr_avgx.json    |    81 +
 .../queries/function_regr_avgx.proto.bin           |   Bin 0 -> 728 bytes
 .../query-tests/queries/function_regr_avgy.json    |    81 +
 .../queries/function_regr_avgy.proto.bin           |   Bin 0 -> 728 bytes
 .../query-tests/queries/function_regr_count.json   |    81 +
 .../queries/function_regr_count.proto.bin          |   Bin 0 -> 730 bytes
 .../queries/function_regr_intercept.json           |    81 +
 .../queries/function_regr_intercept.proto.bin      |   Bin 0 -> 738 bytes
 .../query-tests/queries/function_regr_r2.json      |    81 +
 .../query-tests/queries/function_regr_r2.proto.bin |   Bin 0 -> 724 bytes
 .../query-tests/queries/function_regr_slope.json   |    81 +
 .../queries/function_regr_slope.proto.bin          |   Bin 0 -> 730 bytes
 .../query-tests/queries/function_regr_sxx.json     |    81 +
 .../queries/function_regr_sxx.proto.bin            |   Bin 0 -> 726 bytes
 .../query-tests/queries/function_regr_sxy.json     |    81 +
 .../queries/function_regr_sxy.proto.bin            |   Bin 0 -> 726 bytes
 .../query-tests/queries/function_regr_syy.json     |    81 +
 .../queries/function_regr_syy.proto.bin            |   Bin 0 -> 726 bytes
 .../query-tests/queries/function_replace.json      |    81 +
 .../query-tests/queries/function_replace.proto.bin |   Bin 0 -> 724 bytes
 .../function_replace_with_specified_string.json    |   102 +
 ...unction_replace_with_specified_string.proto.bin |   Bin 0 -> 909 bytes
 .../query-tests/queries/function_reverse.json      |    60 +
 .../query-tests/queries/function_reverse.proto.bin |   Bin 0 -> 539 bytes
 .../query-tests/queries/function_right.json        |    81 +
 .../query-tests/queries/function_right.proto.bin   |   Bin 0 -> 720 bytes
 .../query-tests/queries/function_rint.json         |    60 +
 .../query-tests/queries/function_rint.proto.bin    |   Bin 0 -> 534 bytes
 .../query-tests/queries/function_rlike.json        |    81 +
 .../query-tests/queries/function_rlike.proto.bin   |   Bin 0 -> 726 bytes
 .../query-tests/queries/function_round.json        |    81 +
 .../query-tests/queries/function_round.proto.bin   |   Bin 0 -> 721 bytes
 .../query-tests/queries/function_row_number.json   |   107 +
 .../queries/function_row_number.proto.bin          |   Bin 0 -> 906 bytes
 .../query-tests/queries/function_rpad.json         |   102 +
 .../query-tests/queries/function_rpad.proto.bin    |   Bin 0 -> 904 bytes
 .../query-tests/queries/function_rpad_binary.json  |   102 +
 .../queries/function_rpad_binary.proto.bin         |   Bin 0 -> 802 bytes
 .../query-tests/queries/function_rtrim.json        |    60 +
 .../query-tests/queries/function_rtrim.proto.bin   |   Bin 0 -> 535 bytes
 .../queries/function_rtrim_with_pattern.json       |    81 +
 .../queries/function_rtrim_with_pattern.proto.bin  |   Bin 0 -> 724 bytes
 .../queries/function_schema_of_csv.json            |   125 +
 .../queries/function_schema_of_csv.proto.bin       |   Bin 0 -> 1147 bytes
 .../queries/function_schema_of_json.json           |    60 +
 .../queries/function_schema_of_json.proto.bin      |   Bin 0 -> 564 bytes
 .../function_schema_of_json_with_options.json      |   125 +
 .../function_schema_of_json_with_options.proto.bin |   Bin 0 -> 1183 bytes
 .../queries/function_schema_of_variant.json        |    83 +
 .../queries/function_schema_of_variant.proto.bin   |   Bin 0 -> 763 bytes
 .../queries/function_schema_of_variant_agg.json    |    83 +
 .../function_schema_of_variant_agg.proto.bin       |   Bin 0 -> 771 bytes
 .../query-tests/queries/function_sec.json          |    60 +
 .../query-tests/queries/function_sec.proto.bin     |   Bin 0 -> 531 bytes
 .../query-tests/queries/function_second.json       |    60 +
 .../query-tests/queries/function_second.proto.bin  |   Bin 0 -> 483 bytes
 .../query-tests/queries/function_sentences.json    |    60 +
 .../queries/function_sentences.proto.bin           |   Bin 0 -> 543 bytes
 .../queries/function_sentences_with_language.json  |    81 +
 .../function_sentences_with_language.proto.bin     |   Bin 0 -> 729 bytes
 ...nction_sentences_with_language_and_country.json |   102 +
 ...n_sentences_with_language_and_country.proto.bin |   Bin 0 -> 915 bytes
 .../query-tests/queries/function_sequence.json     |    81 +
 .../queries/function_sequence.proto.bin            |   Bin 0 -> 724 bytes
 .../query-tests/queries/function_session_user.json |    38 +
 .../queries/function_session_user.proto.bin        |   Bin 0 -> 363 bytes
 .../queries/function_session_window.json           |    81 +
 .../queries/function_session_window.proto.bin      |   Bin 0 -> 704 bytes
 .../queries/function_session_window_orphaned.json  |    34 +
 .../function_session_window_orphaned.proto.bin     |   Bin 0 -> 165 bytes
 .../query-tests/queries/function_sha.json          |    60 +
 .../query-tests/queries/function_sha.proto.bin     |   Bin 0 -> 531 bytes
 .../query-tests/queries/function_sha1.json         |    85 +
 .../query-tests/queries/function_sha1.proto.bin    |   Bin 0 -> 714 bytes
 .../query-tests/queries/function_sha2.json         |   106 +
 .../query-tests/queries/function_sha2.proto.bin    |   Bin 0 -> 900 bytes
 .../query-tests/queries/function_shiftleft.json    |    81 +
 .../queries/function_shiftleft.proto.bin           |   Bin 0 -> 733 bytes
 .../query-tests/queries/function_shiftright.json   |    81 +
 .../queries/function_shiftright.proto.bin          |   Bin 0 -> 736 bytes
 .../queries/function_shiftrightunsigned.json       |    81 +
 .../queries/function_shiftrightunsigned.proto.bin  |   Bin 0 -> 760 bytes
 .../query-tests/queries/function_sign.json         |    60 +
 .../query-tests/queries/function_sign.proto.bin    |   Bin 0 -> 533 bytes
 .../query-tests/queries/function_signum.json       |    60 +
 .../query-tests/queries/function_signum.proto.bin  |   Bin 0 -> 540 bytes
 .../query-tests/queries/function_sin.json          |    60 +
 .../query-tests/queries/function_sin.proto.bin     |   Bin 0 -> 531 bytes
 .../query-tests/queries/function_sinh.json         |    60 +
 .../query-tests/queries/function_sinh.proto.bin    |   Bin 0 -> 534 bytes
 .../query-tests/queries/function_size.json         |    60 +
 .../query-tests/queries/function_size.proto.bin    |   Bin 0 -> 533 bytes
 .../query-tests/queries/function_skewness.json     |    60 +
 .../queries/function_skewness.proto.bin            |   Bin 0 -> 546 bytes
 .../query-tests/queries/function_slice.json        |   102 +
 .../query-tests/queries/function_slice.proto.bin   |   Bin 0 -> 907 bytes
 .../query-tests/queries/function_some.json         |    60 +
 .../query-tests/queries/function_some.proto.bin    |   Bin 0 -> 427 bytes
 .../query-tests/queries/function_sort_array.json   |    81 +
 .../queries/function_sort_array.proto.bin          |   Bin 0 -> 736 bytes
 .../queries/function_spark_partition_id.json       |    38 +
 .../queries/function_spark_partition_id.proto.bin  |   Bin 0 -> 375 bytes
 .../query-tests/queries/function_split.json        |    81 +
 .../query-tests/queries/function_split.proto.bin   |   Bin 0 -> 722 bytes
 .../query-tests/queries/function_split_part.json   |   102 +
 .../queries/function_split_part.proto.bin          |   Bin 0 -> 915 bytes
 .../queries/function_split_using_columns.json      |    81 +
 .../queries/function_split_using_columns.proto.bin |   Bin 0 -> 720 bytes
 .../queries/function_split_with_limit.json         |   102 +
 .../queries/function_split_with_limit.proto.bin    |   Bin 0 -> 908 bytes
 .../function_split_with_limit_using_columns.json   |   102 +
 ...nction_split_with_limit_using_columns.proto.bin |   Bin 0 -> 905 bytes
 .../query-tests/queries/function_sqrt.json         |    60 +
 .../query-tests/queries/function_sqrt.proto.bin    |   Bin 0 -> 534 bytes
 .../query-tests/queries/function_stack.json        |   123 +
 .../query-tests/queries/function_stack.proto.bin   |   Bin 0 -> 1089 bytes
 .../query-tests/queries/function_startswith.json   |    81 +
 .../queries/function_startswith.proto.bin          |   Bin 0 -> 730 bytes
 .../query-tests/queries/function_std.json          |    60 +
 .../query-tests/queries/function_std.proto.bin     |   Bin 0 -> 531 bytes
 .../query-tests/queries/function_stddev.json       |    60 +
 .../query-tests/queries/function_stddev.proto.bin  |   Bin 0 -> 540 bytes
 .../query-tests/queries/function_stddev_pop.json   |    60 +
 .../queries/function_stddev_pop.proto.bin          |   Bin 0 -> 552 bytes
 .../query-tests/queries/function_stddev_samp.json  |    60 +
 .../queries/function_stddev_samp.proto.bin         |   Bin 0 -> 555 bytes
 .../query-tests/queries/function_str_to_map.json   |    60 +
 .../queries/function_str_to_map.proto.bin          |   Bin 0 -> 545 bytes
 ...tr_to_map_with_pair_and_keyValue_delimiter.json |    81 +
 ..._map_with_pair_and_keyValue_delimiter.proto.bin |   Bin 0 -> 730 bytes
 .../function_str_to_map_with_pair_delimiter.json   |   102 +
 ...nction_str_to_map_with_pair_delimiter.proto.bin |   Bin 0 -> 915 bytes
 .../query-tests/queries/function_struct.json       |    81 +
 .../query-tests/queries/function_struct.proto.bin  |   Bin 0 -> 728 bytes
 .../query-tests/queries/function_substr.json       |    81 +
 .../query-tests/queries/function_substr.proto.bin  |   Bin 0 -> 722 bytes
 .../queries/function_substr_with_len.json          |   102 +
 .../queries/function_substr_with_len.proto.bin     |   Bin 0 -> 907 bytes
 .../query-tests/queries/function_substring.json    |   102 +
 .../queries/function_substring.proto.bin           |   Bin 0 -> 923 bytes
 .../queries/function_substring_index.json          |   102 +
 .../queries/function_substring_index.proto.bin     |   Bin 0 -> 948 bytes
 .../queries/function_substring_using_columns.json  |   102 +
 .../function_substring_using_columns.proto.bin     |   Bin 0 -> 913 bytes
 .../query-tests/queries/function_sum.json          |    60 +
 .../query-tests/queries/function_sum.proto.bin     |   Bin 0 -> 531 bytes
 .../query-tests/queries/function_sum_distinct.json |    61 +
 .../queries/function_sum_distinct.proto.bin        |   Bin 0 -> 542 bytes
 .../query-tests/queries/function_tan.json          |    60 +
 .../query-tests/queries/function_tan.proto.bin     |   Bin 0 -> 531 bytes
 .../query-tests/queries/function_tanh.json         |    60 +
 .../query-tests/queries/function_tanh.proto.bin    |   Bin 0 -> 534 bytes
 .../queries/function_timestamp_add.json            |   102 +
 .../queries/function_timestamp_add.proto.bin       |   Bin 0 -> 879 bytes
 .../queries/function_timestamp_diff.json           |   102 +
 .../queries/function_timestamp_diff.proto.bin      |   Bin 0 -> 882 bytes
 .../queries/function_timestamp_micros.json         |    60 +
 .../queries/function_timestamp_micros.proto.bin    |   Bin 0 -> 503 bytes
 .../queries/function_timestamp_millis.json         |    60 +
 .../queries/function_timestamp_millis.proto.bin    |   Bin 0 -> 503 bytes
 .../queries/function_timestamp_seconds.json        |    60 +
 .../queries/function_timestamp_seconds.proto.bin   |   Bin 0 -> 505 bytes
 .../query-tests/queries/function_to_binary.json    |    60 +
 .../queries/function_to_binary.proto.bin           |   Bin 0 -> 543 bytes
 .../queries/function_to_binary_with_format.json    |    81 +
 .../function_to_binary_with_format.proto.bin       |   Bin 0 -> 732 bytes
 .../query-tests/queries/function_to_char.json      |    81 +
 .../query-tests/queries/function_to_char.proto.bin |   Bin 0 -> 729 bytes
 .../query-tests/queries/function_to_csv.json       |   125 +
 .../query-tests/queries/function_to_csv.proto.bin  |   Bin 0 -> 1108 bytes
 .../query-tests/queries/function_to_date.json      |    60 +
 .../query-tests/queries/function_to_date.proto.bin |   Bin 0 -> 485 bytes
 .../queries/function_to_date_with_format.json      |    81 +
 .../queries/function_to_date_with_format.proto.bin |   Bin 0 -> 683 bytes
 .../query-tests/queries/function_to_json.json      |   125 +
 .../query-tests/queries/function_to_json.proto.bin |   Bin 0 -> 1134 bytes
 .../query-tests/queries/function_to_number.json    |    81 +
 .../queries/function_to_number.proto.bin           |   Bin 0 -> 733 bytes
 .../query-tests/queries/function_to_timestamp.json |    60 +
 .../queries/function_to_timestamp.proto.bin        |   Bin 0 -> 495 bytes
 .../queries/function_to_timestamp_ltz.json         |    60 +
 .../queries/function_to_timestamp_ltz.proto.bin    |   Bin 0 -> 557 bytes
 .../function_to_timestamp_ltz_with_format.json     |    81 +
 ...function_to_timestamp_ltz_with_format.proto.bin |   Bin 0 -> 742 bytes
 .../queries/function_to_timestamp_ntz.json         |    60 +
 .../queries/function_to_timestamp_ntz.proto.bin    |   Bin 0 -> 557 bytes
 .../function_to_timestamp_ntz_with_format.json     |    81 +
 ...function_to_timestamp_ntz_with_format.proto.bin |   Bin 0 -> 742 bytes
 .../queries/function_to_timestamp_with_format.json |    81 +
 .../function_to_timestamp_with_format.proto.bin    |   Bin 0 -> 712 bytes
 .../queries/function_to_unix_timestamp.json        |    60 +
 .../queries/function_to_unix_timestamp.proto.bin   |   Bin 0 -> 559 bytes
 .../function_to_unix_timestamp_with_format.json    |    81 +
 ...unction_to_unix_timestamp_with_format.proto.bin |   Bin 0 -> 744 bytes
 .../queries/function_to_utc_timestamp.json         |    81 +
 .../queries/function_to_utc_timestamp.proto.bin    |   Bin 0 -> 706 bytes
 .../query-tests/queries/function_to_varchar.json   |    81 +
 .../queries/function_to_varchar.proto.bin          |   Bin 0 -> 735 bytes
 .../query-tests/queries/function_transform.json    |   132 +
 .../queries/function_transform.proto.bin           |   Bin 0 -> 1110 bytes
 .../queries/function_transform_keys.json           |   160 +
 .../queries/function_transform_keys.proto.bin      |   Bin 0 -> 1338 bytes
 .../queries/function_transform_values.json         |   117 +
 .../queries/function_transform_values.proto.bin    |   Bin 0 -> 971 bytes
 .../queries/function_transform_with_index.json     |   117 +
 .../function_transform_with_index.proto.bin        |   Bin 0 -> 946 bytes
 .../query-tests/queries/function_translate.json    |   102 +
 .../queries/function_translate.proto.bin           |   Bin 0 -> 929 bytes
 .../query-tests/queries/function_trim.json         |    60 +
 .../query-tests/queries/function_trim.proto.bin    |   Bin 0 -> 533 bytes
 .../queries/function_trim_with_pattern.json        |    81 +
 .../queries/function_trim_with_pattern.proto.bin   |   Bin 0 -> 721 bytes
 .../query-tests/queries/function_trunc.json        |    81 +
 .../query-tests/queries/function_trunc.proto.bin   |   Bin 0 -> 669 bytes
 .../query-tests/queries/function_try_add.json      |    81 +
 .../query-tests/queries/function_try_add.proto.bin |   Bin 0 -> 724 bytes
 .../queries/function_try_aes_decrypt.json          |    81 +
 .../queries/function_try_aes_decrypt.proto.bin     |   Bin 0 -> 740 bytes
 .../function_try_aes_decrypt_with_mode.json        |   102 +
 .../function_try_aes_decrypt_with_mode.proto.bin   |   Bin 0 -> 925 bytes
 ...function_try_aes_decrypt_with_mode_padding.json |   123 +
 ...ion_try_aes_decrypt_with_mode_padding.proto.bin |   Bin 0 -> 1110 bytes
 ...tion_try_aes_decrypt_with_mode_padding_aad.json |   144 +
 ...try_aes_decrypt_with_mode_padding_aad.proto.bin |   Bin 0 -> 1295 bytes
 .../query-tests/queries/function_try_avg.json      |    60 +
 .../query-tests/queries/function_try_avg.proto.bin |   Bin 0 -> 539 bytes
 .../query-tests/queries/function_try_divide.json   |    81 +
 .../queries/function_try_divide.proto.bin          |   Bin 0 -> 730 bytes
 .../queries/function_try_element_at_array.json     |    81 +
 .../function_try_element_at_array.proto.bin        |   Bin 0 -> 738 bytes
 .../queries/function_try_element_at_map.json       |    81 +
 .../queries/function_try_element_at_map.proto.bin  |   Bin 0 -> 738 bytes
 .../queries/function_try_make_interval_years.json  |    60 +
 .../function_try_make_interval_years.proto.bin     |   Bin 0 -> 559 bytes
 .../function_try_make_interval_years_months.json   |    81 +
 ...nction_try_make_interval_years_months.proto.bin |   Bin 0 -> 744 bytes
 ...ction_try_make_interval_years_months_weeks.json |   102 +
 ..._try_make_interval_years_months_weeks.proto.bin |   Bin 0 -> 929 bytes
 ..._try_make_interval_years_months_weeks_days.json |   123 +
 ...make_interval_years_months_weeks_days.proto.bin |   Bin 0 -> 1114 bytes
 ...ake_interval_years_months_weeks_days_hours.json |   144 +
 ...nterval_years_months_weeks_days_hours.proto.bin |   Bin 0 -> 1299 bytes
 ...nterval_years_months_weeks_days_hours_mins.json |   165 +
 ...al_years_months_weeks_days_hours_mins.proto.bin |   Bin 0 -> 1484 bytes
 ...al_years_months_weeks_days_hours_mins_secs.json |   186 +
 ...ars_months_weeks_days_hours_mins_secs.proto.bin |   Bin 0 -> 1669 bytes
 ...ction_try_make_timestamp_ltz_with_timezone.json |   186 +
 ..._try_make_timestamp_ltz_with_timezone.proto.bin |   Bin 0 -> 1679 bytes
 ...on_try_make_timestamp_ltz_without_timezone.json |   165 +
 ...y_make_timestamp_ltz_without_timezone.proto.bin |   Bin 0 -> 1494 bytes
 .../queries/function_try_make_timestamp_ntz.json   |   165 +
 .../function_try_make_timestamp_ntz.proto.bin      |   Bin 0 -> 1494 bytes
 .../function_try_make_timestamp_with_timezone.json |   186 +
 ...tion_try_make_timestamp_with_timezone.proto.bin |   Bin 0 -> 1671 bytes
 ...nction_try_make_timestamp_without_timezone.json |   165 +
 ...n_try_make_timestamp_without_timezone.proto.bin |   Bin 0 -> 1486 bytes
 .../query-tests/queries/function_try_multiply.json |    81 +
 .../queries/function_try_multiply.proto.bin        |   Bin 0 -> 734 bytes
 .../queries/function_try_parse_json.json           |    60 +
 .../queries/function_try_parse_json.proto.bin      |   Bin 0 -> 553 bytes
 .../queries/function_try_parse_url.json            |    81 +
 .../queries/function_try_parse_url.proto.bin       |   Bin 0 -> 736 bytes
 .../queries/function_try_parse_url_with_key.json   |   102 +
 .../function_try_parse_url_with_key.proto.bin      |   Bin 0 -> 921 bytes
 .../query-tests/queries/function_try_reflect.json  |   102 +
 .../queries/function_try_reflect.proto.bin         |   Bin 0 -> 939 bytes
 .../query-tests/queries/function_try_subtract.json |    81 +
 .../queries/function_try_subtract.proto.bin        |   Bin 0 -> 734 bytes
 .../query-tests/queries/function_try_sum.json      |    60 +
 .../query-tests/queries/function_try_sum.proto.bin |   Bin 0 -> 539 bytes
 .../queries/function_try_to_binary.json            |    81 +
 .../queries/function_try_to_binary.proto.bin       |   Bin 0 -> 741 bytes
 .../function_try_to_binary_without_format.json     |    60 +
 ...function_try_to_binary_without_format.proto.bin |   Bin 0 -> 551 bytes
 .../query-tests/queries/function_try_to_date.json  |    60 +
 .../queries/function_try_to_date.proto.bin         |   Bin 0 -> 493 bytes
 .../queries/function_try_to_date_with_format.json  |    81 +
 .../function_try_to_date_with_format.proto.bin     |   Bin 0 -> 695 bytes
 .../queries/function_try_to_number.json            |    81 +
 .../queries/function_try_to_number.proto.bin       |   Bin 0 -> 741 bytes
 .../queries/function_try_to_timestamp.json         |    81 +
 .../queries/function_try_to_timestamp.proto.bin    |   Bin 0 -> 742 bytes
 .../function_try_to_timestamp_without_format.json  |    60 +
 ...ction_try_to_timestamp_without_format.proto.bin |   Bin 0 -> 557 bytes
 .../queries/function_try_url_decode.json           |    60 +
 .../queries/function_try_url_decode.proto.bin      |   Bin 0 -> 553 bytes
 .../queries/function_try_variant_get.json          |   125 +
 .../queries/function_try_variant_get.proto.bin     |   Bin 0 -> 1155 bytes
 .../query-tests/queries/function_typedLit.json     |  2061 ++
 .../queries/function_typedLit.proto.bin            |   Bin 0 -> 11681 bytes
 .../query-tests/queries/function_typeof.json       |    60 +
 .../query-tests/queries/function_typeof.proto.bin  |   Bin 0 -> 537 bytes
 .../query-tests/queries/function_ucase.json        |    60 +
 .../query-tests/queries/function_ucase.proto.bin   |   Bin 0 -> 535 bytes
 .../query-tests/queries/function_unbase64.json     |    60 +
 .../queries/function_unbase64.proto.bin            |   Bin 0 -> 541 bytes
 .../query-tests/queries/function_unhex.json        |    60 +
 .../query-tests/queries/function_unhex.proto.bin   |   Bin 0 -> 535 bytes
 .../query-tests/queries/function_unix_date.json    |   104 +
 .../queries/function_unix_date.proto.bin           |   Bin 0 -> 885 bytes
 .../query-tests/queries/function_unix_micros.json  |   104 +
 .../queries/function_unix_micros.proto.bin         |   Bin 0 -> 918 bytes
 .../query-tests/queries/function_unix_millis.json  |   104 +
 .../queries/function_unix_millis.proto.bin         |   Bin 0 -> 918 bytes
 .../query-tests/queries/function_unix_seconds.json |   104 +
 .../queries/function_unix_seconds.proto.bin        |   Bin 0 -> 920 bytes
 .../queries/function_unix_timestamp.json           |    61 +
 .../queries/function_unix_timestamp.proto.bin      |   Bin 0 -> 528 bytes
 .../function_unix_timestamp_with_format.json       |    81 +
 .../function_unix_timestamp_with_format.proto.bin  |   Bin 0 -> 718 bytes
 .../query-tests/queries/function_upper.json        |    60 +
 .../query-tests/queries/function_upper.proto.bin   |   Bin 0 -> 535 bytes
 .../query-tests/queries/function_url_decode.json   |    60 +
 .../queries/function_url_decode.proto.bin          |   Bin 0 -> 545 bytes
 .../query-tests/queries/function_url_encode.json   |    60 +
 .../queries/function_url_encode.proto.bin          |   Bin 0 -> 545 bytes
 .../query-tests/queries/function_user.json         |    38 +
 .../query-tests/queries/function_user.proto.bin    |   Bin 0 -> 347 bytes
 .../query-tests/queries/function_var_pop.json      |    60 +
 .../query-tests/queries/function_var_pop.proto.bin |   Bin 0 -> 543 bytes
 .../query-tests/queries/function_var_samp.json     |    60 +
 .../queries/function_var_samp.proto.bin            |   Bin 0 -> 546 bytes
 .../query-tests/queries/function_variance.json     |    60 +
 .../queries/function_variance.proto.bin            |   Bin 0 -> 546 bytes
 .../query-tests/queries/function_variant_get.json  |   125 +
 .../queries/function_variant_get.proto.bin         |   Bin 0 -> 1139 bytes
 .../query-tests/queries/function_weekday.json      |    60 +
 .../query-tests/queries/function_weekday.proto.bin |   Bin 0 -> 485 bytes
 .../query-tests/queries/function_weekofyear.json   |    60 +
 .../queries/function_weekofyear.proto.bin          |   Bin 0 -> 491 bytes
 .../query-tests/queries/function_window.json       |   123 +
 .../query-tests/queries/function_window.proto.bin  |   Bin 0 -> 1068 bytes
 .../query-tests/queries/function_window_time.json  |    94 +
 .../queries/function_window_time.proto.bin         |   Bin 0 -> 739 bytes
 .../query-tests/queries/function_xpath.json        |    81 +
 .../query-tests/queries/function_xpath.proto.bin   |   Bin 0 -> 675 bytes
 .../queries/function_xpath_boolean.json            |    81 +
 .../queries/function_xpath_boolean.proto.bin       |   Bin 0 -> 684 bytes
 .../query-tests/queries/function_xpath_double.json |    81 +
 .../queries/function_xpath_double.proto.bin        |   Bin 0 -> 682 bytes
 .../query-tests/queries/function_xpath_float.json  |    81 +
 .../queries/function_xpath_float.proto.bin         |   Bin 0 -> 680 bytes
 .../query-tests/queries/function_xpath_int.json    |    81 +
 .../queries/function_xpath_int.proto.bin           |   Bin 0 -> 676 bytes
 .../query-tests/queries/function_xpath_long.json   |    81 +
 .../queries/function_xpath_long.proto.bin          |   Bin 0 -> 678 bytes
 .../query-tests/queries/function_xpath_number.json |    81 +
 .../queries/function_xpath_number.proto.bin        |   Bin 0 -> 682 bytes
 .../query-tests/queries/function_xpath_short.json  |    81 +
 .../queries/function_xpath_short.proto.bin         |   Bin 0 -> 680 bytes
 .../query-tests/queries/function_xpath_string.json |    81 +
 .../queries/function_xpath_string.proto.bin        |   Bin 0 -> 682 bytes
 .../query-tests/queries/function_xxhash64.json     |   123 +
 .../queries/function_xxhash64.proto.bin            |   Bin 0 -> 1097 bytes
 .../query-tests/queries/function_year.json         |    60 +
 .../query-tests/queries/function_year.proto.bin    |   Bin 0 -> 479 bytes
 .../query-tests/queries/function_years.json        |    60 +
 .../query-tests/queries/function_years.proto.bin   |   Bin 0 -> 531 bytes
 .../query-tests/queries/function_zip_with.json     |   138 +
 .../queries/function_zip_with.proto.bin            |   Bin 0 -> 1128 bytes
 .../resources/query-tests/queries/groupby_agg.json |   398 +
 .../query-tests/queries/groupby_agg.proto.bin      |   Bin 0 -> 3139 bytes
 .../query-tests/queries/groupby_agg_columns.json   |   127 +
 .../queries/groupby_agg_columns.proto.bin          |   Bin 0 -> 983 bytes
 .../query-tests/queries/groupby_agg_orphaned.json  |   101 +
 .../queries/groupby_agg_orphaned.proto.bin         |   Bin 0 -> 210 bytes
 .../query-tests/queries/groupby_agg_string.json    |   152 +
 .../queries/groupby_agg_string.proto.bin           |   Bin 0 -> 1161 bytes
 .../resources/query-tests/queries/groupby_avg.json |   129 +
 .../query-tests/queries/groupby_avg.proto.bin      |   Bin 0 -> 1017 bytes
 .../query-tests/queries/groupby_count.json         |   105 +
 .../query-tests/queries/groupby_count.proto.bin    |   Bin 0 -> 757 bytes
 .../resources/query-tests/queries/groupby_max.json |   129 +
 .../query-tests/queries/groupby_max.proto.bin      |   Bin 0 -> 1017 bytes
 .../query-tests/queries/groupby_mean.json          |   129 +
 .../query-tests/queries/groupby_mean.proto.bin     |   Bin 0 -> 1017 bytes
 .../resources/query-tests/queries/groupby_min.json |   129 +
 .../query-tests/queries/groupby_min.proto.bin      |   Bin 0 -> 1017 bytes
 .../resources/query-tests/queries/groupby_sum.json |   129 +
 .../query-tests/queries/groupby_sum.proto.bin      |   Bin 0 -> 1017 bytes
 .../query-tests/queries/groupingSets.json          |   154 +
 .../query-tests/queries/groupingSets.proto.bin     |   Bin 0 -> 1147 bytes
 .../queries/grouping_and_grouping_id.json          |   215 +
 .../queries/grouping_and_grouping_id.proto.bin     |   Bin 0 -> 1805 bytes
 .../test/resources/query-tests/queries/hint.json   |    38 +
 .../resources/query-tests/queries/hint.proto.bin   |   Bin 0 -> 248 bytes
 .../query-tests/queries/hll_sketch_agg.json        |    60 +
 .../query-tests/queries/hll_sketch_agg.proto.bin   |   Bin 0 -> 448 bytes
 .../queries/hll_sketch_agg_with_columnName.json    |    60 +
 .../hll_sketch_agg_with_columnName.proto.bin       |   Bin 0 -> 459 bytes
 ...l_sketch_agg_with_columnName_lgConfigK_int.json |    81 +
 ...tch_agg_with_columnName_lgConfigK_int.proto.bin |   Bin 0 -> 654 bytes
 .../hll_sketch_agg_with_column_lgConfigK.json      |    81 +
 .../hll_sketch_agg_with_column_lgConfigK.proto.bin |   Bin 0 -> 632 bytes
 .../hll_sketch_agg_with_column_lgConfigK_int.json  |    81 +
 ..._sketch_agg_with_column_lgConfigK_int.proto.bin |   Bin 0 -> 643 bytes
 .../query-tests/queries/hll_union_agg.json         |    60 +
 .../query-tests/queries/hll_union_agg.proto.bin    |   Bin 0 -> 446 bytes
 .../queries/hll_union_agg_with_columnName.json     |    60 +
 .../hll_union_agg_with_columnName.proto.bin        |   Bin 0 -> 456 bytes
 ...columnName_allowDifferentLgConfigK_boolean.json |    81 +
 ...nName_allowDifferentLgConfigK_boolean.proto.bin |   Bin 0 -> 650 bytes
 ...on_agg_with_column_allowDifferentLgConfigK.json |    81 +
 ...g_with_column_allowDifferentLgConfigK.proto.bin |   Bin 0 -> 630 bytes
 ...ith_column_allowDifferentLgConfigK_boolean.json |    81 +
 ...olumn_allowDifferentLgConfigK_boolean.proto.bin |   Bin 0 -> 640 bytes
 .../resources/query-tests/queries/intersect.json   |    25 +
 .../query-tests/queries/intersect.proto.bin        |   Bin 0 -> 94 bytes
 .../query-tests/queries/intersectAll.json          |    25 +
 .../query-tests/queries/intersectAll.proto.bin     |   Bin 0 -> 94 bytes
 .../query-tests/queries/join_condition.json        |   106 +
 .../query-tests/queries/join_condition.proto.bin   |   Bin 0 -> 686 bytes
 .../query-tests/queries/join_inner_condition.json  |   106 +
 .../queries/join_inner_condition.proto.bin         |   Bin 0 -> 684 bytes
 .../queries/join_inner_no_condition.json           |    24 +
 .../queries/join_inner_no_condition.proto.bin      |   Bin 0 -> 98 bytes
 .../join_inner_using_multiple_col_array.json       |    25 +
 .../join_inner_using_multiple_col_array.proto.bin  |   Bin 0 -> 105 bytes
 .../queries/join_inner_using_multiple_col_seq.json |    25 +
 .../join_inner_using_multiple_col_seq.proto.bin    |   Bin 0 -> 105 bytes
 .../queries/join_inner_using_single_col.json       |    25 +
 .../queries/join_inner_using_single_col.proto.bin  |   Bin 0 -> 102 bytes
 .../queries/join_using_multiple_col_array.json     |    25 +
 .../join_using_multiple_col_array.proto.bin        |   Bin 0 -> 105 bytes
 .../queries/join_using_multiple_col_seq.json       |    25 +
 .../queries/join_using_multiple_col_seq.proto.bin  |   Bin 0 -> 105 bytes
 .../query-tests/queries/join_using_single_col.json |    25 +
 .../queries/join_using_single_col.proto.bin        |   Bin 0 -> 102 bytes
 .../query-tests/queries/json_from_dataset.json     |    39 +
 .../queries/json_from_dataset.proto.bin            |   Bin 0 -> 180 bytes
 .../test/resources/query-tests/queries/limit.json  |    16 +
 .../resources/query-tests/queries/limit.proto.bin  |   Bin 0 -> 50 bytes
 .../query-tests/queries/melt_no_values.json        |    60 +
 .../query-tests/queries/melt_no_values.proto.bin   |   Bin 0 -> 434 bytes
 .../resources/query-tests/queries/melt_values.json |    63 +
 .../query-tests/queries/melt_values.proto.bin      |   Bin 0 -> 437 bytes
 .../test/resources/query-tests/queries/offset.json |    16 +
 .../resources/query-tests/queries/offset.proto.bin |   Bin 0 -> 51 bytes
 .../query-tests/queries/orderBy_columns.json       |    92 +
 .../query-tests/queries/orderBy_columns.proto.bin  |   Bin 0 -> 628 bytes
 .../query-tests/queries/orderBy_strings.json       |    92 +
 .../query-tests/queries/orderBy_strings.proto.bin  |   Bin 0 -> 640 bytes
 .../test/resources/query-tests/queries/pivot.json  |   115 +
 .../resources/query-tests/queries/pivot.proto.bin  |   Bin 0 -> 801 bytes
 .../queries/pivot_without_column_values.json       |   108 +
 .../queries/pivot_without_column_values.proto.bin  |   Bin 0 -> 789 bytes
 .../test/resources/query-tests/queries/range.json  |    11 +
 .../resources/query-tests/queries/range.proto.bin  |   Bin 0 -> 14 bytes
 .../test/resources/query-tests/queries/read.json   |    16 +
 .../resources/query-tests/queries/read.proto.bin   |   Bin 0 -> 142 bytes
 .../query-tests/queries/read_changes.json          |    12 +
 .../query-tests/queries/read_changes.proto.bin     |   Bin 0 -> 58 bytes
 .../queries/read_changes_with_options.json         |    13 +
 .../queries/read_changes_with_options.proto.bin    |   Bin 0 -> 110 bytes
 .../resources/query-tests/queries/read_csv.json    |    11 +
 .../query-tests/queries/read_csv.proto.bin         |   Bin 0 -> 76 bytes
 .../resources/query-tests/queries/read_jdbc.json   |    14 +
 .../query-tests/queries/read_jdbc.proto.bin        |   Bin 0 -> 101 bytes
 .../queries/read_jdbc_with_partition.json          |    18 +
 .../queries/read_jdbc_with_partition.proto.bin     |   Bin 0 -> 177 bytes
 .../queries/read_jdbc_with_predicates.json         |    15 +
 .../queries/read_jdbc_with_predicates.proto.bin    |   Bin 0 -> 121 bytes
 .../resources/query-tests/queries/read_json.json   |    11 +
 .../query-tests/queries/read_json.proto.bin        |   Bin 0 -> 78 bytes
 .../resources/query-tests/queries/read_orc.json    |    11 +
 .../query-tests/queries/read_orc.proto.bin         |   Bin 0 -> 75 bytes
 .../query-tests/queries/read_parquet.json          |    11 +
 .../query-tests/queries/read_parquet.proto.bin     |   Bin 0 -> 83 bytes
 .../resources/query-tests/queries/read_table.json  |    10 +
 .../query-tests/queries/read_table.proto.bin       |   Bin 0 -> 17 bytes
 .../resources/query-tests/queries/read_text.json   |    11 +
 .../query-tests/queries/read_text.proto.bin        |   Bin 0 -> 77 bytes
 .../query-tests/queries/relation_extension.json    |    16 +
 .../queries/relation_extension.proto.bin           |   Bin 0 -> 108 bytes
 .../queries/relation_extension_deprecated.json     |    16 +
 .../relation_extension_deprecated.proto.bin        |   Bin 0 -> 108 bytes
 .../resources/query-tests/queries/repartition.json |    17 +
 .../query-tests/queries/repartition.proto.bin      |   Bin 0 -> 53 bytes
 .../query-tests/queries/repartitionById.json       |    67 +
 .../query-tests/queries/repartitionById.proto.bin  |   Bin 0 -> 426 bytes
 .../queries/repartitionByRange_expressions.json    |   104 +
 .../repartitionByRange_expressions.proto.bin       |   Bin 0 -> 794 bytes
 ...artitionByRange_num_partitions_expressions.json |   105 +
 ...ionByRange_num_partitions_expressions.proto.bin |   Bin 0 -> 821 bytes
 .../queries/repartition_expressions.json           |    58 +
 .../queries/repartition_expressions.proto.bin      |   Bin 0 -> 421 bytes
 .../repartition_num_partitions_expressions.json    |    59 +
 ...epartition_num_partitions_expressions.proto.bin |   Bin 0 -> 423 bytes
 .../resources/query-tests/queries/replace.json     |    24 +
 .../query-tests/queries/replace.proto.bin          |   Bin 0 -> 77 bytes
 .../query-tests/queries/rollup_column.json         |   126 +
 .../query-tests/queries/rollup_column.proto.bin    |   Bin 0 -> 937 bytes
 .../query-tests/queries/rollup_string.json         |   128 +
 .../query-tests/queries/rollup_string.proto.bin    |   Bin 0 -> 961 bytes
 .../resources/query-tests/queries/sampleBy.json    |    49 +
 .../query-tests/queries/sampleBy.proto.bin         |   Bin 0 -> 306 bytes
 .../query-tests/queries/sample_fraction_seed.json  |    18 +
 .../queries/sample_fraction_seed.proto.bin         |   Bin 0 -> 64 bytes
 .../sample_withReplacement_fraction_seed.json      |    18 +
 .../sample_withReplacement_fraction_seed.proto.bin |   Bin 0 -> 62 bytes
 .../test/resources/query-tests/queries/select.json |    37 +
 .../resources/query-tests/queries/select.proto.bin |   Bin 0 -> 235 bytes
 .../resources/query-tests/queries/selectExpr.json  |    58 +
 .../query-tests/queries/selectExpr.proto.bin       |   Bin 0 -> 462 bytes
 .../queries/select_collated_string.json            |    37 +
 .../queries/select_collated_string.proto.bin       |   Bin 0 -> 243 bytes
 .../query-tests/queries/select_strings.json        |    58 +
 .../query-tests/queries/select_strings.proto.bin   |   Bin 0 -> 432 bytes
 .../query-tests/queries/select_typed_1-arg.json    |   127 +
 .../queries/select_typed_1-arg.proto.bin           |   Bin 0 -> 1013 bytes
 .../query-tests/queries/select_typed_2-arg.json    |   146 +
 .../queries/select_typed_2-arg.proto.bin           |   Bin 0 -> 1182 bytes
 .../query-tests/queries/select_typed_3-arg.json    |   211 +
 .../queries/select_typed_3-arg.proto.bin           |   Bin 0 -> 1749 bytes
 .../query-tests/queries/select_typed_4-arg.json    |   276 +
 .../queries/select_typed_4-arg.proto.bin           |   Bin 0 -> 2317 bytes
 .../query-tests/queries/select_typed_5-arg.json    |   341 +
 .../queries/select_typed_5-arg.proto.bin           |   Bin 0 -> 2882 bytes
 .../queries/sortWithinPartitions_columns.json      |    67 +
 .../queries/sortWithinPartitions_columns.proto.bin |   Bin 0 -> 436 bytes
 .../queries/sortWithinPartitions_strings.json      |    67 +
 .../queries/sortWithinPartitions_strings.proto.bin |   Bin 0 -> 476 bytes
 .../query-tests/queries/sort_columns.json          |    67 +
 .../query-tests/queries/sort_columns.proto.bin     |   Bin 0 -> 436 bytes
 .../query-tests/queries/sort_strings.json          |    67 +
 .../query-tests/queries/sort_strings.proto.bin     |   Bin 0 -> 443 bytes
 .../streaming_changes_API_with_options.json        |    13 +
 .../streaming_changes_API_with_options.proto.bin   |   Bin 0 -> 93 bytes
 .../queries/streaming_table_API_with_options.json  |    15 +
 .../streaming_table_API_with_options.proto.bin     |   Bin 0 -> 55 bytes
 .../resources/query-tests/queries/summary.json     |    16 +
 .../query-tests/queries/summary.proto.bin          |   Bin 0 -> 60 bytes
 .../test/resources/query-tests/queries/table.json  |    10 +
 .../resources/query-tests/queries/table.proto.bin  |   Bin 0 -> 17 bytes
 .../queries/table_API_with_options.json            |    14 +
 .../queries/table_API_with_options.proto.bin       |   Bin 0 -> 44 bytes
 .../query-tests/queries/test_broadcast.json        |    33 +
 .../query-tests/queries/test_broadcast.proto.bin   |   Bin 0 -> 122 bytes
 .../src/test/resources/query-tests/queries/to.json |    34 +
 .../resources/query-tests/queries/to.proto.bin     |   Bin 0 -> 77 bytes
 .../test/resources/query-tests/queries/toDF.json   |    16 +
 .../resources/query-tests/queries/toDF.proto.bin   |   Bin 0 -> 61 bytes
 .../test/resources/query-tests/queries/toJSON.json |   105 +
 .../resources/query-tests/queries/toJSON.proto.bin |   Bin 0 -> 941 bytes
 .../query-tests/queries/to_avro_with_schema.json   |    81 +
 .../queries/to_avro_with_schema.proto.bin          |   Bin 0 -> 565 bytes
 .../queries/to_avro_without_schema.json            |    60 +
 .../queries/to_avro_without_schema.proto.bin       |   Bin 0 -> 387 bytes
 .../queries/to_protobuf_messageClassName.json      |   197 +
 .../queries/to_protobuf_messageClassName.proto.bin |   Bin 0 -> 1740 bytes
 .../to_protobuf_messageClassName_descFilePath.json |   218 +
 ...rotobuf_messageClassName_descFilePath.proto.bin |   Bin 0 -> 2135 bytes
 ...obuf_messageClassName_descFilePath_options.json |   283 +
 ...messageClassName_descFilePath_options.proto.bin |   Bin 0 -> 2654 bytes
 .../to_protobuf_messageClassName_options.json      |   262 +
 .../to_protobuf_messageClassName_options.proto.bin |   Bin 0 -> 2270 bytes
 .../queries/transpose_index_column.json            |    37 +
 .../queries/transpose_index_column.proto.bin       |   Bin 0 -> 236 bytes
 .../queries/transpose_no_index_column.json         |    15 +
 .../queries/transpose_no_index_column.proto.bin    |   Bin 0 -> 49 bytes
 .../test/resources/query-tests/queries/union.json  |    25 +
 .../resources/query-tests/queries/union.proto.bin  |   Bin 0 -> 94 bytes
 .../resources/query-tests/queries/unionAll.json    |    25 +
 .../query-tests/queries/unionAll.proto.bin         |   Bin 0 -> 94 bytes
 .../resources/query-tests/queries/unionByName.json |    43 +
 .../query-tests/queries/unionByName.proto.bin      |   Bin 0 -> 135 bytes
 .../queries/unionByName_allowMissingColumns.json   |    27 +
 .../unionByName_allowMissingColumns.proto.bin      |   Bin 0 -> 104 bytes
 .../query-tests/queries/unpivot_no_values.json     |    39 +
 .../queries/unpivot_no_values.proto.bin            |   Bin 0 -> 249 bytes
 .../query-tests/queries/unpivot_values.json        |    84 +
 .../query-tests/queries/unpivot_values.proto.bin   |   Bin 0 -> 622 bytes
 .../query-tests/queries/where_column.json          |    81 +
 .../query-tests/queries/where_column.proto.bin     |   Bin 0 -> 606 bytes
 .../resources/query-tests/queries/where_expr.json  |    37 +
 .../query-tests/queries/where_expr.proto.bin       |   Bin 0 -> 251 bytes
 .../query-tests/queries/width_bucket.json          |   123 +
 .../query-tests/queries/width_bucket.proto.bin     |   Bin 0 -> 997 bytes
 .../test/resources/query-tests/queries/window.json |   745 +
 .../resources/query-tests/queries/window.proto.bin |   Bin 0 -> 5728 bytes
 .../queries/withColumnRenamed_java_map.json        |    22 +
 .../queries/withColumnRenamed_java_map.proto.bin   |   Bin 0 -> 72 bytes
 .../queries/withColumnRenamed_scala_map.json       |    22 +
 .../queries/withColumnRenamed_scala_map.proto.bin  |   Bin 0 -> 72 bytes
 .../queries/withColumnRenamed_single.json          |    19 +
 .../queries/withColumnRenamed_single.proto.bin     |   Bin 0 -> 60 bytes
 .../query-tests/queries/withColumn_single.json     |    40 +
 .../queries/withColumn_single.proto.bin            |   Bin 0 -> 248 bytes
 .../query-tests/queries/withColumns_java_map.json  |    64 +
 .../queries/withColumns_java_map.proto.bin         |   Bin 0 -> 435 bytes
 .../query-tests/queries/withColumns_scala_map.json |    64 +
 .../queries/withColumns_scala_map.proto.bin        |   Bin 0 -> 446 bytes
 .../query-tests/queries/withMetadata.json          |    42 +
 .../query-tests/queries/withMetadata.proto.bin     |   Bin 0 -> 294 bytes
 .../query-tests/queries/zipWithIndex.json          |    81 +
 .../query-tests/queries/zipWithIndex.proto.bin     |   Bin 0 -> 644 bytes
 .../queries/zipWithIndex_custom_column.json        |    81 +
 .../queries/zipWithIndex_custom_column.proto.bin   |   Bin 0 -> 647 bytes
 .../resources/query-tests/test-data/people.csv     |     3 +
 .../resources/query-tests/test-data/people.json    |     3 +
 .../resources/query-tests/test-data/people.txt     |     3 +
 .../query-tests/test-data/streaming/csv/people.csv |     3 +
 .../query-tests/test-data/streaming/txt/people.txt |     3 +
 .../test/resources/query-tests/test-data/users.orc |   Bin 0 -> 547 bytes
 .../resources/query-tests/test-data/users.parquet  |   Bin 0 -> 615 bytes
 .../test/scala/org/apache/spark/sql/TestUDFs.scala |    43 +
 sql/connect/docs/adding-proto-messages.md          |    40 +
 sql/connect/docs/client-connection-string.md       |   144 +
 sql/connect/server/README.md                       |     5 +
 sql/connect/server/pom.xml                         |   394 +
 .../spark/sql/connect/plugin/CommandPlugin.java    |    32 +
 .../spark/sql/connect/plugin/ExpressionPlugin.java |    34 +
 .../spark/sql/connect/plugin/GetStatusPlugin.java  |    77 +
 .../spark/sql/connect/plugin/MLBackendPlugin.java  |    37 +
 .../spark/sql/connect/plugin/RelationPlugin.java   |    35 +
 .../spark/sql/connect/IllegalStateErrors.scala     |   125 +
 .../sql/connect/SimpleSparkConnectService.scala    |    64 +
 .../spark/sql/connect/SparkConnectPlugin.scala     |    58 +
 .../apache/spark/sql/connect/config/Connect.scala  |   465 +
 .../config/ConnectPlanCompressionAlgorithm.scala   |    21 +
 .../connect/execution/CachedStreamResponse.scala   |    32 +
 .../ConnectProgressExecutionListener.scala         |   197 +
 .../execution/ExecuteGrpcResponseSender.scala      |   418 +
 .../execution/ExecuteResponseObserver.scala        |   367 +
 .../connect/execution/ExecuteThreadRunner.scala    |   371 +
 .../execution/SparkConnectPlanExecution.scala      |   391 +
 .../org/apache/spark/sql/connect/ml/MLCache.scala  |   283 +
 .../apache/spark/sql/connect/ml/MLException.scala  |    59 +
 .../apache/spark/sql/connect/ml/MLHandler.scala    |   567 +
 .../org/apache/spark/sql/connect/ml/MLUtils.scala  |   742 +
 .../apache/spark/sql/connect/ml/Serializer.scala   |   217 +
 .../connect/pipelines/DataflowGraphRegistry.scala  |    77 +
 .../connect/pipelines/PipelineEventSender.scala    |   176 +
 .../sql/connect/pipelines/PipelinesHandler.scala   |   532 +
 .../sql/connect/planner/InvalidInputErrors.scala   |   264 +
 .../planner/LiteralExpressionProtoConverter.scala  |    39 +
 .../spark/sql/connect/planner/MockObserver.scala   |    43 +
 .../sql/connect/planner/SaveModeConverter.scala    |    49 +
 .../sql/connect/planner/SparkConnectPlanner.scala  |  4456 +++
 ...SparkConnectStreamingQueryListenerHandler.scala |   128 +
 .../planner/StreamingForeachBatchHelper.scala      |   284 +
 .../planner/StreamingQueryListenerHelper.scala     |   112 +
 .../connect/planner/TableSaveMethodConverter.scala |    40 +
 .../plugin/SparkConnectPluginRegistry.scala        |   227 +
 .../sql/connect/service/ExecuteEventsManager.scala |   560 +
 .../spark/sql/connect/service/ExecuteHolder.scala  |   435 +
 .../LocalPropertiesCleanupInterceptor.scala        |    51 +
 .../sql/connect/service/LoggingInterceptor.scala   |    78 +
 .../PreSharedKeyAuthenticationInterceptor.scala    |    47 +
 .../service/RequestDecompressionInterceptor.scala  |   331 +
 .../sql/connect/service/SessionEventsManager.scala |   128 +
 .../spark/sql/connect/service/SessionHolder.scala  |   659 +
 .../service/SparkConnectAddArtifactsHandler.scala  |   318 +
 .../service/SparkConnectAnalyzeHandler.scala       |   258 +
 .../SparkConnectArtifactStatusesHandler.scala      |    71 +
 .../service/SparkConnectCloneSessionHandler.scala  |    62 +
 .../service/SparkConnectConfigHandler.scala        |   204 +
 .../service/SparkConnectExecutePlanHandler.scala   |    62 +
 .../service/SparkConnectExecutionManager.scala     |   377 +
 .../SparkConnectFetchErrorDetailsHandler.scala     |    59 +
 .../service/SparkConnectGetStatusHandler.scala     |   218 +
 .../service/SparkConnectInterceptorRegistry.scala  |   119 +
 .../service/SparkConnectInterruptHandler.scala     |    74 +
 .../service/SparkConnectListenerBusListener.scala  |   177 +
 .../SparkConnectReattachExecuteHandler.scala       |    67 +
 .../SparkConnectReleaseExecuteHandler.scala        |    70 +
 .../SparkConnectReleaseSessionHandler.scala        |    46 +
 .../sql/connect/service/SparkConnectServer.scala   |    58 +
 .../sql/connect/service/SparkConnectService.scala  |   580 +
 .../service/SparkConnectSessionManager.scala       |   403 +
 .../service/SparkConnectStreamingQueryCache.scala  |   338 +
 .../ui/SparkConnectServerAppStatusStore.scala      |   131 +
 .../ui/SparkConnectServerHistoryServerPlugin.scala |    41 +
 .../connect/ui/SparkConnectServerListener.scala    |   452 +
 .../sql/connect/ui/SparkConnectServerPage.scala    |   528 +
 .../connect/ui/SparkConnectServerSessionPage.scala |   132 +
 .../sql/connect/ui/SparkConnectServerTab.scala     |    59 +
 .../org/apache/spark/sql/connect/ui/ToolTips.scala |    39 +
 .../spark/sql/connect/utils/ErrorUtils.scala       |   450 +
 .../spark/sql/connect/utils/MetricGenerator.scala  |    78 +
 .../utils/PipelineAnalysisContextUtils.scala       |    85 +
 .../sql/connect/utils/PlanCompressionUtils.scala   |   121 +
 .../services/org.apache.spark.ml.Estimator         |    20 +
 .../services/org.apache.spark.ml.Transformer       |    21 +
 .../org.apache.spark.ml.evaluation.Evaluator       |    21 +
 .../server/src/test/resources/log4j2.properties    |    44 +
 sql/connect/server/src/test/resources/udf          |   Bin 0 -> 973 bytes
 .../sql/connect/IllegalStateErrorsSuite.scala      |   206 +
 .../sql/connect/ProtoToParsedPlanTestSuite.scala   |   281 +
 .../spark/sql/connect/PythonTestDepsChecker.scala  |    44 +
 .../apache/spark/sql/connect/ResourceHelper.scala  |    32 +
 .../spark/sql/connect/SparkConnectServerTest.scala |   393 +
 .../sql/connect/SparkConnectServerTestSuite.scala  |   207 +
 .../spark/sql/connect/SparkConnectTestUtils.scala  |    62 +
 .../org/apache/spark/sql/connect/dsl/package.scala |  1207 +
 .../ConnectProgressExecutionListenerSuite.scala    |   177 +
 .../execution/ReattachableExecuteSuite.scala       |   506 +
 .../sql/connect/messages/AbbreviateSuite.scala     |   333 +
 .../messages/ConnectProtoMessagesSuite.scala       |    89 +
 .../spark/sql/connect/ml/MLBackendSuite.scala      |   199 +
 .../org/apache/spark/sql/connect/ml/MLHelper.scala |   423 +
 .../org/apache/spark/sql/connect/ml/MLSuite.scala  |   490 +
 .../sql/connect/pipelines/EndToEndAPISuite.scala   |   181 +
 .../pipelines/PipelineEventSenderSuite.scala       |   233 +
 .../pipelines/PipelineEventStreamSuite.scala       |   176 +
 .../pipelines/PipelineRefreshFunctionalSuite.scala |   361 +
 .../connect/pipelines/PythonPipelineSuite.scala    |  1180 +
 .../SparkDeclarativePipelinesServerSuite.scala     |   947 +
 .../SparkDeclarativePipelinesServerTest.scala      |   166 +
 .../connect/pipelines/TestPipelineDefinition.scala |   152 +
 .../connect/planner/InvalidInputErrorsSuite.scala  |   187 +
 .../LiteralExpressionProtoConverterSuite.scala     |   233 +
 .../connect/planner/SparkConnectPlannerSuite.scala |  1091 +
 .../planner/SparkConnectPlannerTestUtils.scala     |    62 +
 .../connect/planner/SparkConnectProtoSuite.scala   |  1158 +
 .../connect/planner/SparkConnectServiceSuite.scala |  1291 +
 .../SparkConnectWithSessionExtensionSuite.scala    |    87 +
 .../planner/StreamingForeachBatchHelperSuite.scala |    81 +
 .../plugin/SparkConnectPluginRegistrySuite.scala   |   324 +
 .../connect/service/AddArtifactsHandlerSuite.scala |   581 +
 .../service/ArtifactStatusesHandlerSuite.scala     |    99 +
 .../service/ExecuteEventsManagerSuite.scala        |   491 +
 .../service/FetchErrorDetailsHandlerSuite.scala    |   372 +
 .../connect/service/GetStatusHandlerE2ESuite.scala |   216 +
 .../connect/service/GetStatusHandlerSuite.scala    |   354 +
 .../connect/service/InterceptorRegistrySuite.scala |   197 +
 .../RequestDecompressionInterceptorSuite.scala     |   346 +
 .../service/SessionEventsManagerSuite.scala        |   102 +
 .../connect/service/SparkConnectAuthSuite.scala    |    46 +
 .../service/SparkConnectCloneSessionSuite.scala    |   278 +
 .../SparkConnectExecutionManagerSuite.scala        |   118 +
 .../SparkConnectListenerBusListenerSuite.scala     |   241 +
 .../service/SparkConnectServiceE2ESuite.scala      |   530 +
 .../SparkConnectServiceInternalServerSuite.scala   |   321 +
 .../service/SparkConnectSessionHolderSuite.scala   |   574 +
 .../service/SparkConnectSessionManagerSuite.scala  |   222 +
 .../SparkConnectStreamingQueryCacheSuite.scala     |   159 +
 .../ui/SparkConnectServerListenerSuite.scala       |   237 +
 .../connect/ui/SparkConnectServerPageSuite.scala   |   134 +
 sql/connect/shims/README.md                        |     1 +
 sql/connect/shims/pom.xml                          |    48 +
 .../src/main/scala/org/apache/spark/shims.scala    |    49 +
 .../AggregateBenchmark-jdk11-results.txt           |   148 -
 .../AggregateBenchmark-jdk21-results.txt           |   148 +
 .../AggregateBenchmark-jdk25-results.txt           |   148 +
 sql/core/benchmarks/AggregateBenchmark-results.txt |   130 +-
 .../AnsiIntervalSortBenchmark-jdk21-results.txt    |    28 +
 .../AnsiIntervalSortBenchmark-jdk25-results.txt    |    28 +
 .../AnsiIntervalSortBenchmark-results.txt          |    28 +
 .../benchmarks/Base64Benchmark-jdk21-results.txt   |    56 +
 .../benchmarks/Base64Benchmark-jdk25-results.txt   |    56 +
 sql/core/benchmarks/Base64Benchmark-results.txt    |    56 +
 .../BloomFilterBenchmark-jdk11-results.txt         |    24 -
 .../BloomFilterBenchmark-jdk21-results.txt         |   196 +
 .../BloomFilterBenchmark-jdk25-results.txt         |   196 +
 .../benchmarks/BloomFilterBenchmark-results.txt    |   188 +-
 ...iltInDataSourceWriteBenchmark-jdk11-results.txt |    60 -
 ...iltInDataSourceWriteBenchmark-jdk21-results.txt |    70 +
 ...iltInDataSourceWriteBenchmark-jdk25-results.txt |    70 +
 .../BuiltInDataSourceWriteBenchmark-results.txt    |    68 +-
 .../ByteArrayBenchmark-jdk21-results.txt           |    27 +
 .../ByteArrayBenchmark-jdk25-results.txt           |    27 +
 sql/core/benchmarks/ByteArrayBenchmark-results.txt |    27 +
 sql/core/benchmarks/CSVBenchmark-jdk11-results.txt |    67 -
 sql/core/benchmarks/CSVBenchmark-jdk21-results.txt |    77 +
 sql/core/benchmarks/CSVBenchmark-jdk25-results.txt |    77 +
 sql/core/benchmarks/CSVBenchmark-results.txt       |   102 +-
 .../CharVarcharBenchmark-jdk21-results.txt         |   122 +
 .../CharVarcharBenchmark-jdk25-results.txt         |   122 +
 .../benchmarks/CharVarcharBenchmark-results.txt    |   140 +-
 .../CollationBenchmark-jdk21-results.txt           |    88 +
 .../CollationBenchmark-jdk25-results.txt           |    88 +
 sql/core/benchmarks/CollationBenchmark-results.txt |    88 +
 .../CollationNonASCIIBenchmark-jdk21-results.txt   |    88 +
 .../CollationNonASCIIBenchmark-jdk25-results.txt   |    88 +
 .../CollationNonASCIIBenchmark-results.txt         |    88 +
 .../ColumnarBatchBenchmark-jdk11-results.txt       |    59 -
 .../ColumnarBatchBenchmark-jdk21-results.txt       |    59 +
 .../ColumnarBatchBenchmark-jdk25-results.txt       |    59 +
 .../benchmarks/ColumnarBatchBenchmark-results.txt  |    54 +-
 .../CompressionSchemeBenchmark-jdk11-results.txt   |   137 -
 .../CompressionSchemeBenchmark-jdk21-results.txt   |   137 +
 .../CompressionSchemeBenchmark-jdk25-results.txt   |   137 +
 .../CompressionSchemeBenchmark-results.txt         |   168 +-
 ...ConstantColumnVectorBenchmark-jdk21-results.txt |   280 +
 ...ConstantColumnVectorBenchmark-jdk25-results.txt |   280 +
 .../ConstantColumnVectorBenchmark-results.txt      |   280 +
 .../DataSourceReadBenchmark-jdk11-results.txt      |   252 -
 .../DataSourceReadBenchmark-jdk21-results.txt      |   438 +
 .../DataSourceReadBenchmark-jdk25-results.txt      |   438 +
 .../benchmarks/DataSourceReadBenchmark-results.txt |   560 +-
 .../benchmarks/DatasetBenchmark-jdk11-results.txt  |    46 -
 .../benchmarks/DatasetBenchmark-jdk21-results.txt  |    46 +
 .../benchmarks/DatasetBenchmark-jdk25-results.txt  |    46 +
 sql/core/benchmarks/DatasetBenchmark-results.txt   |    52 +-
 .../benchmarks/DateTimeBenchmark-jdk11-results.txt |   461 -
 .../benchmarks/DateTimeBenchmark-jdk21-results.txt |   461 +
 .../benchmarks/DateTimeBenchmark-jdk25-results.txt |   461 +
 sql/core/benchmarks/DateTimeBenchmark-results.txt  |   482 +-
 .../DateTimeRebaseBenchmark-jdk11-results.txt      |   154 -
 .../DateTimeRebaseBenchmark-jdk21-results.txt      |   154 +
 .../DateTimeRebaseBenchmark-jdk25-results.txt      |   154 +
 .../benchmarks/DateTimeRebaseBenchmark-results.txt |   230 +-
 .../benchmarks/EncodeBenchmark-jdk21-results.txt   |     8 +
 .../benchmarks/EncodeBenchmark-jdk25-results.txt   |     8 +
 sql/core/benchmarks/EncodeBenchmark-results.txt    |     8 +
 .../benchmarks/ExpandBenchmark-jdk21-results.txt   |    59 +
 .../benchmarks/ExpandBenchmark-jdk25-results.txt   |    59 +
 sql/core/benchmarks/ExpandBenchmark-results.txt    |    59 +
 ...ndOnlyUnsafeRowArrayBenchmark-jdk11-results.txt |    45 -
 ...ndOnlyUnsafeRowArrayBenchmark-jdk21-results.txt |    45 +
 ...ndOnlyUnsafeRowArrayBenchmark-jdk25-results.txt |    45 +
 ...alAppendOnlyUnsafeRowArrayBenchmark-results.txt |    40 +-
 .../benchmarks/ExtractBenchmark-jdk11-results.txt  |   104 -
 .../benchmarks/ExtractBenchmark-jdk21-results.txt  |   104 +
 .../benchmarks/ExtractBenchmark-jdk25-results.txt  |   104 +
 sql/core/benchmarks/ExtractBenchmark-results.txt   |   172 +-
 .../FilterPushdownBenchmark-jdk11-results.txt      |   670 -
 .../FilterPushdownBenchmark-jdk21-results.txt      |   734 +
 .../FilterPushdownBenchmark-jdk25-results.txt      |   734 +
 .../benchmarks/FilterPushdownBenchmark-results.txt |   904 +-
 .../GenerateExecBenchmark-jdk11-results.txt        |    12 -
 .../GenerateExecBenchmark-jdk21-results.txt        |    12 +
 .../GenerateExecBenchmark-jdk25-results.txt        |    12 +
 .../benchmarks/GenerateExecBenchmark-results.txt   |     8 +-
 ...ashedRelationMetricsBenchmark-jdk11-results.txt |    11 -
 ...ashedRelationMetricsBenchmark-jdk21-results.txt |    11 +
 ...ashedRelationMetricsBenchmark-jdk25-results.txt |    11 +
 .../HashedRelationMetricsBenchmark-results.txt     |     6 +-
 .../InExpressionBenchmark-jdk11-results.txt        |   740 -
 .../InExpressionBenchmark-jdk21-results.txt        |   740 +
 .../InExpressionBenchmark-jdk25-results.txt        |   740 +
 .../benchmarks/InExpressionBenchmark-results.txt   |   840 +-
 .../InMemoryColumnarBenchmark-jdk21-results.txt    |    12 +
 .../InMemoryColumnarBenchmark-jdk25-results.txt    |    12 +
 .../InMemoryColumnarBenchmark-results.txt          |    12 +
 ...ithDynamicPartitionsBenchmark-jdk11-results.txt |     8 -
 ...ithDynamicPartitionsBenchmark-jdk21-results.txt |     8 +
 ...ithDynamicPartitionsBenchmark-jdk25-results.txt |     8 +
 ...TableWithDynamicPartitionsBenchmark-results.txt |    10 +-
 .../benchmarks/IntervalBenchmark-jdk11-results.txt |    40 -
 .../benchmarks/IntervalBenchmark-jdk21-results.txt |    40 +
 .../benchmarks/IntervalBenchmark-jdk25-results.txt |    40 +
 sql/core/benchmarks/IntervalBenchmark-results.txt  |    68 +-
 .../benchmarks/JoinBenchmark-jdk11-results.txt     |    82 -
 .../benchmarks/JoinBenchmark-jdk21-results.txt     |    82 +
 .../benchmarks/JoinBenchmark-jdk25-results.txt     |    82 +
 sql/core/benchmarks/JoinBenchmark-results.txt      |    88 +-
 .../benchmarks/JsonBenchmark-jdk11-results.txt     |   120 -
 .../benchmarks/JsonBenchmark-jdk21-results.txt     |   130 +
 .../benchmarks/JsonBenchmark-jdk25-results.txt     |   130 +
 sql/core/benchmarks/JsonBenchmark-results.txt      |   154 +-
 .../benchmarks/LargeRowBenchmark-jdk21-results.txt |    26 +
 .../benchmarks/LargeRowBenchmark-jdk25-results.txt |    26 +
 sql/core/benchmarks/LargeRowBenchmark-results.txt  |    26 +
 .../MakeDateTimeBenchmark-jdk11-results.txt        |    22 -
 .../MakeDateTimeBenchmark-jdk21-results.txt        |    22 +
 .../MakeDateTimeBenchmark-jdk25-results.txt        |    22 +
 .../benchmarks/MakeDateTimeBenchmark-results.txt   |    32 +-
 .../MapLookupBenchmark-jdk21-results.txt           |   144 +
 .../MapLookupBenchmark-jdk25-results.txt           |   144 +
 sql/core/benchmarks/MapLookupBenchmark-results.txt |   144 +
 .../MergeRowsExecBenchmark-jdk21-results.txt       |    54 +
 .../MergeRowsExecBenchmark-jdk25-results.txt       |    54 +
 .../benchmarks/MergeRowsExecBenchmark-results.txt  |    54 +
 .../MetadataStructBenchmark-jdk21-results.txt      |    46 +
 .../MetadataStructBenchmark-jdk25-results.txt      |    46 +
 .../benchmarks/MetadataStructBenchmark-results.txt |    46 +
 .../MetricsAggregationBenchmark-jdk11-results.txt  |    12 -
 .../MetricsAggregationBenchmark-jdk21-results.txt  |    12 +
 .../MetricsAggregationBenchmark-jdk25-results.txt  |    12 +
 .../MetricsAggregationBenchmark-results.txt        |    16 +-
 .../benchmarks/MiscBenchmark-jdk11-results.txt     |   127 -
 .../benchmarks/MiscBenchmark-jdk21-results.txt     |   127 +
 .../benchmarks/MiscBenchmark-jdk25-results.txt     |   127 +
 sql/core/benchmarks/MiscBenchmark-results.txt      |   106 +-
 ...cNestedSchemaPruningBenchmark-jdk11-results.txt |    53 -
 ...cNestedSchemaPruningBenchmark-jdk21-results.txt |    53 +
 ...cNestedSchemaPruningBenchmark-jdk25-results.txt |    53 +
 .../OrcNestedSchemaPruningBenchmark-results.txt    |    60 +-
 ...2NestedSchemaPruningBenchmark-jdk11-results.txt |    53 -
 ...2NestedSchemaPruningBenchmark-jdk21-results.txt |    53 +
 ...2NestedSchemaPruningBenchmark-jdk25-results.txt |    53 +
 .../OrcV2NestedSchemaPruningBenchmark-results.txt  |    60 +-
 ...tedPredicatePushDownBenchmark-jdk11-results.txt |    21 -
 ...tedPredicatePushDownBenchmark-jdk21-results.txt |    21 +
 ...tedPredicatePushDownBenchmark-jdk25-results.txt |    21 +
 ...uetNestedPredicatePushDownBenchmark-results.txt |    24 +-
 ...tNestedSchemaPruningBenchmark-jdk11-results.txt |    53 -
 ...tNestedSchemaPruningBenchmark-jdk21-results.txt |    53 +
 ...tNestedSchemaPruningBenchmark-jdk25-results.txt |    53 +
 ...ParquetNestedSchemaPruningBenchmark-results.txt |    60 +-
 ...ParquetVectorUpdaterBenchmark-jdk21-results.txt |    84 +
 ...ParquetVectorUpdaterBenchmark-jdk25-results.txt |    84 +
 .../ParquetVectorUpdaterBenchmark-results.txt      |    84 +
 .../PrimitiveArrayBenchmark-jdk11-results.txt      |    12 -
 .../PrimitiveArrayBenchmark-jdk21-results.txt      |    12 +
 .../PrimitiveArrayBenchmark-jdk25-results.txt      |    12 +
 .../benchmarks/PrimitiveArrayBenchmark-results.txt |     8 +-
 .../benchmarks/RangeBenchmark-jdk11-results.txt    |    15 -
 .../benchmarks/RangeBenchmark-jdk21-results.txt    |    15 +
 .../benchmarks/RangeBenchmark-jdk25-results.txt    |    15 +
 sql/core/benchmarks/RangeBenchmark-results.txt     |    14 +-
 .../RecursiveCTEBenchmark-jdk21-results.txt        |    95 +
 .../RecursiveCTEBenchmark-jdk25-results.txt        |    95 +
 .../benchmarks/RecursiveCTEBenchmark-results.txt   |    95 +
 .../SetOperationsBenchmark-jdk21-results.txt       |    13 +
 .../SetOperationsBenchmark-jdk25-results.txt       |    13 +
 .../benchmarks/SetOperationsBenchmark-results.txt  |    13 +
 .../benchmarks/SortBenchmark-jdk11-results.txt     |    16 -
 .../benchmarks/SortBenchmark-jdk21-results.txt     |    16 +
 .../benchmarks/SortBenchmark-jdk25-results.txt     |    16 +
 sql/core/benchmarks/SortBenchmark-results.txt      |    16 +-
 ...StoreBasicOperationsBenchmark-jdk21-results.txt |   144 +
 ...StoreBasicOperationsBenchmark-jdk25-results.txt |   144 +
 .../StateStoreBasicOperationsBenchmark-results.txt |   144 +
 .../StringFunctionsBenchmark-jdk21-results.txt     |    11 +
 .../StringFunctionsBenchmark-jdk25-results.txt     |    11 +
 .../StringFunctionsBenchmark-results.txt           |    11 +
 .../SubExprEliminationBenchmark-jdk11-results.txt  |    25 -
 .../SubExprEliminationBenchmark-jdk21-results.txt  |    25 +
 .../SubExprEliminationBenchmark-jdk25-results.txt  |    25 +
 .../SubExprEliminationBenchmark-results.txt        |    24 +-
 .../TPCDSQueryBenchmark-jdk11-results.txt          |   810 -
 .../TPCDSQueryBenchmark-jdk21-results.txt          |   810 +
 .../TPCDSQueryBenchmark-jdk25-results.txt          |   810 +
 .../benchmarks/TPCDSQueryBenchmark-results.txt     |  1080 +-
 ...akeOrderedAndProjectBenchmark-jdk21-results.txt |    12 +
 ...akeOrderedAndProjectBenchmark-jdk25-results.txt |    12 +
 .../TakeOrderedAndProjectBenchmark-results.txt     |    12 +
 .../benchmarks/TopKBenchmark-jdk21-results.txt     |    22 +
 .../benchmarks/TopKBenchmark-jdk25-results.txt     |    22 +
 sql/core/benchmarks/TopKBenchmark-results.txt      |    22 +
 sql/core/benchmarks/UDFBenchmark-jdk11-results.txt |    59 -
 sql/core/benchmarks/UDFBenchmark-jdk21-results.txt |    59 +
 sql/core/benchmarks/UDFBenchmark-jdk25-results.txt |    59 +
 sql/core/benchmarks/UDFBenchmark-results.txt       |    56 +-
 .../benchmarks/UnionBenchmark-jdk21-results.txt    |   480 +
 .../benchmarks/UnionBenchmark-jdk25-results.txt    |   480 +
 sql/core/benchmarks/UnionBenchmark-results.txt     |   480 +
 .../UnsafeArrayDataBenchmark-jdk11-results.txt     |    33 -
 .../UnsafeArrayDataBenchmark-jdk21-results.txt     |    33 +
 .../UnsafeArrayDataBenchmark-jdk25-results.txt     |    33 +
 .../UnsafeArrayDataBenchmark-results.txt           |    32 +-
 .../UpdateFieldsBenchmark-jdk21-results.txt        |    26 +
 .../UpdateFieldsBenchmark-jdk25-results.txt        |    26 +
 .../benchmarks/UpdateFieldsBenchmark-results.txt   |    20 +-
 .../V2FunctionBenchmark-jdk11-results.txt          |    44 -
 .../V2FunctionBenchmark-jdk21-results.txt          |    44 +
 .../V2FunctionBenchmark-jdk25-results.txt          |    44 +
 .../benchmarks/V2FunctionBenchmark-results.txt     |    64 +-
 ...ectorizedDeltaReaderBenchmark-jdk21-results.txt |    93 +
 ...ectorizedDeltaReaderBenchmark-jdk25-results.txt |    93 +
 .../VectorizedDeltaReaderBenchmark-results.txt     |    93 +
 ...zedPlainValuesReaderBenchmark-jdk21-results.txt |    86 +
 ...zedPlainValuesReaderBenchmark-jdk25-results.txt |    86 +
 ...ectorizedPlainValuesReaderBenchmark-results.txt |    86 +
 ...rizedRleValuesReaderBenchmark-jdk21-results.txt |   154 +
 ...rizedRleValuesReaderBenchmark-jdk25-results.txt |   154 +
 .../VectorizedRleValuesReaderBenchmark-results.txt |   154 +
 .../WideSchemaBenchmark-jdk11-results.txt          |   145 -
 .../WideSchemaBenchmark-jdk21-results.txt          |   158 +
 .../WideSchemaBenchmark-jdk25-results.txt          |   158 +
 .../benchmarks/WideSchemaBenchmark-results.txt     |   191 +-
 .../WideTableBenchmark-jdk11-results.txt           |    17 -
 .../WideTableBenchmark-jdk21-results.txt           |    17 +
 .../WideTableBenchmark-jdk25-results.txt           |    17 +
 sql/core/benchmarks/WideTableBenchmark-results.txt |    18 +-
 sql/core/pom.xml                                   |   254 +-
 sql/core/src/main/buf.gen.yaml                     |    24 +
 sql/core/src/main/buf.work.yaml                    |    19 +
 .../parquet/filter2/predicate/SparkFilterApi.java  |    26 +-
 .../function/FlatMapGroupsWithStateFunction.java   |    39 -
 .../java/function/MapGroupsWithStateFunction.java  |    38 -
 .../spark/sql/avro/AvroCompressionCodec.java       |    71 +
 .../spark/sql/avro/SparkAvroKeyOutputFormat.java   |    99 +
 .../spark/sql/execution/BufferedRowIterator.java   |     3 +-
 .../sql/execution/RecordBinaryComparator.java      |     5 +-
 .../sql/execution/UnsafeExternalRowSorter.java     |     3 +
 .../execution/UnsafeFixedWidthAggregationMap.java  |    17 +-
 .../sql/execution/UnsafeKVExternalSorter.java      |    10 +-
 .../sql/execution/columnar/ColumnDictionary.java   |    21 +-
 .../datasources/HadoopLineRecordReader.java        |   298 +
 .../SchemaColumnConvertNotSupportedException.java  |     3 +-
 .../datasources/orc/OrcArrayColumnVector.java      |    38 +-
 .../datasources/orc/OrcAtomicColumnVector.java     |    29 +-
 .../datasources/orc/OrcColumnStatistics.java       |    80 +
 .../execution/datasources/orc/OrcColumnVector.java |     2 +-
 .../datasources/orc/OrcColumnVectorUtils.java      |    15 +-
 .../datasources/orc/OrcColumnarBatchReader.java    |    43 +-
 .../datasources/orc/OrcCompressionCodec.java       |    61 +
 .../execution/datasources/orc/OrcFooterReader.java |    67 +
 .../datasources/orc/OrcMapColumnVector.java        |    38 +-
 .../datasources/orc/OrcStructColumnVector.java     |    25 +-
 .../datasources/parquet/OpenedParquetFooter.java   |    34 +
 .../datasources/parquet/ParquetColumnVector.java   |   432 +
 .../parquet/ParquetCompressionCodec.java           |    76 +
 .../datasources/parquet/ParquetDictionary.java     |     2 +-
 .../datasources/parquet/ParquetFooterReader.java   |    88 +-
 .../datasources/parquet/ParquetLogRedirector.java  |    72 -
 .../datasources/parquet/ParquetReadState.java      |    69 +-
 .../parquet/ParquetVectorUpdaterFactory.java       |   861 +-
 .../parquet/SpecificParquetRecordReaderBase.java   |   183 +-
 .../parquet/VectorizedColumnReader.java            |   215 +-
 .../parquet/VectorizedDeltaBinaryPackedReader.java |   331 +
 .../parquet/VectorizedDeltaByteArrayReader.java    |   188 +
 .../VectorizedDeltaLengthByteArrayReader.java      |   124 +
 .../parquet/VectorizedParquetRecordReader.java     |   337 +-
 .../parquet/VectorizedPlainValuesReader.java       |   250 +-
 .../datasources/parquet/VectorizedReaderBase.java  |   167 +
 .../parquet/VectorizedRleValuesReader.java         |   739 +-
 .../parquet/VectorizedValuesReader.java            |    43 +-
 .../datasources/parquet/WKBConverterStrategy.java  |    51 +
 .../execution/vectorized/ColumnVectorUtils.java    |   105 +-
 .../execution/vectorized/ConstantColumnVector.java |   328 +
 .../execution/vectorized/MutableColumnarRow.java   |    59 +-
 .../execution/vectorized/OffHeapColumnVector.java  |   139 +-
 .../execution/vectorized/OnHeapColumnVector.java   |   148 +-
 .../execution/vectorized/WritableColumnVector.java |   323 +-
 .../spark/sql/expressions/javalang/typed.java      |    74 -
 .../org/apache/spark/sql/streaming/Trigger.java    |   153 -
 .../sql/execution/streaming/StateMessage.proto     |   296 +
 ....apache.spark.deploy.history.EventFilterBuilder |    17 +
 ...rg.apache.spark.sql.jdbc.JdbcConnectionProvider |    17 +
 .../services/org.apache.spark.sql.jdbc.JdbcDialect |    29 +
 ...org.apache.spark.sql.sources.DataSourceRegister |    21 +
 .../org.apache.spark.status.AppHistoryServerPlugin |    17 +
 .../org.apache.spark.status.protobuf.ProtobufSerDe |    21 +
 .../sql/execution/ui/static/allexecutionspage.js   |   242 +
 .../sql/execution/ui/static/spark-sql-viz.css      |   109 +-
 .../spark/sql/execution/ui/static/spark-sql-viz.js |   604 +-
 .../sql/execution/streaming/StreamProgress.scala   |    54 -
 .../sql/execution/streaming/StreamProgress.scala   |    53 -
 .../main/scala/org/apache/spark/sql/Column.scala   |  1484 -
 .../apache/spark/sql/DataFrameNaFunctions.scala    |   542 -
 .../org/apache/spark/sql/DataFrameReader.scala     |   742 -
 .../apache/spark/sql/DataFrameStatFunctions.scala  |   629 -
 .../org/apache/spark/sql/DataFrameWriter.scala     |   876 -
 .../org/apache/spark/sql/DataFrameWriterV2.scala   |   342 -
 .../apache/spark/sql/DataSourceRegistration.scala  |    79 +
 .../main/scala/org/apache/spark/sql/Dataset.scala  |  3795 ---
 .../scala/org/apache/spark/sql/DatasetHolder.scala |    45 -
 .../org/apache/spark/sql/ExperimentalMethods.scala |     1 +
 .../spark/sql/ExtendedExplainGenerator.scala       |    32 +
 .../scala/org/apache/spark/sql/ForeachWriter.scala |   136 -
 .../apache/spark/sql/KeyValueGroupedDataset.scala  |   796 -
 .../scala/org/apache/spark/sql/Observation.scala   |   177 -
 .../spark/sql/RelationalGroupedDataset.scala       |   650 -
 .../scala/org/apache/spark/sql/RuntimeConfig.scala |   160 -
 .../scala/org/apache/spark/sql/SQLContext.scala    |  1096 -
 .../scala/org/apache/spark/sql/SQLImplicits.scala  |   266 -
 .../scala/org/apache/spark/sql/SparkSession.scala  |  1226 -
 .../apache/spark/sql/SparkSessionExtensions.scala  |   116 +-
 .../spark/sql/SparkSessionExtensionsProvider.scala |     5 +-
 .../main/scala/org/apache/spark/sql/TableArg.scala |    82 +
 .../org/apache/spark/sql/UDFRegistration.scala     |  1094 -
 .../org/apache/spark/sql/UDTFRegistration.scala    |    51 +
 .../spark/sql/api/python/PythonSQLUtils.scala      |   216 +-
 .../org/apache/spark/sql/api/r/SQLUtils.scala      |    50 +-
 .../spark/sql/artifact/ArtifactManager.scala       |   615 +
 .../apache/spark/sql/avro/AvroDeserializer.scala   |   523 +
 .../org/apache/spark/sql/avro/AvroFileFormat.scala |   194 +
 .../org/apache/spark/sql/avro/AvroOptions.scala    |   247 +
 .../apache/spark/sql/avro/AvroOutputWriter.scala   |    95 +
 .../spark/sql/avro/AvroOutputWriterFactory.scala   |    56 +
 .../org/apache/spark/sql/avro/AvroSerializer.scala |   413 +
 .../org/apache/spark/sql/avro/AvroUtils.scala      |   404 +
 .../org/apache/spark/sql/avro/CustomDecimal.scala  |    78 +
 .../apache/spark/sql/avro/SchemaConverters.scala   |   506 +
 .../org/apache/spark/sql/catalog/Catalog.scala     |   592 -
 .../org/apache/spark/sql/catalog/interface.scala   |   147 -
 .../analysis/EvalSubqueriesForTimeTravel.scala     |    48 +
 .../sql/catalyst/analysis/InvokeProcedures.scala   |    85 +
 .../catalyst/analysis/ReplaceCharWithVarchar.scala |    73 +
 .../sql/catalyst/analysis/ResolveDataSource.scala  |   212 +
 .../analysis/ResolveExecuteImmediate.scala         |   217 +
 .../sql/catalyst/analysis/ResolveMetricView.scala  |   344 +
 .../catalyst/analysis/ResolveSessionCatalog.scala  |   985 +-
 .../analysis/ResolveSetCatalogCommand.scala        |    49 +
 .../sql/catalyst/analysis/ResolveTranspose.scala   |   211 +
 .../catalyst/plans/logical/SQLFunctionNode.scala   |    45 +
 .../plans/logical/UnresolvedDataSource.scala       |    39 +
 .../org/apache/spark/sql/classic/Catalog.scala     |  1121 +
 .../spark/sql/classic/DataFrameNaFunctions.scala   |   320 +
 .../apache/spark/sql/classic/DataFrameReader.scala |   360 +
 .../spark/sql/classic/DataFrameStatFunctions.scala |   123 +
 .../apache/spark/sql/classic/DataFrameWriter.scala |   625 +
 .../spark/sql/classic/DataFrameWriterV2.scala      |   272 +
 .../spark/sql/classic/DataStreamReader.scala       |   189 +
 .../spark/sql/classic/DataStreamWriter.scala       |   460 +
 .../org/apache/spark/sql/classic/Dataset.scala     |  2385 ++
 .../spark/sql/classic/KeyValueGroupedDataset.scala |   582 +
 .../apache/spark/sql/classic/MergeIntoWriter.scala |   129 +
 .../spark/sql/classic/ObservationManager.scala     |    75 +
 .../sql/classic/RelationalGroupedDataset.scala     |   722 +
 .../apache/spark/sql/classic/RuntimeConfig.scala   |   124 +
 .../org/apache/spark/sql/classic/SQLContext.scala  |   443 +
 .../apache/spark/sql/classic/SQLImplicits.scala    |    42 +
 .../apache/spark/sql/classic/SparkSession.scala    |  1301 +
 .../sql/classic/StreamingCheckpointManager.scala   |    57 +
 .../apache/spark/sql/classic/StreamingQuery.scala  |    25 +
 .../spark/sql/classic/StreamingQueryManager.scala  |   394 +
 .../spark/sql/classic/TableValuedFunction.scala    |   104 +
 .../apache/spark/sql/classic/TypedAggUtils.scala   |    65 +
 .../apache/spark/sql/classic/UDFRegistration.scala |   198 +
 .../sql/classic/UserDefinedFunctionUtils.scala     |    44 +
 .../spark/sql/classic/columnNodeSupport.scala      |   356 +
 .../org/apache/spark/sql/classic/conversions.scala |   111 +
 .../org/apache/spark/sql/classic/package.scala     |    46 +
 .../spark/sql/columnar/CachedBatchSerializer.scala |    40 +-
 .../sql/execution/AggregatingAccumulator.scala     |    30 +-
 .../sql/execution/AliasAwareOutputExpression.scala |   165 +-
 .../execution/BaseScriptTransformationExec.scala   |    26 +-
 .../apache/spark/sql/execution/CacheManager.scala  |   454 +-
 .../spark/sql/execution/CollectMetricsExec.scala   |    20 +-
 .../org/apache/spark/sql/execution/Columnar.scala  |   192 +-
 .../sql/execution/ColumnarEvaluatorFactory.scala   |   108 +
 .../spark/sql/execution/CommandResultExec.scala    |    21 +-
 .../spark/sql/execution/DataSourceScanExec.scala   |   625 +-
 .../spark/sql/execution/EmptyRelationExec.scala    |    88 +
 .../apache/spark/sql/execution/ExistingRDD.scala   |   260 +-
 .../apache/spark/sql/execution/ExpandExec.scala    |    28 +-
 .../apache/spark/sql/execution/ExplainUtils.scala  |   125 +-
 .../ExternalAppendOnlyUnsafeRowArray.scala         |    80 +-
 .../apache/spark/sql/execution/FileRelation.scala  |     5 +-
 .../sql/execution/FilterEvaluatorFactory.scala     |    48 +
 .../apache/spark/sql/execution/GenerateExec.scala  |    51 +-
 .../spark/sql/execution/GroupedIterator.scala      |     4 +-
 .../apache/spark/sql/execution/HiveResult.scala    |    83 +-
 .../execution/InsertSortForLimitAndOffset.scala    |   102 +
 .../spark/sql/execution/LocalTableScanExec.scala   |    31 +-
 .../spark/sql/execution/MultiResultExec.scala      |    36 +
 .../sql/execution/OptimizeMetadataOnlyQuery.scala  |    15 +-
 .../spark/sql/execution/PartitionedFileUtil.scala  |    22 +-
 .../sql/execution/ProjectEvaluatorFactory.scala    |    41 +
 .../spark/sql/execution/QueryExecution.scala       |   629 +-
 .../sql/execution/RemoveRedundantProjects.scala    |     8 +-
 .../RemoveRedundantWindowGroupLimits.scala         |    37 +
 .../sql/execution/ReplaceHashWithSortAgg.scala     |   120 +
 .../apache/spark/sql/execution/SQLExecution.scala  |   338 +-
 .../spark/sql/execution/SafeForKWayMerge.scala     |    27 +
 .../spark/sql/execution/ScanFileListing.scala      |    86 +
 .../spark/sql/execution/ShuffledRowRDD.scala       |    10 +-
 .../org/apache/spark/sql/execution/SortExec.scala  |    18 +-
 .../spark/sql/execution/SortPrefixUtils.scala      |     6 +-
 .../spark/sql/execution/SparkOptimizer.scala       |    78 +-
 .../org/apache/spark/sql/execution/SparkPlan.scala |   164 +-
 .../apache/spark/sql/execution/SparkPlanInfo.scala |    30 +-
 .../apache/spark/sql/execution/SparkPlanner.scala  |    21 +-
 .../spark/sql/execution/SparkSqlParser.scala       |  1236 +-
 .../spark/sql/execution/SparkStrategies.scala      |   618 +-
 .../sql/execution/StreamSourceAwareSparkPlan.scala |    32 +
 .../execution/SubqueryAdaptiveBroadcastExec.scala  |     2 +-
 .../sql/execution/SubqueryBroadcastExec.scala      |    60 +-
 .../apache/spark/sql/execution/UnionLoopExec.scala |   302 +
 .../spark/sql/execution/UnsafeRowSerializer.scala  |    24 +-
 .../WholeStageCodegenEvaluatorFactory.scala        |    58 +
 .../sql/execution/WholeStageCodegenExec.scala      |   135 +-
 .../sql/execution/adaptive/AQEOptimizer.scala      |    39 +-
 .../adaptive/AQEPropagateEmptyRelation.scala       |    48 +-
 .../execution/adaptive/AQEShuffleReadExec.scala    |    52 +-
 .../sql/execution/adaptive/AQETestHelper.scala     |    79 +
 .../spark/sql/execution/adaptive/AQEUtils.scala    |    13 +-
 .../execution/adaptive/AdaptiveRulesHolder.scala   |    40 +
 .../execution/adaptive/AdaptiveSparkPlanExec.scala |   509 +-
 .../adaptive/AdaptiveSparkPlanHelper.scala         |    27 +-
 .../adaptive/AdjustShuffleExchangePosition.scala   |    43 +
 .../adaptive/CoalesceShufflePartitions.scala       |   177 +-
 .../execution/adaptive/DynamicJoinSelection.scala  |    77 +-
 .../adaptive/InsertAdaptiveSparkPlan.scala         |   101 +-
 .../adaptive/InvalidAQEPlanException.scala         |    30 +
 .../sql/execution/adaptive/LogicalQueryStage.scala |    68 +-
 .../adaptive/LogicalQueryStageStrategy.scala       |    54 +-
 .../OptimizeSkewInRebalancePartitions.scala        |    44 +-
 .../execution/adaptive/OptimizeSkewedJoin.scala    |   170 +-
 .../PlanAdaptiveDynamicPruningFilters.scala        |    29 +-
 .../adaptive/PlanAdaptiveSubqueries.scala          |    26 +-
 .../sql/execution/adaptive/QueryStageExec.scala    |   217 +-
 .../execution/adaptive/ShufflePartitionsUtil.scala |   107 +-
 .../sql/execution/adaptive/ValidateSparkPlan.scala |    68 +
 .../sql/execution/adaptive/simpleCosting.scala     |    23 +-
 .../spark/sql/execution/aggregate/AggUtils.scala   |   126 +-
 .../aggregate/AggregateCodegenSupport.scala        |   359 +
 .../execution/aggregate/AggregationIterator.scala  |     9 +-
 .../execution/aggregate/BaseAggregateExec.scala    |    36 +-
 .../execution/aggregate/HashAggregateExec.scala    |   354 +-
 .../sql/execution/aggregate/HashMapGenerator.scala |    12 +-
 .../execution/aggregate/MergingSessionsExec.scala  |    15 +-
 .../aggregate/MergingSessionsIterator.scala        |    18 +-
 .../aggregate/ObjectAggregationIterator.scala      |    33 +-
 .../execution/aggregate/ObjectAggregationMap.scala |    16 +-
 .../aggregate/ObjectHashAggregateExec.scala        |    13 +-
 .../aggregate/RowBasedHashMapGenerator.scala       |     2 +-
 .../execution/aggregate/SortAggregateExec.scala    |    34 +-
 .../aggregate/SortBasedAggregationIterator.scala   |    21 +-
 .../aggregate/TungstenAggregationIterator.scala    |    25 +-
 .../execution/aggregate/UpdatingSessionsExec.scala |    22 +-
 .../aggregate/UpdatingSessionsIterator.scala       |    16 +-
 .../sql/execution/aggregate/typedaggregators.scala |   101 -
 .../spark/sql/execution/aggregate/udaf.scala       |    46 +-
 .../analysis/DetectAmbiguousSelfJoin.scala         |    21 +-
 .../sql/execution/arrow/ArrowConverters.scala      |   596 +-
 .../sql/execution/arrow/ArrowFileReadWrite.scala   |   102 +
 .../spark/sql/execution/arrow/ArrowWriter.scala    |   437 -
 .../sql/execution/basicPhysicalOperators.scala     |   529 +-
 .../bucketing/CoalesceBucketsInJoin.scala          |    18 +-
 .../bucketing/DisableUnnecessaryBucketedScan.scala |     8 +-
 .../sql/execution/columnar/ColumnAccessor.scala    |    38 +-
 .../sql/execution/columnar/ColumnBuilder.scala     |    27 +-
 .../spark/sql/execution/columnar/ColumnStats.scala |    51 +-
 .../spark/sql/execution/columnar/ColumnType.scala  |   172 +-
 .../columnar/GenerateColumnAccessor.scala          |    18 +-
 .../sql/execution/columnar/InMemoryRelation.scala  |   226 +-
 .../execution/columnar/InMemoryTableScanExec.scala |   142 +-
 .../compression/CompressibleColumnAccessor.scala   |     4 +-
 .../compression/CompressibleColumnBuilder.scala    |     4 +-
 .../columnar/compression/CompressionScheme.scala   |    12 +-
 .../columnar/compression/compressionSchemes.scala  |   109 +-
 .../execution/command/AnalyzeColumnCommand.scala   |    26 +-
 .../command/AnalyzePartitionCommand.scala          |    56 +-
 .../execution/command/AnalyzeTableCommand.scala    |     2 +-
 .../execution/command/AnalyzeTablesCommand.scala   |     7 +-
 .../spark/sql/execution/command/CommandCheck.scala |     2 +-
 .../spark/sql/execution/command/CommandUtils.scala |   200 +-
 .../command/CreateSQLFunctionCommand.scala         |   535 +
 .../command/CreateUserDefinedFunctionCommand.scala |   175 +
 .../sql/execution/command/DataWritingCommand.scala |     7 +-
 .../command/DescribeFunctionCommandUtils.scala     |    89 +
 .../command/DescribeProcedureCommand.scala         |   114 +
 .../command/DescribeRelationJsonCommand.scala      |   354 +
 .../command/InsertIntoDataSourceDirCommand.scala   |    13 +-
 .../sql/execution/command/SetCatalogCommand.scala  |    62 +
 .../spark/sql/execution/command/SetCommand.scala   |    80 +-
 .../execution/command/SetNamespaceCollation.scala  |    44 +
 .../execution/command/SetNamespaceCommand.scala    |    33 +
 .../sql/execution/command/SetPathCommand.scala     |   124 +
 .../execution/command/ShowCatalogsCommand.scala    |    36 +
 .../execution/command/ShowCollationsCommand.scala  |    57 +
 .../command/ShowCurrentNamespaceCommand.scala      |    37 +
 .../execution/command/ShowNamespacesCommand.scala  |    77 +
 .../execution/command/ShowProceduresCommand.scala  |    64 +
 .../command/UnsetNamespacePropertiesCommand.scala  |    55 +
 .../apache/spark/sql/execution/command/cache.scala |     3 +-
 .../spark/sql/execution/command/commands.scala     |    62 +-
 .../execution/command/createDataSourceTables.scala |    58 +-
 .../apache/spark/sql/execution/command/ddl.scala   |   386 +-
 .../spark/sql/execution/command/functions.scala    |   181 +-
 .../sql/execution/command/metricViewCommands.scala |   203 +
 .../spark/sql/execution/command/tables.scala       |   271 +-
 .../sql/execution/command/v2/CloseCursorExec.scala |    68 +
 .../execution/command/v2/CreateVariableExec.scala  |   109 +
 .../execution/command/v2/CursorCommandUtils.scala  |    46 +
 .../execution/command/v2/DeclareCursorExec.scala   |    77 +
 .../execution/command/v2/DropVariableExec.scala    |    55 +
 .../sql/execution/command/v2/FetchCursorExec.scala |   265 +
 .../sql/execution/command/v2/OpenCursorExec.scala  |    99 +
 .../command/v2/ParameterBindingUtils.scala         |   117 +
 .../command/v2/ParameterizedQueryExecutor.scala    |    74 +
 .../sql/execution/command/v2/SetVariableExec.scala |   102 +
 .../execution/command/v2/V2CommandStrategy.scala   |    57 +
 .../command/v2/VariableAssignmentUtils.scala       |    87 +
 .../apache/spark/sql/execution/command/views.scala |   679 +-
 .../datasources/AggregatePushDownUtils.scala       |   207 +
 .../datasources/ApplyCharTypePadding.scala         |    65 +
 .../datasources/BasicWriteStatsTracker.scala       |    20 +-
 .../sql/execution/datasources/BucketingUtils.scala |     7 +
 .../sql/execution/datasources/CodecStreams.scala   |    13 +-
 .../sql/execution/datasources/DataSource.scala     |   249 +-
 .../execution/datasources/DataSourceManager.scala  |   134 +
 .../datasources/DataSourceMetricsMixin.scala       |    24 +
 .../execution/datasources/DataSourceResolver.scala |    64 +
 .../execution/datasources/DataSourceStrategy.scala |   458 +-
 .../execution/datasources/DataSourceUtils.scala    |   234 +-
 .../datasources/ExternalEngineDatasourceRDD.scala  |    26 +
 .../datasources/FallBackFileSourceV2.scala         |    10 +-
 .../sql/execution/datasources/FileFormat.scala     |   195 +-
 .../datasources/FileFormatDataWriter.scala         |    88 +-
 .../execution/datasources/FileFormatWriter.scala   |   374 +-
 .../sql/execution/datasources/FileIndex.scala      |    80 +-
 .../execution/datasources/FileIndexOptions.scala   |    32 +
 .../sql/execution/datasources/FilePartition.scala  |    74 +-
 .../sql/execution/datasources/FileResolver.scala   |    59 +
 .../sql/execution/datasources/FileScanRDD.scala    |   226 +-
 .../execution/datasources/FileSourceStrategy.scala |   186 +-
 .../execution/datasources/FileStatusCache.scala    |    22 +-
 .../datasources/HadoopFileLinesReader.scala        |    24 +-
 .../datasources/HadoopFileWholeTextReader.scala    |    16 +-
 .../execution/datasources/HadoopFsRelation.scala   |    10 +-
 .../execution/datasources/InMemoryFileIndex.scala  |    54 +-
 .../datasources/InsertIntoDataSourceCommand.scala  |    12 +-
 .../InsertIntoHadoopFsRelationCommand.scala        |    31 +-
 .../execution/datasources/LogicalRelation.scala    |    66 +-
 .../datasources/LogicalRelationResolver.scala      |    38 +
 .../datasources/PartitioningAwareFileIndex.scala   |    58 +-
 .../execution/datasources/PartitioningUtils.scala  |   182 +-
 .../datasources/PruneFileSourcePartitions.scala    |    83 +-
 .../datasources/PushVariantIntoScan.scala          |   351 +
 .../datasources/RecordReaderIterator.scala         |     7 +
 .../SQLHadoopMapReduceCommitProtocol.scala         |     7 +-
 .../datasources/SaveIntoDataSourceCommand.scala    |    50 +-
 .../execution/datasources/SchemaMergeUtils.scala   |    20 +-
 .../sql/execution/datasources/SchemaPruning.scala  |   127 +-
 .../spark/sql/execution/datasources/V1Writes.scala |   232 +
 .../sql/execution/datasources/WriteFiles.scala     |   118 +
 .../datasources/binaryfile/BinaryFileFormat.scala  |    84 +-
 .../execution/datasources/csv/CSVDataSource.scala  |    88 +-
 .../execution/datasources/csv/CSVFileFormat.scala  |    92 +-
 .../sql/execution/datasources/csv/CSVUtils.scala   |    45 +-
 .../spark/sql/execution/datasources/ddl.scala      |    42 +-
 .../datasources/jdbc/DriverRegistry.scala          |     5 +-
 .../datasources/jdbc/JDBCDatabaseMetadata.scala    |   102 +
 .../execution/datasources/jdbc/JDBCOptions.scala   |    73 +-
 .../sql/execution/datasources/jdbc/JDBCRDD.scala   |   357 +-
 .../execution/datasources/jdbc/JDBCRelation.scala  |    70 +-
 .../datasources/jdbc/JdbcRelationProvider.scala    |    25 +-
 .../sql/execution/datasources/jdbc/JdbcUtils.scala |   860 +-
 .../jdbc/connection/BasicConnectionProvider.scala  |    13 +-
 .../jdbc/connection/ConnectionProvider.scala       |    39 +-
 .../jdbc/connection/SecureConnectionProvider.scala |     2 +-
 .../datasources/json/JsonDataSource.scala          |    32 +-
 .../datasources/json/JsonFileFormat.scala          |    55 +-
 .../datasources/json/JsonOutputWriter.scala        |     6 +-
 .../sql/execution/datasources/json/JsonUtils.scala |    17 +-
 .../datasources/noop/NoopDataSource.scala          |    12 +-
 .../datasources/orc/OrcDeserializer.scala          |    94 +-
 .../execution/datasources/orc/OrcFileFormat.scala  |   119 +-
 .../sql/execution/datasources/orc/OrcFilters.scala |    14 +-
 .../execution/datasources/orc/OrcFiltersBase.scala |     9 +-
 .../sql/execution/datasources/orc/OrcOptions.scala |    28 +-
 .../datasources/orc/OrcOutputWriter.scala          |     6 +-
 .../execution/datasources/orc/OrcSerializer.scala  |    71 +-
 .../sql/execution/datasources/orc/OrcUtils.scala   |   340 +-
 .../parquet/InferVariantShreddingSchema.scala      |   386 +
 .../datasources/parquet/ParquetColumn.scala        |    58 +
 .../datasources/parquet/ParquetFileFormat.scala    |   566 +-
 .../datasources/parquet/ParquetFilters.scala       |   341 +-
 .../datasources/parquet/ParquetOptions.scala       |    50 +-
 .../ParquetOutputWriterWithVariantShredding.scala  |   120 +
 .../datasources/parquet/ParquetReadSupport.scala   |   419 +-
 .../parquet/ParquetRecordMaterializer.scala        |    18 +-
 .../datasources/parquet/ParquetRowConverter.scala  |   433 +-
 .../datasources/parquet/ParquetRowIndexUtil.scala  |   132 +
 .../parquet/ParquetSchemaConverter.scala           |   538 +-
 .../datasources/parquet/ParquetUtils.scala         |   446 +-
 .../datasources/parquet/ParquetWriteSupport.scala  |   221 +-
 .../datasources/parquet/SparkShreddingUtils.scala  |   893 +
 .../sql/execution/datasources/pathFilters.scala    |    16 +-
 .../spark/sql/execution/datasources/rules.scala    |   503 +-
 .../datasources/text/TextFileFormat.scala          |    29 +-
 .../execution/datasources/text/TextOptions.scala   |    18 +-
 .../datasources/v2/AddPartitionExec.scala          |     2 +-
 .../execution/datasources/v2/AlterTableExec.scala  |    40 +-
 .../execution/datasources/v2/AlterV2ViewExec.scala |   217 +
 .../execution/datasources/v2/BatchScanExec.scala   |   107 +-
 .../execution/datasources/v2/CacheTableExec.scala  |    81 +-
 .../datasources/v2/ContinuousScanExec.scala        |    17 +-
 .../execution/datasources/v2/CreateIndexExec.scala |    66 +
 .../datasources/v2/CreateNamespaceExec.scala       |    27 +-
 .../execution/datasources/v2/CreateTableExec.scala |    24 +-
 .../datasources/v2/CreateTableLikeExec.scala       |   102 +
 .../datasources/v2/CreateV2MetricViewExec.scala    |    69 +
 .../datasources/v2/CreateV2ViewExec.scala          |   208 +
 .../datasources/v2/DataSourcePartitioning.scala    |    56 -
 .../execution/datasources/v2/DataSourceRDD.scala   |   265 +-
 .../datasources/v2/DataSourceV2ScanExecBase.scala  |    93 +-
 .../datasources/v2/DataSourceV2Strategy.scala      |   855 +-
 .../datasources/v2/DataSourceV2Utils.scala         |   133 +-
 .../datasources/v2/DeleteFromTableExec.scala       |    30 +-
 .../datasources/v2/DescribeColumnExec.scala        |    62 +-
 .../datasources/v2/DescribeNamespaceExec.scala     |    18 +-
 .../datasources/v2/DescribeTableExec.scala         |   217 +-
 .../v2/DescribeTablePartitionExec.scala            |    91 +
 .../v2/DistributionAndOrderingUtils.scala          |    84 +-
 .../execution/datasources/v2/DropIndexExec.scala   |    44 +
 .../datasources/v2/DropNamespaceExec.scala         |    21 +-
 .../execution/datasources/v2/DropTableExec.scala   |    32 +-
 .../execution/datasources/v2/DropViewExec.scala    |    65 +
 .../datasources/v2/ExplainOnlySparkPlan.scala      |    38 +
 .../execution/datasources/v2/FileBatchWrite.scala  |    18 +-
 .../datasources/v2/FileDataSourceV2.scala          |    41 +-
 .../datasources/v2/FilePartitionReader.scala       |    67 +-
 .../v2/FilePartitionReaderFactory.scala            |    19 +-
 .../sql/execution/datasources/v2/FileScan.scala    |    57 +-
 .../execution/datasources/v2/FileScanBuilder.scala |    51 +-
 .../sql/execution/datasources/v2/FileTable.scala   |    44 +-
 .../sql/execution/datasources/v2/FileWrite.scala   |    25 +-
 .../datasources/v2/FileWriterFactory.scala         |    17 +-
 .../GroupBasedRowLevelOperationScanPlanning.scala  |   138 +
 .../datasources/v2/GroupPartitionsExec.scala       |   385 +
 .../execution/datasources/v2/MergeRowsExec.scala   |   606 +
 .../datasources/v2/MicroBatchScanExec.scala        |    22 +-
 .../v2/OptimizeMetadataOnlyDeleteFromTable.scala   |   125 +
 .../v2/PartitionReaderWithPartitionValues.scala    |     3 +-
 .../execution/datasources/v2/PushDownUtils.scala   |   416 +-
 .../datasources/v2/PushedDownOperators.scala       |    38 +
 .../datasources/v2/RealTimeStreamScanExec.scala    |   172 +
 .../execution/datasources/v2/RenameTableExec.scala |     5 +-
 .../datasources/v2/ReplaceTableExec.scala          |    73 +-
 .../execution/datasources/v2/ShowColumnsExec.scala |    34 +
 .../datasources/v2/ShowCreateTableExec.scala       |   100 +-
 .../datasources/v2/ShowCurrentNamespaceExec.scala  |    35 -
 .../datasources/v2/ShowFunctionsExec.scala         |    70 +
 .../datasources/v2/ShowNamespacesExec.scala        |    54 -
 .../datasources/v2/ShowPartitionsExec.scala        |     3 +-
 .../datasources/v2/ShowTablePropertiesExec.scala   |    15 +-
 .../execution/datasources/v2/ShowTablesExec.scala  |    26 +-
 .../datasources/v2/ShowTablesExtendedExec.scala    |   189 +
 .../execution/datasources/v2/ShowViewsExec.scala   |    51 +
 .../v2/SupportsCustomDriverMetrics.scala           |    80 +
 .../datasources/v2/TableCapabilityCheck.scala      |    14 +-
 .../execution/datasources/v2/TableSampleInfo.scala |    24 +
 .../datasources/v2/TextBasedFileScan.scala         |     2 +-
 .../datasources/v2/TruncateTableExec.scala         |    16 +-
 .../datasources/v2/V1FallbackWriters.scala         |    26 +-
 .../execution/datasources/v2/V2ColumnUtils.scala   |    27 +
 .../execution/datasources/v2/V2CommandExec.scala   |     8 +-
 .../v2/V2ScanPartitioningAndOrdering.scala         |    77 +
 .../datasources/v2/V2ScanRelationPushDown.scala    |  1091 +-
 .../datasources/v2/V2SessionCatalog.scala          |   366 +-
 .../datasources/v2/V2TableRefreshUtil.scala        |   156 +
 .../datasources/v2/V2ViewInspectionExecs.scala     |   272 +
 .../sql/execution/datasources/v2/V2Writes.scala    |   154 +-
 .../datasources/v2/WriteToDataSourceV2Exec.scala   |   757 +-
 .../v2/csv/CSVPartitionReaderFactory.scala         |    21 +-
 .../sql/execution/datasources/v2/csv/CSVScan.scala |    36 +-
 .../datasources/v2/csv/CSVScanBuilder.scala        |    20 +-
 .../execution/datasources/v2/csv/CSVTable.scala    |    14 +-
 .../execution/datasources/v2/csv/CSVWrite.scala    |     5 +-
 .../execution/datasources/v2/jdbc/JDBCScan.scala   |    46 +-
 .../datasources/v2/jdbc/JDBCScanBuilder.scala      |   279 +-
 .../execution/datasources/v2/jdbc/JDBCTable.scala  |    82 +-
 .../datasources/v2/jdbc/JDBCTableCatalog.scala     |   273 +-
 .../v2/jdbc/JDBCV1RelationFromV2Scan.scala         |    57 +
 .../datasources/v2/jdbc/JDBCWriteBuilder.scala     |     4 +-
 .../v2/json/JsonPartitionReaderFactory.scala       |    10 +-
 .../execution/datasources/v2/json/JsonScan.scala   |    26 +-
 .../datasources/v2/json/JsonScanBuilder.scala      |    22 +-
 .../execution/datasources/v2/json/JsonTable.scala  |    12 +-
 .../v2/orc/OrcPartitionReaderFactory.scala         |   143 +-
 .../sql/execution/datasources/v2/orc/OrcScan.scala |    62 +-
 .../datasources/v2/orc/OrcScanBuilder.scala        |    57 +-
 .../execution/datasources/v2/orc/OrcTable.scala    |    12 +-
 .../execution/datasources/v2/orc/OrcWrite.scala    |     8 +-
 .../v2/parquet/ParquetPartitionReaderFactory.scala |   369 +-
 .../datasources/v2/parquet/ParquetScan.scala       |   164 +-
 .../v2/parquet/ParquetScanBuilder.scala            |   113 +-
 .../datasources/v2/parquet/ParquetTable.scala      |    14 +-
 .../datasources/v2/parquet/ParquetWrite.scala      |    78 +-
 .../v2/python/PythonBatchWriterFactory.scala       |    82 +
 .../datasources/v2/python/PythonCustomMetric.scala |    54 +
 .../datasources/v2/python/PythonDataSourceV2.scala |    86 +
 .../v2/python/PythonMicroBatchStream.scala         |   189 +
 .../v2/python/PythonPartitionReaderFactory.scala   |    68 +
 .../datasources/v2/python/PythonScan.scala         |    99 +
 .../datasources/v2/python/PythonScanBuilder.scala  |    61 +
 .../PythonStreamingPartitionReaderFactory.scala    |    92 +
 .../python/PythonStreamingSinkCommitRunner.scala   |    85 +
 .../v2/python/PythonStreamingWrite.scala           |   103 +
 .../datasources/v2/python/PythonTable.scala        |    46 +
 .../datasources/v2/python/PythonWrite.scala        |    89 +
 .../datasources/v2/python/PythonWriteBuilder.scala |    36 +
 .../v2/python/UserDefinedPythonDataSource.scala    |   708 +
 .../datasources/v2/state/StateDataSource.scala     |   851 +
 .../v2/state/StateDataSourceErrors.scala           |   201 +
 .../v2/state/StatePartitionReader.scala            |   544 +
 .../datasources/v2/state/StateScanBuilder.scala    |   192 +
 .../datasources/v2/state/StateTable.scala          |   107 +
 .../v2/state/StreamStreamJoinStateHelper.scala     |   160 +
 .../StreamStreamJoinStatePartitionReader.scala     |   236 +
 .../v2/state/metadata/StateMetadataSource.scala    |   281 +
 .../datasources/v2/state/utils/SchemaUtil.scala    |   525 +
 .../v2/text/TextPartitionReaderFactory.scala       |    16 +-
 .../execution/datasources/v2/text/TextScan.scala   |    25 +-
 .../datasources/v2/text/TextScanBuilder.scala      |     6 +-
 .../execution/datasources/v2/text/TextTable.scala  |    10 +-
 .../execution/datasources/xml/XSDToSchema.scala    |   270 +
 .../execution/datasources/xml/XmlDataSource.scala  |   276 +
 .../execution/datasources/xml/XmlFileFormat.scala  |   152 +
 .../execution/datasources/xml/XmlInputFormat.scala |   345 +
 .../datasources/xml/XmlOutputWriter.scala          |    52 +
 .../sql/execution/datasources/xml/XmlUtils.scala   |    66 +
 .../execution/debug/DebugEvaluatorFactory.scala    |    58 +
 .../apache/spark/sql/execution/debug/package.scala |    69 +-
 .../CleanupDynamicPruningFilters.scala             |    48 +-
 .../dynamicpruning/PartitionPruning.scala          |   112 +-
 .../dynamicpruning/PlanDynamicPruningFilters.scala |    40 +-
 .../RowLevelOperationRuntimeGroupFiltering.scala   |   156 +
 .../execution/exchange/BroadcastExchangeExec.scala |    93 +-
 .../execution/exchange/EnsureRequirements.scala    |   818 +-
 .../spark/sql/execution/exchange/Exchange.scala    |     2 +-
 .../execution/exchange/ShuffleExchangeExec.scala   |   203 +-
 .../execution/exchange/ValidateRequirements.scala  |    29 +-
 .../spark/sql/execution/joins/BaseJoinExec.scala   |     2 +
 .../execution/joins/BroadcastHashJoinExec.scala    |   125 +-
 .../joins/BroadcastNestedLoopJoinExec.scala        |    97 +-
 .../sql/execution/joins/CartesianProductExec.scala |    14 +-
 .../spark/sql/execution/joins/HashJoin.scala       |    89 +-
 .../spark/sql/execution/joins/HashedRelation.scala |   294 +-
 .../sql/execution/joins/JoinCodegenSupport.scala   |    62 +-
 .../sql/execution/joins/ShuffledHashJoinExec.scala |   413 +-
 .../spark/sql/execution/joins/ShuffledJoin.scala   |    12 +-
 .../joins/SortMergeJoinEvaluatorFactory.scala      |   311 +
 .../sql/execution/joins/SortMergeJoinExec.scala    |   693 +-
 .../org/apache/spark/sql/execution/limit.scala     |   212 +-
 .../spark/sql/execution/metric/CustomMetrics.scala |    19 +-
 .../metric/SQLLastAttemptAccumulator.scala         |   435 +
 .../execution/metric/SQLLastAttemptMetric.scala    |    88 +
 .../spark/sql/execution/metric/SQLMetrics.scala    |   174 +-
 .../metric/SQLShuffleMetricsReporter.scala         |    86 +-
 .../org/apache/spark/sql/execution/objects.scala   |    55 +-
 .../execution/python/AggregateInPandasExec.scala   |   199 -
 .../python/ArrowAggregatePythonExec.scala          |   259 +
 .../sql/execution/python/ArrowEvalPythonExec.scala |   172 +-
 .../execution/python/ArrowEvalPythonUDTFExec.scala |   107 +
 .../sql/execution/python/ArrowPythonRunner.scala   |   219 +-
 .../execution/python/ArrowPythonUDTFRunner.scala   |   102 +
 .../python/ArrowWindowPythonEvaluatorFactory.scala |   394 +
 .../execution/python/ArrowWindowPythonExec.scala   |   146 +
 .../python/AttachDistributedSequenceExec.scala     |    92 +-
 .../sql/execution/python/BatchEvalPythonExec.scala |   134 +-
 .../execution/python/BatchEvalPythonUDTFExec.scala |   162 +
 .../python/CoGroupedArrowPythonRunner.scala        |   215 +-
 .../ColumnarArrowEvalPythonEvaluatorFactory.scala  |   363 +
 .../python/ColumnarArrowPythonInput.scala          |   146 +
 .../python/ColumnarArrowPythonRunner.scala         |    60 +
 .../python/EvalPythonEvaluatorFactory.scala        |   123 +
 .../sql/execution/python/EvalPythonExec.scala      |    92 +-
 .../sql/execution/python/EvalPythonUDTFExec.scala  |   142 +
 .../sql/execution/python/EvaluatePython.scala      |   179 +-
 .../sql/execution/python/ExtractPythonUDFs.scala   |   155 +-
 .../python/FlatMapCoGroupsInArrowExec.scala        |    58 +
 .../python/FlatMapCoGroupsInBatchExec.scala        |   104 +
 .../python/FlatMapCoGroupsInPandasExec.scala       |    60 +-
 .../python/FlatMapGroupsInArrowExec.scala          |    62 +
 .../python/FlatMapGroupsInBatchExec.scala          |   106 +
 .../python/FlatMapGroupsInPandasExec.scala         |    54 +-
 .../spark/sql/execution/python/HybridQueue.scala   |   178 +
 .../sql/execution/python/MapInArrowExec.scala      |    41 +
 .../python/MapInBatchEvaluatorFactory.scala        |   104 +
 .../sql/execution/python/MapInBatchExec.scala      |    96 +
 .../sql/execution/python/MapInPandasExec.scala     |    74 +-
 .../sql/execution/python/PandasGroupUtils.scala    |     9 +-
 .../sql/execution/python/PythonArrowInput.scala    |   317 +
 .../sql/execution/python/PythonArrowOutput.scala   |   258 +-
 .../sql/execution/python/PythonForeachWriter.scala |   158 -
 .../sql/execution/python/PythonPlannerRunner.scala |   272 +
 .../sql/execution/python/PythonSQLMetrics.scala    |    57 +
 .../sql/execution/python/PythonUDFRunner.scala     |   150 +-
 .../execution/python/PythonWorkerLogsExec.scala    |    63 +
 .../spark/sql/execution/python/RowQueue.scala      |   144 +-
 .../python/UserDefinedPythonFunction.scala         |   296 +-
 .../sql/execution/python/WindowInPandasExec.scala  |   408 -
 .../ApplyInPandasWithStatePythonRunner.scala       |   277 +
 .../streaming/ApplyInPandasWithStateWriter.scala   |   259 +
 .../streaming/BaseStreamingArrowWriter.scala       |   101 +
 .../FlatMapGroupsInPandasWithStateExec.scala       |   244 +
 .../python/streaming/PythonForeachWriter.scala     |   234 +
 .../streaming/PythonStreamingSourceRunner.scala    |   368 +
 .../TransformWithStateInPySparkDeserializer.scala  |    85 +
 .../TransformWithStateInPySparkExec.scala          |   457 +
 .../TransformWithStateInPySparkPythonRunner.scala  |   418 +
 .../TransformWithStateInPySparkStateServer.scala   |   974 +
 ...arkTransformWithStateInPySparkStateServer.scala |   371 +
 .../spark/sql/execution/r/ArrowRRunner.scala       |    30 +-
 .../spark/sql/execution/stat/FrequentItems.scala   |   216 +-
 .../spark/sql/execution/stat/StatFunctions.scala   |   328 +-
 .../streaming/CheckpointFileManager.scala          |   384 -
 .../spark/sql/execution/streaming/CommitLog.scala  |    88 -
 .../streaming/CompactibleFileStreamLog.scala       |   399 -
 .../streaming/ContinuousRecordEndpoint.scala       |    71 -
 .../streaming/EventTimeWatermarkExec.scala         |   131 -
 .../execution/streaming/FileStreamOptions.scala    |   136 -
 .../sql/execution/streaming/FileStreamSink.scala   |   186 -
 .../execution/streaming/FileStreamSinkLog.scala    |   127 -
 .../sql/execution/streaming/FileStreamSource.scala |   576 -
 .../execution/streaming/FileStreamSourceLog.scala  |   152 -
 .../streaming/FileStreamSourceOffset.scala         |    54 -
 .../streaming/FlatMapGroupsWithStateExec.scala     |   470 -
 .../sql/execution/streaming/GroupStateImpl.scala   |   217 -
 .../sql/execution/streaming/HDFSMetadataLog.scala  |   337 -
 .../execution/streaming/IncrementalExecution.scala |   225 -
 .../spark/sql/execution/streaming/LongOffset.scala |    38 -
 .../streaming/ManifestFileCommitProtocol.scala     |    15 +-
 ...MergingSortWithSessionWindowStateIterator.scala |   168 -
 .../sql/execution/streaming/MetadataLog.scala      |    58 -
 .../execution/streaming/MetadataLogFileIndex.scala |    74 -
 .../execution/streaming/MetadataVersionUtil.scala  |    51 -
 .../sql/execution/streaming/MetricsReporter.scala  |    72 -
 .../execution/streaming/MicroBatchExecution.scala  |   640 -
 .../spark/sql/execution/streaming/OffsetSeq.scala  |   170 -
 .../sql/execution/streaming/OffsetSeqLog.scala     |   110 -
 .../sql/execution/streaming/ProgressReporter.scala |   393 -
 .../sql/execution/streaming/RateStreamOffset.scala |    32 -
 .../execution/streaming/ResolveWriteToStream.scala |   128 -
 .../sql/execution/streaming/SerializedOffset.scala |    29 -
 .../spark/sql/execution/streaming/Sink.scala       |     2 +-
 .../sql/execution/streaming/StreamExecution.scala  |   662 -
 .../sql/execution/streaming/StreamMetadata.scala   |    98 -
 .../sql/execution/streaming/StreamingErrors.scala  |    62 +
 .../streaming/StreamingQueryListenerBus.scala      |   148 -
 .../StreamingQueryPlanTraverseHelper.scala         |    61 +
 .../streaming/StreamingQueryWrapper.scala          |   107 -
 .../execution/streaming/StreamingRelation.scala    |   100 -
 .../streaming/StreamingSymmetricHashJoinExec.scala |   635 -
 .../StreamingSymmetricHashJoinHelper.scala         |   263 -
 .../sql/execution/streaming/TriggerExecutor.scala  |    90 -
 .../spark/sql/execution/streaming/Triggers.scala   |   109 -
 .../sql/execution/streaming/WatermarkTracker.scala |   138 -
 .../streaming/checkpointing/AsyncCommitLog.scala   |   138 +
 .../checkpointing/AsyncOffsetSeqLog.scala          |   185 +
 .../checkpointing/CheckpointFileManager.scala      |   394 +
 .../ChecksumCheckpointFileManager.scala            |   564 +
 .../streaming/checkpointing/CommitLog.scala        |   122 +
 .../streaming/checkpointing/HDFSMetadataLog.scala  |   426 +
 .../streaming/checkpointing/MetadataLog.scala      |    58 +
 .../checkpointing/MetadataVersionUtil.scala        |    67 +
 .../streaming/checkpointing/OffsetSeq.scala        |   457 +
 .../streaming/checkpointing/OffsetSeqLog.scala     |   157 +
 .../spark/sql/execution/streaming/console.scala    |    10 +-
 .../continuous/ContinuousDataSourceRDD.scala       |     6 +-
 .../streaming/continuous/ContinuousExecution.scala |   146 +-
 .../continuous/ContinuousQueuedDataReader.scala    |     4 +-
 .../continuous/ContinuousRateStreamSource.scala    |     5 +-
 .../continuous/ContinuousTextSocketSource.scala    |    23 +-
 .../streaming/continuous/ContinuousWriteRDD.scala  |    25 +-
 .../streaming/continuous/EpochCoordinator.scala    |     2 +-
 .../streaming/continuous/EpochTracker.scala        |     2 +-
 .../WriteToContinuousDataSourceExec.scala          |    17 +-
 .../spark/sql/execution/streaming/legacy.scala     |    33 +
 .../spark/sql/execution/streaming/memory.scala     |   299 -
 .../stateful/EventTimeWatermarkExec.scala          |   184 +
 ...MergingSortWithSessionWindowStateIterator.scala |   168 +
 .../StatePartitionKeyExtractorFactory.scala        |    84 +
 .../stateful/StatefulOperatorPartitioning.scala    |    53 +
 .../StreamingAggregationStateManager.scala         |   213 +
 .../StreamingSessionWindowStateManager.scala       |   289 +
 .../FlatMapGroupsWithStateExec.scala               |   604 +
... 6451 lines suppressed ...


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org