You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/07/31 12:56:32 UTC

(spark) branch master updated: [SPARK-52926][SQL] Added SQLMetric for remote schema fetching time duration

This is an automated email from the ASF dual-hosted git repository.

wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 605e5203a16f [SPARK-52926][SQL] Added SQLMetric for remote schema fetching time duration
605e5203a16f is described below

commit 605e5203a16fac830c2a39694b3d1e99b4d57a65
Author: vanja-vujovic-db <va...@databricks.com>
AuthorDate: Thu Jul 31 20:56:15 2025 +0800

    [SPARK-52926][SQL] Added SQLMetric for remote schema fetching time duration
    
    ### What changes were proposed in this pull request?
    
    Added SQLMetric that gets time of remote schema duration.
    
    ### Why are the changes needed?
    
    Needed for better debugging of time lost in schema fetching.
    
    ### Does this PR introduce _any_ user-facing change?
    
    ### How was this patch tested?
    
    Added a test for checking if metric is inside external plan.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No
    
    Closes #51633 from vanja-vujovic-db/testingBranch.
    
    Authored-by: vanja-vujovic-db <va...@databricks.com>
    Signed-off-by: Wenchen Fan <we...@databricks.com>
---
 .../spark/sql/jdbc/OracleIntegrationSuite.scala    |   4 +-
 .../sql/execution/datasources/jdbc/JDBCRDD.scala   |  11 ++-
 .../execution/datasources/jdbc/JDBCRelation.scala  |  21 +++-
 .../datasources/jdbc/JdbcRelationProvider.scala    |  13 ++-
 .../sql/execution/datasources/jdbc/JdbcUtils.scala |   9 +-
 .../datasources/v2/jdbc/JDBCScanBuilder.scala      |   9 +-
 .../execution/datasources/v2/jdbc/JDBCTable.scala  |  14 ++-
 .../datasources/v2/jdbc/JDBCTableCatalog.scala     |  13 ++-
 .../apache/spark/sql/jdbc/JDBCMetricsSuite.scala   | 110 +++++++++++++++++++++
 .../org/apache/spark/sql/jdbc/JDBCSuite.scala      |   6 +-
 10 files changed, 185 insertions(+), 25 deletions(-)

diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
index 33a89f4f20c4..c1cf376db63f 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
@@ -458,7 +458,7 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite
       .load()
 
     df1.logicalPlan match {
-      case LogicalRelationWithTable(JDBCRelation(_, parts, _), _) =>
+      case LogicalRelationWithTable(JDBCRelation(_, parts, _, _), _) =>
         val whereClauses = parts.map(_.asInstanceOf[JDBCPartition].whereClause).toSet
         assert(whereClauses === Set(
           """"D" < '2018-07-11' or "D" is null""",
@@ -481,7 +481,7 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite
       .load()
 
     df2.logicalPlan match {
-      case LogicalRelationWithTable(JDBCRelation(_, parts, _), _) =>
+      case LogicalRelationWithTable(JDBCRelation(_, parts, _, _), _) =>
         val whereClauses = parts.map(_.asInstanceOf[JDBCPartition].whereClause).toSet
         assert(whereClauses === Set(
           """"T" < '2018-07-15 20:50:32.5' or "T" is null""",
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
index 8aea4919ff5a..0ae5f0c14235 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
@@ -130,7 +130,8 @@ object JDBCRDD extends Logging {
       sample: Option[TableSampleInfo] = None,
       limit: Int = 0,
       sortOrders: Array[String] = Array.empty[String],
-      offset: Int = 0): RDD[InternalRow] = {
+      offset: Int = 0,
+      additionalMetrics: Map[String, SQLMetric] = Map()): RDD[InternalRow] = {
     val url = options.url
     val dialect = JdbcDialects.get(url)
     val quotedColumns = if (groupByColumns.isEmpty) {
@@ -155,7 +156,8 @@ object JDBCRDD extends Logging {
       sample,
       limit,
       sortOrders,
-      offset)
+      offset,
+      additionalMetrics)
   }
   // scalastyle:on argcount
 }
@@ -179,7 +181,8 @@ class JDBCRDD(
     sample: Option[TableSampleInfo],
     limit: Int,
     sortOrders: Array[String],
-    offset: Int)
+    offset: Int,
+    additionalMetrics: Map[String, SQLMetric])
   extends RDD[InternalRow](sc, Nil) with DataSourceMetricsMixin with ExternalEngineDatasourceRDD {
 
   /**
@@ -340,6 +343,6 @@ class JDBCRDD(
     Seq(
       "fetchAndTransformToInternalRowsNs" -> fetchAndTransformToInternalRowsMetric,
       "queryExecutionTime" -> queryExecutionTimeMetric
-    )
+    ) ++ additionalMetrics
   }
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
index 2c4158dfe153..e1ef86f54eb1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
@@ -31,6 +31,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, stringToDate
 import org.apache.spark.sql.connector.expressions.filter.Predicate
 import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.execution.datasources.v2.TableSampleInfo
+import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.jdbc.JdbcDialects
 import org.apache.spark.sql.sources._
@@ -48,6 +49,9 @@ private[sql] case class JDBCPartitioningInfo(
     numPartitions: Int)
 
 private[sql] object JDBCRelation extends Logging {
+
+  val schemaFetchName = "Remote JDBC schema fetch time"
+  val schemaFetchKey = "remoteSchemaFetchTime"
   /**
    * Given a partitioning schematic (a column of integral type, a number of
    * partitions, and upper and lower bounds on the column's value), generate
@@ -255,15 +259,20 @@ private[sql] object JDBCRelation extends Logging {
       parts: Array[Partition],
       jdbcOptions: JDBCOptions)(
       sparkSession: SparkSession): JDBCRelation = {
-    val schema = JDBCRelation.getSchema(sparkSession.sessionState.conf.resolver, jdbcOptions)
-    JDBCRelation(schema, parts, jdbcOptions)(sparkSession)
+    val remoteSchemaFetchMetric = JdbcUtils.createSchemaFetchMetric(sparkSession.sparkContext)
+    val schema = SQLMetrics.withTimingNs(remoteSchemaFetchMetric) {
+      JDBCRelation.getSchema(sparkSession.sessionState.conf.resolver, jdbcOptions)
+    }
+    JDBCRelation(schema, parts, jdbcOptions,
+      Map(schemaFetchKey -> remoteSchemaFetchMetric))(sparkSession)
   }
 }
 
 private[sql] case class JDBCRelation(
     override val schema: StructType,
     parts: Array[Partition],
-    jdbcOptions: JDBCOptions)(@transient val sparkSession: SparkSession)
+    jdbcOptions: JDBCOptions,
+    additionalMetrics: Map[String, SQLMetric] = Map())(@transient val sparkSession: SparkSession)
   extends BaseRelation
   with PrunedFilteredScan
   with InsertableRelation {
@@ -296,7 +305,8 @@ private[sql] case class JDBCRelation(
       requiredColumns,
       pushedPredicates,
       parts,
-      jdbcOptions).asInstanceOf[RDD[Row]]
+      jdbcOptions,
+      additionalMetrics = additionalMetrics).asInstanceOf[RDD[Row]]
   }
 
   def buildScan(
@@ -321,7 +331,8 @@ private[sql] case class JDBCRelation(
       tableSample,
       limit,
       sortOrders,
-      offset).asInstanceOf[RDD[Row]]
+      offset,
+      additionalMetrics).asInstanceOf[RDD[Row]]
   }
 
   override def insert(data: DataFrame, overwrite: Boolean): Unit = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
index d9be1a1e3f67..6b9d0520f2eb 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.datasources.jdbc
 import org.apache.spark.sql.{DataFrame, SaveMode, SQLContext}
 import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils._
+import org.apache.spark.sql.execution.metric.SQLMetrics
 import org.apache.spark.sql.jdbc.JdbcDialects
 import org.apache.spark.sql.sources.{BaseRelation, CreatableRelationProvider, DataSourceRegister, RelationProvider}
 
@@ -35,9 +36,17 @@ class JdbcRelationProvider extends CreatableRelationProvider
     val sparkSession = sqlContext.sparkSession
     val resolver = sparkSession.sessionState.conf.resolver
     val timeZoneId = sparkSession.sessionState.conf.sessionLocalTimeZone
-    val schema = JDBCRelation.getSchema(resolver, jdbcOptions)
+    val remoteSchemaFetchMetric = JdbcUtils.createSchemaFetchMetric(sparkSession.sparkContext)
+    val schema = SQLMetrics.withTimingNs(remoteSchemaFetchMetric) {
+      JDBCRelation.getSchema(resolver, jdbcOptions)
+    }
     val parts = JDBCRelation.columnPartition(schema, resolver, timeZoneId, jdbcOptions)
-    JDBCRelation(schema, parts, jdbcOptions)(sparkSession)
+    JDBCRelation(
+      schema,
+      parts,
+      jdbcOptions,
+      Map(JDBCRelation.schemaFetchKey -> remoteSchemaFetchMetric)
+    )(sparkSession)
   }
 
   override def createRelation(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
index f34299b08726..6a97c4e0cf46 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
@@ -29,7 +29,7 @@ import scala.jdk.CollectionConverters._
 import scala.util.{Failure, Success, Try}
 import scala.util.control.NonFatal
 
-import org.apache.spark.{SparkThrowable, SparkUnsupportedOperationException, TaskContext}
+import org.apache.spark.{SparkContext, SparkThrowable, SparkUnsupportedOperationException, TaskContext}
 import org.apache.spark.executor.InputMetrics
 import org.apache.spark.internal.{Logging, MDC}
 import org.apache.spark.internal.LogKeys.{DEFAULT_ISOLATION_LEVEL, ISOLATION_LEVEL}
@@ -402,6 +402,13 @@ object JdbcUtils extends Logging with SQLConfHelper {
     }
   }
 
+  def createSchemaFetchMetric(sparkContext: SparkContext): SQLMetric = {
+    SQLMetrics.createNanoTimingMetric(
+      sparkContext,
+      JDBCRelation.schemaFetchName
+    )
+  }
+
   // A `JDBCValueGetter` is responsible for getting a value from `ResultSet` into a field
   // for `MutableRow`. The last argument `Int` means the index for the value to be set in
   // the row and also used for the value in `ResultSet`.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala
index ebaa8750f797..1880b0cec9d5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala
@@ -28,13 +28,15 @@ import org.apache.spark.sql.connector.read.{ScanBuilder, SupportsPushDownAggrega
 import org.apache.spark.sql.execution.datasources.PartitioningUtils
 import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JDBCPartition, JDBCRDD, JDBCRelation}
 import org.apache.spark.sql.execution.datasources.v2.TableSampleInfo
+import org.apache.spark.sql.execution.metric.SQLMetric
 import org.apache.spark.sql.jdbc.{JdbcDialects, JdbcSQLQueryBuilder, JoinPushdownAliasGenerator}
 import org.apache.spark.sql.types.StructType
 
 case class JDBCScanBuilder(
     session: SparkSession,
     schema: StructType,
-    var jdbcOptions: JDBCOptions)
+    var jdbcOptions: JDBCOptions,
+    additionalMetrics: Map[String, SQLMetric] = Map())
   extends ScanBuilder
     with SupportsPushDownV2Filters
     with SupportsPushDownRequiredColumns
@@ -330,8 +332,9 @@ case class JDBCScanBuilder(
     // "DEPT","NAME",MAX("SALARY"),MIN("BONUS"), instead of getting column names from
     // prunedSchema and quote them (will become "MAX(SALARY)", "MIN(BONUS)" and can't
     // be used in sql string.
-    JDBCScan(JDBCRelation(schema, parts, jdbcOptions)(session), finalSchema, pushedPredicate,
-      pushedAggregateList, pushedGroupBys, tableSample, pushedLimit, sortOrders, pushedOffset)
+    JDBCScan(JDBCRelation(schema, parts, jdbcOptions, additionalMetrics)(session),
+      finalSchema, pushedPredicate, pushedAggregateList, pushedGroupBys,
+      tableSample, pushedLimit, sortOrders, pushedOffset)
   }
 
 }
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala
index 3f8753407232..a242a24baf9c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.connector.expressions.NamedReference
 import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder}
 import org.apache.spark.sql.errors.DataTypeErrorsBase
 import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JdbcOptionsInWrite, JdbcUtils}
+import org.apache.spark.sql.execution.metric.SQLMetric
 import org.apache.spark.sql.jdbc.JdbcDialects
 import org.apache.spark.sql.types.StructType
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -35,13 +36,22 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap
 case class JDBCTable(
     ident: Identifier,
     override val schema: StructType,
-    jdbcOptions: JDBCOptions)
+    jdbcOptions: JDBCOptions,
+    additionalMetrics: Map[String, SQLMetric] = Map())
   extends Table
   with SupportsRead
   with SupportsWrite
   with SupportsIndex
   with DataTypeErrorsBase {
 
+  override def hashCode(): Int = (ident, schema, jdbcOptions).##
+
+  override def equals(obj: Any): Boolean = obj match {
+    case that: JDBCTable =>
+      this.ident == that.ident && this.schema == that.schema && this.jdbcOptions == that.jdbcOptions
+    case _ => false
+  }
+
   override def name(): String = ident.toString
 
   override def capabilities(): util.Set[TableCapability] = {
@@ -51,7 +61,7 @@ case class JDBCTable(
   override def newScanBuilder(options: CaseInsensitiveStringMap): JDBCScanBuilder = {
     val mergedOptions = new JDBCOptions(
       jdbcOptions.parameters.originalMap ++ options.asCaseSensitiveMap().asScala)
-    JDBCScanBuilder(SparkSession.active, schema, mergedOptions)
+    JDBCScanBuilder(SparkSession.active, schema, mergedOptions, additionalMetrics)
   }
 
   override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala
index 63048028c587..152029dcb6be 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala
@@ -24,11 +24,13 @@ import scala.jdk.CollectionConverters._
 
 import org.apache.spark.internal.Logging
 import org.apache.spark.sql.catalyst.analysis.NoSuchFunctionException
+import org.apache.spark.sql.classic.SparkSession
 import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Column, FunctionCatalog, Identifier, NamespaceChange, SupportsNamespaces, Table, TableCatalog, TableChange, TableSummary}
 import org.apache.spark.sql.connector.catalog.functions.UnboundFunction
 import org.apache.spark.sql.connector.expressions.Transform
 import org.apache.spark.sql.errors.{DataTypeErrorsBase, QueryCompilationErrors, QueryExecutionErrors}
-import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JdbcOptionsInWrite, JDBCRDD, JdbcUtils}
+import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JdbcOptionsInWrite, JDBCRDD, JDBCRelation, JdbcUtils}
+import org.apache.spark.sql.execution.metric.SQLMetrics
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.jdbc.{JdbcDialect, JdbcDialects}
 import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -151,8 +153,13 @@ class JDBCTableCatalog extends TableCatalog
       description = s"Failed to load table: $ident",
       isRuntime = false
     ) {
-      val schema = JDBCRDD.resolveTable(optionsWithTableName)
-      JDBCTable(ident, schema, optionsWithTableName)
+      val remoteSchemaFetchMetric = JdbcUtils
+        .createSchemaFetchMetric(SparkSession.active.sparkContext)
+      val schema = SQLMetrics.withTimingNs(remoteSchemaFetchMetric) {
+        JDBCRDD.resolveTable(optionsWithTableName)
+      }
+      JDBCTable(ident, schema, optionsWithTableName,
+        Map(JDBCRelation.schemaFetchKey -> remoteSchemaFetchMetric))
     }
   }
 
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCMetricsSuite.scala
new file mode 100644
index 000000000000..07c73e088e98
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCMetricsSuite.scala
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc
+
+import java.sql.{Connection, DriverManager, Statement}
+import java.util.Properties
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.QueryTest
+import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog
+import org.apache.spark.sql.execution.metric.SQLMetric
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.util.Utils
+
+class JDBCMetricsSuite extends QueryTest with SharedSparkSession {
+
+  val tempDir = Utils.createTempDir()
+  val url = s"jdbc:h2:${tempDir.getCanonicalPath};user=testUser;password=testPass"
+  val properties = new Properties()
+  properties.setProperty("user", "testUser")
+  properties.setProperty("password", "testPass")
+
+  private def withConnection[T](f: Connection => T): T = {
+    val conn = DriverManager.getConnection(url, new Properties())
+    try {
+      f(conn)
+    } finally {
+      conn.close()
+    }
+  }
+
+  override def sparkConf: SparkConf =
+    super.sparkConf
+      .set("spark.sql.catalog.h2", classOf[JDBCTableCatalog].getName)
+      .set("spark.sql.catalog.h2.url", url)
+      .set("spark.sql.catalog.h2.driver", "org.h2.Driver")
+      .set("spark.sql.catalog.h2.pushDownAggregate", "true")
+      .set("spark.sql.catalog.h2.pushDownLimit", "true")
+      .set("spark.sql.catalog.h2.pushDownOffset", "true")
+      .set("spark.sql.catalog.h2.pushDownJoin", "true")
+
+  override def beforeAll(): Unit = {
+    super.beforeAll()
+    Utils.classForName("org.h2.Driver")
+    withConnection { conn =>
+      val statement: Statement = conn.createStatement()
+      statement.addBatch("create schema test")
+      statement.addBatch(
+        "create table test.people (name TEXT(32) NOT NULL, theid INTEGER NOT NULL)")
+      statement.addBatch("insert into test.people values ('dany', 1)")
+      statement.addBatch("insert into test.people values ('mary', 2)")
+      statement.addBatch("insert into test.people values ('alex', 3)")
+      statement.executeBatch()
+    }
+  }
+
+  override def afterAll(): Unit = {
+    Utils.deleteRecursively(tempDir)
+    super.afterAll()
+  }
+
+  private def hasMetricKeyValue(metrics: Map[String, SQLMetric], metricKey: String): Unit = {
+    val optionMetric = metrics.get(metricKey)
+    assert(optionMetric.isDefined)
+    if(optionMetric.isDefined) {
+      val metric = optionMetric.get
+      assert(metric.value >= 0)
+    }
+  }
+
+  test("schema fetch time metric: JDBC v1") {
+    val df = spark.read
+      .format("jdbc")
+      .option("url", url)
+      .option("query", "SELECT * FROM TEST.PEOPLE")
+      .load()
+    hasMetricKeyValue(
+      df.queryExecution.executedPlan.collectLeaves().head.metrics,
+      "remoteSchemaFetchTime")
+  }
+
+  test("schema fetch time metric: JDBC v2") {
+    val df = sql("SELECT * FROM h2.TEST.PEOPLE")
+    hasMetricKeyValue(
+      df.queryExecution.executedPlan.collectLeaves().head.metrics,
+      "remoteSchemaFetchTime")
+  }
+
+  test("schema fetch time metric: DataFrameReader jdbc") {
+    val df = spark.read.jdbc(url, "TEST.PEOPLE", Array[String](), new Properties())
+    hasMetricKeyValue(
+      df.queryExecution.executedPlan.collectLeaves().head.metrics,
+      "remoteSchemaFetchTime")
+  }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index bed8a1a7b363..276062fb1daa 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -1693,7 +1693,7 @@ class JDBCSuite extends QueryTest with SharedSparkSession {
 
       val quotedPrtColName = testH2Dialect.quoteIdentifier(expectedColumnName)
       df.logicalPlan match {
-        case LogicalRelationWithTable(JDBCRelation(_, parts, _), _) =>
+        case LogicalRelationWithTable(JDBCRelation(_, parts, _, _), _) =>
           val whereClauses = parts.map(_.asInstanceOf[JDBCPartition].whereClause).toSet
           assert(whereClauses === Set(
             s"$quotedPrtColName < 2 or $quotedPrtColName is null",
@@ -1835,7 +1835,7 @@ class JDBCSuite extends QueryTest with SharedSparkSession {
       .load()
 
     df1.logicalPlan match {
-      case LogicalRelationWithTable(JDBCRelation(_, parts, _), _) =>
+      case LogicalRelationWithTable(JDBCRelation(_, parts, _, _), _) =>
         val whereClauses = parts.map(_.asInstanceOf[JDBCPartition].whereClause).toSet
         assert(whereClauses === Set(
           """"D" < '2018-07-11' or "D" is null""",
@@ -1855,7 +1855,7 @@ class JDBCSuite extends QueryTest with SharedSparkSession {
       .load()
 
     df2.logicalPlan match {
-      case LogicalRelationWithTable(JDBCRelation(_, parts, _), _) =>
+      case LogicalRelationWithTable(JDBCRelation(_, parts, _, _), _) =>
         val whereClauses = parts.map(_.asInstanceOf[JDBCPartition].whereClause).toSet
         assert(whereClauses === Set(
           """"T" < '2018-07-15 20:50:32.5' or "T" is null""",


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org