You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2025/07/31 12:56:32 UTC
(spark) branch master updated: [SPARK-52926][SQL] Added SQLMetric for remote schema fetching time duration
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 605e5203a16f [SPARK-52926][SQL] Added SQLMetric for remote schema fetching time duration
605e5203a16f is described below
commit 605e5203a16fac830c2a39694b3d1e99b4d57a65
Author: vanja-vujovic-db <va...@databricks.com>
AuthorDate: Thu Jul 31 20:56:15 2025 +0800
[SPARK-52926][SQL] Added SQLMetric for remote schema fetching time duration
### What changes were proposed in this pull request?
Added SQLMetric that gets time of remote schema duration.
### Why are the changes needed?
Needed for better debugging of time lost in schema fetching.
### Does this PR introduce _any_ user-facing change?
### How was this patch tested?
Added a test for checking if metric is inside external plan.
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #51633 from vanja-vujovic-db/testingBranch.
Authored-by: vanja-vujovic-db <va...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../spark/sql/jdbc/OracleIntegrationSuite.scala | 4 +-
.../sql/execution/datasources/jdbc/JDBCRDD.scala | 11 ++-
.../execution/datasources/jdbc/JDBCRelation.scala | 21 +++-
.../datasources/jdbc/JdbcRelationProvider.scala | 13 ++-
.../sql/execution/datasources/jdbc/JdbcUtils.scala | 9 +-
.../datasources/v2/jdbc/JDBCScanBuilder.scala | 9 +-
.../execution/datasources/v2/jdbc/JDBCTable.scala | 14 ++-
.../datasources/v2/jdbc/JDBCTableCatalog.scala | 13 ++-
.../apache/spark/sql/jdbc/JDBCMetricsSuite.scala | 110 +++++++++++++++++++++
.../org/apache/spark/sql/jdbc/JDBCSuite.scala | 6 +-
10 files changed, 185 insertions(+), 25 deletions(-)
diff --git a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
index 33a89f4f20c4..c1cf376db63f 100644
--- a/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
+++ b/connector/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala
@@ -458,7 +458,7 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite
.load()
df1.logicalPlan match {
- case LogicalRelationWithTable(JDBCRelation(_, parts, _), _) =>
+ case LogicalRelationWithTable(JDBCRelation(_, parts, _, _), _) =>
val whereClauses = parts.map(_.asInstanceOf[JDBCPartition].whereClause).toSet
assert(whereClauses === Set(
""""D" < '2018-07-11' or "D" is null""",
@@ -481,7 +481,7 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite
.load()
df2.logicalPlan match {
- case LogicalRelationWithTable(JDBCRelation(_, parts, _), _) =>
+ case LogicalRelationWithTable(JDBCRelation(_, parts, _, _), _) =>
val whereClauses = parts.map(_.asInstanceOf[JDBCPartition].whereClause).toSet
assert(whereClauses === Set(
""""T" < '2018-07-15 20:50:32.5' or "T" is null""",
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
index 8aea4919ff5a..0ae5f0c14235 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
@@ -130,7 +130,8 @@ object JDBCRDD extends Logging {
sample: Option[TableSampleInfo] = None,
limit: Int = 0,
sortOrders: Array[String] = Array.empty[String],
- offset: Int = 0): RDD[InternalRow] = {
+ offset: Int = 0,
+ additionalMetrics: Map[String, SQLMetric] = Map()): RDD[InternalRow] = {
val url = options.url
val dialect = JdbcDialects.get(url)
val quotedColumns = if (groupByColumns.isEmpty) {
@@ -155,7 +156,8 @@ object JDBCRDD extends Logging {
sample,
limit,
sortOrders,
- offset)
+ offset,
+ additionalMetrics)
}
// scalastyle:on argcount
}
@@ -179,7 +181,8 @@ class JDBCRDD(
sample: Option[TableSampleInfo],
limit: Int,
sortOrders: Array[String],
- offset: Int)
+ offset: Int,
+ additionalMetrics: Map[String, SQLMetric])
extends RDD[InternalRow](sc, Nil) with DataSourceMetricsMixin with ExternalEngineDatasourceRDD {
/**
@@ -340,6 +343,6 @@ class JDBCRDD(
Seq(
"fetchAndTransformToInternalRowsNs" -> fetchAndTransformToInternalRowsMetric,
"queryExecutionTime" -> queryExecutionTimeMetric
- )
+ ) ++ additionalMetrics
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
index 2c4158dfe153..e1ef86f54eb1 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRelation.scala
@@ -31,6 +31,7 @@ import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getZoneId, stringToDate
import org.apache.spark.sql.connector.expressions.filter.Predicate
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.execution.datasources.v2.TableSampleInfo
+import org.apache.spark.sql.execution.metric.{SQLMetric, SQLMetrics}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.jdbc.JdbcDialects
import org.apache.spark.sql.sources._
@@ -48,6 +49,9 @@ private[sql] case class JDBCPartitioningInfo(
numPartitions: Int)
private[sql] object JDBCRelation extends Logging {
+
+ val schemaFetchName = "Remote JDBC schema fetch time"
+ val schemaFetchKey = "remoteSchemaFetchTime"
/**
* Given a partitioning schematic (a column of integral type, a number of
* partitions, and upper and lower bounds on the column's value), generate
@@ -255,15 +259,20 @@ private[sql] object JDBCRelation extends Logging {
parts: Array[Partition],
jdbcOptions: JDBCOptions)(
sparkSession: SparkSession): JDBCRelation = {
- val schema = JDBCRelation.getSchema(sparkSession.sessionState.conf.resolver, jdbcOptions)
- JDBCRelation(schema, parts, jdbcOptions)(sparkSession)
+ val remoteSchemaFetchMetric = JdbcUtils.createSchemaFetchMetric(sparkSession.sparkContext)
+ val schema = SQLMetrics.withTimingNs(remoteSchemaFetchMetric) {
+ JDBCRelation.getSchema(sparkSession.sessionState.conf.resolver, jdbcOptions)
+ }
+ JDBCRelation(schema, parts, jdbcOptions,
+ Map(schemaFetchKey -> remoteSchemaFetchMetric))(sparkSession)
}
}
private[sql] case class JDBCRelation(
override val schema: StructType,
parts: Array[Partition],
- jdbcOptions: JDBCOptions)(@transient val sparkSession: SparkSession)
+ jdbcOptions: JDBCOptions,
+ additionalMetrics: Map[String, SQLMetric] = Map())(@transient val sparkSession: SparkSession)
extends BaseRelation
with PrunedFilteredScan
with InsertableRelation {
@@ -296,7 +305,8 @@ private[sql] case class JDBCRelation(
requiredColumns,
pushedPredicates,
parts,
- jdbcOptions).asInstanceOf[RDD[Row]]
+ jdbcOptions,
+ additionalMetrics = additionalMetrics).asInstanceOf[RDD[Row]]
}
def buildScan(
@@ -321,7 +331,8 @@ private[sql] case class JDBCRelation(
tableSample,
limit,
sortOrders,
- offset).asInstanceOf[RDD[Row]]
+ offset,
+ additionalMetrics).asInstanceOf[RDD[Row]]
}
override def insert(data: DataFrame, overwrite: Boolean): Unit = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
index d9be1a1e3f67..6b9d0520f2eb 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcRelationProvider.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.datasources.jdbc
import org.apache.spark.sql.{DataFrame, SaveMode, SQLContext}
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils._
+import org.apache.spark.sql.execution.metric.SQLMetrics
import org.apache.spark.sql.jdbc.JdbcDialects
import org.apache.spark.sql.sources.{BaseRelation, CreatableRelationProvider, DataSourceRegister, RelationProvider}
@@ -35,9 +36,17 @@ class JdbcRelationProvider extends CreatableRelationProvider
val sparkSession = sqlContext.sparkSession
val resolver = sparkSession.sessionState.conf.resolver
val timeZoneId = sparkSession.sessionState.conf.sessionLocalTimeZone
- val schema = JDBCRelation.getSchema(resolver, jdbcOptions)
+ val remoteSchemaFetchMetric = JdbcUtils.createSchemaFetchMetric(sparkSession.sparkContext)
+ val schema = SQLMetrics.withTimingNs(remoteSchemaFetchMetric) {
+ JDBCRelation.getSchema(resolver, jdbcOptions)
+ }
val parts = JDBCRelation.columnPartition(schema, resolver, timeZoneId, jdbcOptions)
- JDBCRelation(schema, parts, jdbcOptions)(sparkSession)
+ JDBCRelation(
+ schema,
+ parts,
+ jdbcOptions,
+ Map(JDBCRelation.schemaFetchKey -> remoteSchemaFetchMetric)
+ )(sparkSession)
}
override def createRelation(
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
index f34299b08726..6a97c4e0cf46 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcUtils.scala
@@ -29,7 +29,7 @@ import scala.jdk.CollectionConverters._
import scala.util.{Failure, Success, Try}
import scala.util.control.NonFatal
-import org.apache.spark.{SparkThrowable, SparkUnsupportedOperationException, TaskContext}
+import org.apache.spark.{SparkContext, SparkThrowable, SparkUnsupportedOperationException, TaskContext}
import org.apache.spark.executor.InputMetrics
import org.apache.spark.internal.{Logging, MDC}
import org.apache.spark.internal.LogKeys.{DEFAULT_ISOLATION_LEVEL, ISOLATION_LEVEL}
@@ -402,6 +402,13 @@ object JdbcUtils extends Logging with SQLConfHelper {
}
}
+ def createSchemaFetchMetric(sparkContext: SparkContext): SQLMetric = {
+ SQLMetrics.createNanoTimingMetric(
+ sparkContext,
+ JDBCRelation.schemaFetchName
+ )
+ }
+
// A `JDBCValueGetter` is responsible for getting a value from `ResultSet` into a field
// for `MutableRow`. The last argument `Int` means the index for the value to be set in
// the row and also used for the value in `ResultSet`.
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala
index ebaa8750f797..1880b0cec9d5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCScanBuilder.scala
@@ -28,13 +28,15 @@ import org.apache.spark.sql.connector.read.{ScanBuilder, SupportsPushDownAggrega
import org.apache.spark.sql.execution.datasources.PartitioningUtils
import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JDBCPartition, JDBCRDD, JDBCRelation}
import org.apache.spark.sql.execution.datasources.v2.TableSampleInfo
+import org.apache.spark.sql.execution.metric.SQLMetric
import org.apache.spark.sql.jdbc.{JdbcDialects, JdbcSQLQueryBuilder, JoinPushdownAliasGenerator}
import org.apache.spark.sql.types.StructType
case class JDBCScanBuilder(
session: SparkSession,
schema: StructType,
- var jdbcOptions: JDBCOptions)
+ var jdbcOptions: JDBCOptions,
+ additionalMetrics: Map[String, SQLMetric] = Map())
extends ScanBuilder
with SupportsPushDownV2Filters
with SupportsPushDownRequiredColumns
@@ -330,8 +332,9 @@ case class JDBCScanBuilder(
// "DEPT","NAME",MAX("SALARY"),MIN("BONUS"), instead of getting column names from
// prunedSchema and quote them (will become "MAX(SALARY)", "MIN(BONUS)" and can't
// be used in sql string.
- JDBCScan(JDBCRelation(schema, parts, jdbcOptions)(session), finalSchema, pushedPredicate,
- pushedAggregateList, pushedGroupBys, tableSample, pushedLimit, sortOrders, pushedOffset)
+ JDBCScan(JDBCRelation(schema, parts, jdbcOptions, additionalMetrics)(session),
+ finalSchema, pushedPredicate, pushedAggregateList, pushedGroupBys,
+ tableSample, pushedLimit, sortOrders, pushedOffset)
}
}
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala
index 3f8753407232..a242a24baf9c 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTable.scala
@@ -28,6 +28,7 @@ import org.apache.spark.sql.connector.expressions.NamedReference
import org.apache.spark.sql.connector.write.{LogicalWriteInfo, WriteBuilder}
import org.apache.spark.sql.errors.DataTypeErrorsBase
import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JdbcOptionsInWrite, JdbcUtils}
+import org.apache.spark.sql.execution.metric.SQLMetric
import org.apache.spark.sql.jdbc.JdbcDialects
import org.apache.spark.sql.types.StructType
import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -35,13 +36,22 @@ import org.apache.spark.sql.util.CaseInsensitiveStringMap
case class JDBCTable(
ident: Identifier,
override val schema: StructType,
- jdbcOptions: JDBCOptions)
+ jdbcOptions: JDBCOptions,
+ additionalMetrics: Map[String, SQLMetric] = Map())
extends Table
with SupportsRead
with SupportsWrite
with SupportsIndex
with DataTypeErrorsBase {
+ override def hashCode(): Int = (ident, schema, jdbcOptions).##
+
+ override def equals(obj: Any): Boolean = obj match {
+ case that: JDBCTable =>
+ this.ident == that.ident && this.schema == that.schema && this.jdbcOptions == that.jdbcOptions
+ case _ => false
+ }
+
override def name(): String = ident.toString
override def capabilities(): util.Set[TableCapability] = {
@@ -51,7 +61,7 @@ case class JDBCTable(
override def newScanBuilder(options: CaseInsensitiveStringMap): JDBCScanBuilder = {
val mergedOptions = new JDBCOptions(
jdbcOptions.parameters.originalMap ++ options.asCaseSensitiveMap().asScala)
- JDBCScanBuilder(SparkSession.active, schema, mergedOptions)
+ JDBCScanBuilder(SparkSession.active, schema, mergedOptions, additionalMetrics)
}
override def newWriteBuilder(info: LogicalWriteInfo): WriteBuilder = {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala
index 63048028c587..152029dcb6be 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/jdbc/JDBCTableCatalog.scala
@@ -24,11 +24,13 @@ import scala.jdk.CollectionConverters._
import org.apache.spark.internal.Logging
import org.apache.spark.sql.catalyst.analysis.NoSuchFunctionException
+import org.apache.spark.sql.classic.SparkSession
import org.apache.spark.sql.connector.catalog.{CatalogV2Util, Column, FunctionCatalog, Identifier, NamespaceChange, SupportsNamespaces, Table, TableCatalog, TableChange, TableSummary}
import org.apache.spark.sql.connector.catalog.functions.UnboundFunction
import org.apache.spark.sql.connector.expressions.Transform
import org.apache.spark.sql.errors.{DataTypeErrorsBase, QueryCompilationErrors, QueryExecutionErrors}
-import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JdbcOptionsInWrite, JDBCRDD, JdbcUtils}
+import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JdbcOptionsInWrite, JDBCRDD, JDBCRelation, JdbcUtils}
+import org.apache.spark.sql.execution.metric.SQLMetrics
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.jdbc.{JdbcDialect, JdbcDialects}
import org.apache.spark.sql.util.CaseInsensitiveStringMap
@@ -151,8 +153,13 @@ class JDBCTableCatalog extends TableCatalog
description = s"Failed to load table: $ident",
isRuntime = false
) {
- val schema = JDBCRDD.resolveTable(optionsWithTableName)
- JDBCTable(ident, schema, optionsWithTableName)
+ val remoteSchemaFetchMetric = JdbcUtils
+ .createSchemaFetchMetric(SparkSession.active.sparkContext)
+ val schema = SQLMetrics.withTimingNs(remoteSchemaFetchMetric) {
+ JDBCRDD.resolveTable(optionsWithTableName)
+ }
+ JDBCTable(ident, schema, optionsWithTableName,
+ Map(JDBCRelation.schemaFetchKey -> remoteSchemaFetchMetric))
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCMetricsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCMetricsSuite.scala
new file mode 100644
index 000000000000..07c73e088e98
--- /dev/null
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCMetricsSuite.scala
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.jdbc
+
+import java.sql.{Connection, DriverManager, Statement}
+import java.util.Properties
+
+import org.apache.spark.SparkConf
+import org.apache.spark.sql.QueryTest
+import org.apache.spark.sql.execution.datasources.v2.jdbc.JDBCTableCatalog
+import org.apache.spark.sql.execution.metric.SQLMetric
+import org.apache.spark.sql.test.SharedSparkSession
+import org.apache.spark.util.Utils
+
+class JDBCMetricsSuite extends QueryTest with SharedSparkSession {
+
+ val tempDir = Utils.createTempDir()
+ val url = s"jdbc:h2:${tempDir.getCanonicalPath};user=testUser;password=testPass"
+ val properties = new Properties()
+ properties.setProperty("user", "testUser")
+ properties.setProperty("password", "testPass")
+
+ private def withConnection[T](f: Connection => T): T = {
+ val conn = DriverManager.getConnection(url, new Properties())
+ try {
+ f(conn)
+ } finally {
+ conn.close()
+ }
+ }
+
+ override def sparkConf: SparkConf =
+ super.sparkConf
+ .set("spark.sql.catalog.h2", classOf[JDBCTableCatalog].getName)
+ .set("spark.sql.catalog.h2.url", url)
+ .set("spark.sql.catalog.h2.driver", "org.h2.Driver")
+ .set("spark.sql.catalog.h2.pushDownAggregate", "true")
+ .set("spark.sql.catalog.h2.pushDownLimit", "true")
+ .set("spark.sql.catalog.h2.pushDownOffset", "true")
+ .set("spark.sql.catalog.h2.pushDownJoin", "true")
+
+ override def beforeAll(): Unit = {
+ super.beforeAll()
+ Utils.classForName("org.h2.Driver")
+ withConnection { conn =>
+ val statement: Statement = conn.createStatement()
+ statement.addBatch("create schema test")
+ statement.addBatch(
+ "create table test.people (name TEXT(32) NOT NULL, theid INTEGER NOT NULL)")
+ statement.addBatch("insert into test.people values ('dany', 1)")
+ statement.addBatch("insert into test.people values ('mary', 2)")
+ statement.addBatch("insert into test.people values ('alex', 3)")
+ statement.executeBatch()
+ }
+ }
+
+ override def afterAll(): Unit = {
+ Utils.deleteRecursively(tempDir)
+ super.afterAll()
+ }
+
+ private def hasMetricKeyValue(metrics: Map[String, SQLMetric], metricKey: String): Unit = {
+ val optionMetric = metrics.get(metricKey)
+ assert(optionMetric.isDefined)
+ if(optionMetric.isDefined) {
+ val metric = optionMetric.get
+ assert(metric.value >= 0)
+ }
+ }
+
+ test("schema fetch time metric: JDBC v1") {
+ val df = spark.read
+ .format("jdbc")
+ .option("url", url)
+ .option("query", "SELECT * FROM TEST.PEOPLE")
+ .load()
+ hasMetricKeyValue(
+ df.queryExecution.executedPlan.collectLeaves().head.metrics,
+ "remoteSchemaFetchTime")
+ }
+
+ test("schema fetch time metric: JDBC v2") {
+ val df = sql("SELECT * FROM h2.TEST.PEOPLE")
+ hasMetricKeyValue(
+ df.queryExecution.executedPlan.collectLeaves().head.metrics,
+ "remoteSchemaFetchTime")
+ }
+
+ test("schema fetch time metric: DataFrameReader jdbc") {
+ val df = spark.read.jdbc(url, "TEST.PEOPLE", Array[String](), new Properties())
+ hasMetricKeyValue(
+ df.queryExecution.executedPlan.collectLeaves().head.metrics,
+ "remoteSchemaFetchTime")
+ }
+}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index bed8a1a7b363..276062fb1daa 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -1693,7 +1693,7 @@ class JDBCSuite extends QueryTest with SharedSparkSession {
val quotedPrtColName = testH2Dialect.quoteIdentifier(expectedColumnName)
df.logicalPlan match {
- case LogicalRelationWithTable(JDBCRelation(_, parts, _), _) =>
+ case LogicalRelationWithTable(JDBCRelation(_, parts, _, _), _) =>
val whereClauses = parts.map(_.asInstanceOf[JDBCPartition].whereClause).toSet
assert(whereClauses === Set(
s"$quotedPrtColName < 2 or $quotedPrtColName is null",
@@ -1835,7 +1835,7 @@ class JDBCSuite extends QueryTest with SharedSparkSession {
.load()
df1.logicalPlan match {
- case LogicalRelationWithTable(JDBCRelation(_, parts, _), _) =>
+ case LogicalRelationWithTable(JDBCRelation(_, parts, _, _), _) =>
val whereClauses = parts.map(_.asInstanceOf[JDBCPartition].whereClause).toSet
assert(whereClauses === Set(
""""D" < '2018-07-11' or "D" is null""",
@@ -1855,7 +1855,7 @@ class JDBCSuite extends QueryTest with SharedSparkSession {
.load()
df2.logicalPlan match {
- case LogicalRelationWithTable(JDBCRelation(_, parts, _), _) =>
+ case LogicalRelationWithTable(JDBCRelation(_, parts, _, _), _) =>
val whereClauses = parts.map(_.asInstanceOf[JDBCPartition].whereClause).toSet
assert(whereClauses === Set(
""""T" < '2018-07-15 20:50:32.5' or "T" is null""",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org