You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by gi...@apache.org on 2026/10/13 10:45:00 UTC
(datafusion-comet) branch gh-readonly-queue/main/pr-6925-987ee7397f081f8f360803c9f593308eedde0a56 created (now 474b5ece84)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a change to branch gh-readonly-queue/main/pr-6925-987ee7397f081f8f360803c9f593308eedde0a56
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
at 474b5ece84 perf: avoid duplicate Hadoop configs in Iceberg write detection (#6925)
This branch includes the following new commits:
new 474b5ece84 perf: avoid duplicate Hadoop configs in Iceberg write detection (#6925)
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org
(datafusion-comet) 01/01: perf: avoid duplicate Hadoop configs in Iceberg write detection (#6925)
Posted by gi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-6925-987ee7397f081f8f360803c9f593308eedde0a56
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
commit 474b5ece841a7fc6dff37ae04bd6787e654d7be7
Author: Andy Grove <ag...@apache.org>
AuthorDate: Tue Oct 13 10:44:38 2026 +0000
perf: avoid duplicate Hadoop configs in Iceberg write detection (#6925)
---
.../serde/operator/CometIcebergNativeWrite.scala | 33 ++++++++++++++--------
.../comet/CometIcebergWriteDetectionSuite.scala | 29 +++++++++++++++++++
2 files changed, 51 insertions(+), 11 deletions(-)
diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala
index 9c708e2d0d..42220ea9b2 100644
--- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala
@@ -326,12 +326,11 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] {
.getWritePropertiesFromSparkWrite(sparkWrite)
.getOrElse(return Some("could not read SparkWrite.writeProperties"))
- val context = TriggerContext(
- table,
- tableProperties ++ writeProperties,
- sparkWrite,
- op.session.sessionState.newHadoopConf(),
- effectiveHadoopConf(op, table))
+ val hadoopConfs = new LazyHadoopConfs(
+ () => op.session.sessionState.newHadoopConf(),
+ sessionConf => effectiveHadoopConf(op, table, sessionConf))
+ val context =
+ TriggerContext(table, tableProperties ++ writeProperties, sparkWrite, hadoopConfs)
triggers.iterator.map(rule => rule(context)).collectFirst { case Some(reason) => reason }
}
@@ -339,13 +338,24 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] {
table: Any,
properties: Map[String, String],
sparkWrite: Any,
- hadoopConf: Configuration,
- s3HadoopConf: Configuration)
+ hadoopConfs: LazyHadoopConfs) {
+ def hadoopConf: Configuration = hadoopConfs.session
+ def s3HadoopConf: Configuration = hadoopConfs.effective
+ }
+
+ private[comet] final class LazyHadoopConfs(
+ sessionConfBuilder: () => Configuration,
+ effectiveConfBuilder: Configuration => Configuration) {
+ lazy val session: Configuration = sessionConfBuilder()
+ lazy val effective: Configuration = effectiveConfBuilder(session)
+ }
private type TriggerRule = TriggerContext => Option[String]
- private def effectiveHadoopConf(op: IcebergWriteExec, table: Any): Configuration = {
- val sessionConf = op.session.sessionState.newHadoopConf()
+ private def effectiveHadoopConf(
+ op: IcebergWriteExec,
+ table: Any,
+ sessionConf: Configuration): Configuration = {
val catalogOverrides = IcebergReflection
.deriveCatalogName(table)
.map { catalogName =>
@@ -1147,7 +1157,8 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] {
// effective FileIO Hadoop configuration, translate them to the s3.* keys iceberg-rust
// consumes, and let FileIO/vended properties win on conflict. The FileIO configuration
// includes SparkCatalog's catalog-specific `hadoop.*` overrides.
- val writeHadoopConf = effectiveHadoopConf(op, table)
+ val writeHadoopConf =
+ effectiveHadoopConf(op, table, op.session.sessionState.newHadoopConf())
val dataUri = new java.net.URI(dataLocation)
// Promote the data bucket's per-bucket `fs.s3a.bucket.<b>.*` settings to global, mirroring the
// scan path: iceberg-rust's pinned S3 parser reads only global `s3.*`. Only an S3-family data
diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala
index 430a92f97a..2d0c40357f 100644
--- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala
@@ -65,6 +65,35 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes
}
}
+ test("Hadoop configurations are built lazily and share the session configuration") {
+ var sessionBuilds = 0
+ var effectiveBuilds = 0
+ val sessionConf = new Configuration(false)
+ val hadoopConfs = new CometIcebergNativeWrite.LazyHadoopConfs(
+ () => {
+ sessionBuilds += 1
+ sessionConf
+ },
+ conf => {
+ effectiveBuilds += 1
+ assert(conf eq sessionConf)
+ new Configuration(conf)
+ })
+
+ assert(sessionBuilds == 0)
+ assert(effectiveBuilds == 0)
+ assert(hadoopConfs.session eq sessionConf)
+ assert(sessionBuilds == 1)
+ assert(effectiveBuilds == 0)
+ assert(hadoopConfs.effective ne sessionConf)
+ assert(sessionBuilds == 1)
+ assert(effectiveBuilds == 1)
+ assert(hadoopConfs.session eq sessionConf)
+ assert(hadoopConfs.effective ne sessionConf)
+ assert(sessionBuilds == 1)
+ assert(effectiveBuilds == 1)
+ }
+
test("clean parquet V2 table planned as AppendData yields Compatible") {
withDetectionCatalog { _ =>
createTable("ok", partitionSpec = "")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org