You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@datafusion.apache.org by ag...@apache.org on 2026/07/31 18:02:57 UTC
(datafusion-comet) branch main updated: chore: supply explicit empty argument lists for 2.13 (#5175)
This is an automated email from the ASF dual-hosted git repository.
andygrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new bfc57995c5 chore: supply explicit empty argument lists for 2.13 (#5175)
bfc57995c5 is described below
commit bfc57995c5813dea212609c261a09b40c3341d5f
Author: Andy Grove <ag...@apache.org>
AuthorDate: Fri Jul 31 12:02:53 2026 -0600
chore: supply explicit empty argument lists for 2.13 (#5175)
Scala 2.13 deprecates auto-application: calling `foo` when the method is
declared `def foo()`. Comet does this at 20 sites, 21 warnings of the 127
under the 2.13 profiles once the related `hasNext` override is counted.
Auto-application is purely syntactic, so 19 of the sites just gain `()`:
SparkSession.builder in nine benchmark sessions, Random.nextLong /
nextInt in two benchmarks, Dataset.count in two tests,
TaskContext.taskAttemptId, TimestampFormatter.defaultPattern (twice),
and ConfigEntry.get - the last three next to existing call sites that
already write `()`.
NativeBatchDecoderIterator is the exception. It declares
`def hasNext(): Boolean` while Iterator declares `hasNext` without a
parameter list, which warns on the declaration and then again on the
internal call. Dropping the parameter list from the override fixes both
and is safe: the only external consumer, CometBlockStoreShuffleReader,
goes through Iterator.
spotless reflows the nine `SparkSession.builder()` calls onto their own
line, which is the only formatting change.
---
spark/src/main/scala/org/apache/comet/CometExecIterator.scala | 2 +-
spark/src/main/scala/org/apache/comet/serde/unixtime.scala | 4 ++--
.../spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala | 2 +-
.../sql/comet/execution/shuffle/NativeBatchDecoderIterator.scala | 2 +-
.../test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala | 2 +-
spark/src/test/scala/org/apache/spark/sql/CometTPCQueryBase.scala | 3 ++-
spark/src/test/scala/org/apache/spark/sql/Tables.scala | 2 +-
.../scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala | 5 +++--
.../apache/spark/sql/benchmark/CometBroadcastHashJoinBenchmark.scala | 3 ++-
.../spark/sql/benchmark/CometBroadcastNestedLoopJoinBenchmark.scala | 3 ++-
.../org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala | 3 ++-
.../scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala | 3 ++-
.../org/apache/spark/sql/benchmark/CometHashJoinBenchmark.scala | 3 ++-
.../scala/org/apache/spark/sql/benchmark/CometReadBenchmark.scala | 2 +-
.../org/apache/spark/sql/benchmark/CometRegExpExtractBenchmark.scala | 2 +-
.../scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala | 3 ++-
.../org/apache/spark/sql/benchmark/CometSortMergeJoinBenchmark.scala | 3 ++-
17 files changed, 28 insertions(+), 19 deletions(-)
diff --git a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala
index c9dc8d3414..a3216404ac 100644
--- a/spark/src/main/scala/org/apache/comet/CometExecIterator.scala
+++ b/spark/src/main/scala/org/apache/comet/CometExecIterator.scala
@@ -80,7 +80,7 @@ class CometExecIterator(
private val memoryMXBean = ManagementFactory.getMemoryMXBean
private val nativeLib = new Native()
private val nativeUtil = new NativeUtil()
- private val taskAttemptId = TaskContext.get().taskAttemptId
+ private val taskAttemptId = TaskContext.get().taskAttemptId()
private val taskCPUs = TaskContext.get().cpus()
private val cometTaskMemoryManager = new CometTaskMemoryManager(id, taskAttemptId)
diff --git a/spark/src/main/scala/org/apache/comet/serde/unixtime.scala b/spark/src/main/scala/org/apache/comet/serde/unixtime.scala
index 256baffa95..f88528c468 100644
--- a/spark/src/main/scala/org/apache/comet/serde/unixtime.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/unixtime.scala
@@ -54,7 +54,7 @@ object CometFromUnixTime extends CometExpressionSerde[FromUnixTime] with Codegen
// only appear on the format argument, and any collated format is a non-default format, which is
// already `Unsupported` here.
override def getSupportLevel(expr: FromUnixTime): SupportLevel = {
- if (expr.format != Literal(TimestampFormatter.defaultPattern)) {
+ if (expr.format != Literal(TimestampFormatter.defaultPattern())) {
Unsupported(Some(formatReason))
} else {
Incompatible(Some(timestampRangeReason))
@@ -73,7 +73,7 @@ object CometFromUnixTime extends CometExpressionSerde[FromUnixTime] with Codegen
val formatExpr = exprToProtoInternal(Literal("%Y-%m-%d %H:%M:%S"), inputs, binding)
val timeZone = exprToProtoInternal(Literal(expr.timeZoneId.orNull), inputs, binding)
- if (expr.format != Literal(TimestampFormatter.defaultPattern)) {
+ if (expr.format != Literal(TimestampFormatter.defaultPattern())) {
withFallbackReason(expr, formatReason)
None
} else if (secExpr.isDefined && formatExpr.isDefined) {
diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala
index 15f620339d..7287a3599e 100644
--- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala
+++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala
@@ -231,7 +231,7 @@ class CometNativeShuffleWriter[K, V](
} else {
shuffleWriterBuilder.setCodec(CompressionCodec.None)
}
- shuffleWriterBuilder.setCompressionLevel(CometConf.COMET_SHUFFLE_COMPRESSION_ZSTD_LEVEL.get)
+ shuffleWriterBuilder.setCompressionLevel(CometConf.COMET_SHUFFLE_COMPRESSION_ZSTD_LEVEL.get())
shuffleWriterBuilder.setWriteBufferSize(
CometConf.COMET_SHUFFLE_NATIVE_WRITE_BUFFER_SIZE.get().min(Int.MaxValue).toInt)
shuffleWriterBuilder.setMaxBufferBytes(CometConf.COMET_SHUFFLE_NATIVE_MAX_BUFFER_BYTES.get())
diff --git a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/NativeBatchDecoderIterator.scala b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/NativeBatchDecoderIterator.scala
index f96c8f16dd..28665f4aae 100644
--- a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/NativeBatchDecoderIterator.scala
+++ b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/NativeBatchDecoderIterator.scala
@@ -55,7 +55,7 @@ case class NativeBatchDecoderIterator(
null
}
- def hasNext(): Boolean = {
+ override def hasNext: Boolean = {
if (channel == null || isClosed) {
return false
}
diff --git a/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala
index 67a8292180..fd7d4ee3f4 100644
--- a/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/exec/CometNativeReaderSuite.scala
@@ -598,7 +598,7 @@ class CometNativeReaderSuite extends CometTestBase with AdaptiveSparkPlanHelper
test("SPARK-18053: ARRAY equality is broken") {
withTable("array_tbl") {
spark.range(10).select(array(col("id")).as("arr")).write.saveAsTable("array_tbl")
- assert(sql("SELECT * FROM array_tbl where arr = ARRAY(1L)").count == 1)
+ assert(sql("SELECT * FROM array_tbl where arr = ARRAY(1L)").count() == 1)
}
}
diff --git a/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryBase.scala b/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryBase.scala
index 99f5f3f427..a343755975 100644
--- a/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryBase.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/CometTPCQueryBase.scala
@@ -52,7 +52,8 @@ trait CometTPCQueryBase extends Logging {
"spark.shuffle.manager",
"org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
- val sparkSession = SparkSession.builder
+ val sparkSession = SparkSession
+ .builder()
.config(conf)
.withExtensions(new CometSparkSessionExtensions)
.getOrCreate()
diff --git a/spark/src/test/scala/org/apache/spark/sql/Tables.scala b/spark/src/test/scala/org/apache/spark/sql/Tables.scala
index 5970e05b50..31a49460aa 100644
--- a/spark/src/test/scala/org/apache/spark/sql/Tables.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/Tables.scala
@@ -174,7 +174,7 @@ abstract class Tables(
// in case data has more than maxRecordsPerFile, split into multiple writers to improve
// datagen speed files will be truncated to maxRecordsPerFile value, so the final result
// will be the same
- val numRows = data.count
+ val numRows = data.count()
val maxRecordPerFile =
Try(sqlContext.getConf("spark.sql.files.maxRecordsPerFile").toInt).getOrElse(0)
diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala
index 42dba5ba30..5c5f84ebbb 100644
--- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBenchmarkBase.scala
@@ -61,7 +61,8 @@ trait CometBenchmarkBase
"spark.shuffle.manager",
"org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
- val sparkSession = SparkSession.builder
+ val sparkSession = SparkSession
+ .builder()
.config(conf)
.withExtensions(new CometSparkSessionExtensions)
.getOrCreate()
@@ -98,7 +99,7 @@ trait CometBenchmarkBase
import spark.implicits._
spark
.range(values)
- .map(_ => if (useDictionary) Random.nextLong % 5 else Random.nextLong)
+ .map(_ => if (useDictionary) Random.nextLong() % 5 else Random.nextLong())
.createOrReplaceTempView(tbl)
runBenchmark(benchmarkName)(f(values))
}
diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastHashJoinBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastHashJoinBenchmark.scala
index bcbb3afca5..152d79dbb9 100644
--- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastHashJoinBenchmark.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastHashJoinBenchmark.scala
@@ -44,7 +44,8 @@ object CometBroadcastHashJoinBenchmark extends CometBenchmarkBase {
"spark.shuffle.manager",
"org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
- val sparkSession = SparkSession.builder
+ val sparkSession = SparkSession
+ .builder()
.config(conf)
.withExtensions(new CometSparkSessionExtensions)
.getOrCreate()
diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastNestedLoopJoinBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastNestedLoopJoinBenchmark.scala
index e86903f6ea..f8fe6100d4 100644
--- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastNestedLoopJoinBenchmark.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometBroadcastNestedLoopJoinBenchmark.scala
@@ -45,7 +45,8 @@ object CometBroadcastNestedLoopJoinBenchmark extends CometBenchmarkBase {
"spark.shuffle.manager",
"org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
- val sparkSession = SparkSession.builder
+ val sparkSession = SparkSession
+ .builder()
.config(conf)
.withExtensions(new CometSparkSessionExtensions)
.getOrCreate()
diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala
index 95eb44d1c9..91db9796fa 100644
--- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometColumnarToRowBenchmark.scala
@@ -49,7 +49,8 @@ object CometColumnarToRowBenchmark extends CometBenchmarkBase {
.set("spark.memory.offHeap.enabled", "true")
.set("spark.memory.offHeap.size", "2g")
- val sparkSession = SparkSession.builder
+ val sparkSession = SparkSession
+ .builder()
.config(conf)
.withExtensions(new CometSparkSessionExtensions)
.getOrCreate()
diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala
index 012f92cc10..2e7e143a47 100644
--- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometExecBenchmark.scala
@@ -48,7 +48,8 @@ object CometExecBenchmark extends CometBenchmarkBase {
"org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
.set("spark.comet.shuffle.jvm.spillThreshold", "30000")
- val sparkSession = SparkSession.builder
+ val sparkSession = SparkSession
+ .builder()
.config(conf)
.withExtensions(new CometSparkSessionExtensions)
.getOrCreate()
diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashJoinBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashJoinBenchmark.scala
index d98b20284b..ef2c325c54 100644
--- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashJoinBenchmark.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometHashJoinBenchmark.scala
@@ -44,7 +44,8 @@ object CometHashJoinBenchmark extends CometBenchmarkBase {
"spark.shuffle.manager",
"org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
- val sparkSession = SparkSession.builder
+ val sparkSession = SparkSession
+ .builder()
.config(conf)
.withExtensions(new CometSparkSessionExtensions)
.getOrCreate()
diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometReadBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometReadBenchmark.scala
index 5b0371b277..30090d1d12 100644
--- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometReadBenchmark.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometReadBenchmark.scala
@@ -347,7 +347,7 @@ class CometReadBaseBenchmark extends CometBenchmarkBase {
runBenchmark("SQL Decimal Column Scan") {
withTempTable(tbl) {
import spark.implicits._
- spark.range(1024 * 1024 * 15).map(_ => Random.nextInt).createOrReplaceTempView(tbl)
+ spark.range(1024 * 1024 * 15).map(_ => Random.nextInt()).createOrReplaceTempView(tbl)
Seq((5, 2), (18, 4), (20, 8)).foreach { case (precision, scale) =>
decimalScanBenchmark(1024 * 1024 * 15, precision, scale)
diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpExtractBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpExtractBenchmark.scala
index 2db4fff134..26cc7f73f1 100644
--- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpExtractBenchmark.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometRegExpExtractBenchmark.scala
@@ -74,7 +74,7 @@ object CometRegExpExtractBenchmark extends CometBenchmarkBase {
"spark.shuffle.manager",
"org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
- val sparkSession = SparkSession.builder.config(conf).getOrCreate()
+ val sparkSession = SparkSession.builder().config(conf).getOrCreate()
sparkSession.conf.set(SQLConf.PARQUET_VECTORIZED_READER_ENABLED.key, "true")
sparkSession.conf.set(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key, "true")
sparkSession.conf.set(CometConf.COMET_ENABLED.key, "false")
diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala
index e13cfca353..05f8e6c836 100644
--- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometShuffleBenchmark.scala
@@ -55,7 +55,8 @@ object CometShuffleBenchmark extends CometBenchmarkBase {
"org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
.set("spark.comet.shuffle.jvm.spillThreshold", "30000")
- val sparkSession = SparkSession.builder
+ val sparkSession = SparkSession
+ .builder()
.config(conf)
.withExtensions(new CometSparkSessionExtensions)
.getOrCreate()
diff --git a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSortMergeJoinBenchmark.scala b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSortMergeJoinBenchmark.scala
index 2b4272cdab..a429da5835 100644
--- a/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSortMergeJoinBenchmark.scala
+++ b/spark/src/test/scala/org/apache/spark/sql/benchmark/CometSortMergeJoinBenchmark.scala
@@ -44,7 +44,8 @@ object CometSortMergeJoinBenchmark extends CometBenchmarkBase {
"spark.shuffle.manager",
"org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager")
- val sparkSession = SparkSession.builder
+ val sparkSession = SparkSession
+ .builder()
.config(conf)
.withExtensions(new CometSparkSessionExtensions)
.getOrCreate()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org