You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by et...@apache.org on 2023/05/15 13:21:43 UTC
[iceberg] branch master updated: Spark: Fix Parquet read benchmarks for Spark 3.3 + 3.4 (#7587)
This is an automated email from the ASF dual-hosted git repository.
etudenhoefner pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 81af3f76ab Spark: Fix Parquet read benchmarks for Spark 3.3 + 3.4 (#7587)
81af3f76ab is described below
commit 81af3f76abe98419495bd4d18440a14bfaae2e0e
Author: Eduard Tudenhoefner <et...@gmail.com>
AuthorDate: Mon May 15 15:21:35 2023 +0200
Spark: Fix Parquet read benchmarks for Spark 3.3 + 3.4 (#7587)
---
.../spark/data/parquet/SparkParquetReadersFlatDataBenchmark.java | 2 ++
.../spark/data/parquet/SparkParquetReadersNestedDataBenchmark.java | 2 ++
.../spark/data/parquet/SparkParquetReadersFlatDataBenchmark.java | 4 ++++
.../spark/data/parquet/SparkParquetReadersNestedDataBenchmark.java | 4 ++++
4 files changed, 12 insertions(+)
diff --git a/spark/v3.3/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersFlatDataBenchmark.java b/spark/v3.3/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersFlatDataBenchmark.java
index 63f111a37d..434794f26e 100644
--- a/spark/v3.3/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersFlatDataBenchmark.java
+++ b/spark/v3.3/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersFlatDataBenchmark.java
@@ -156,6 +156,7 @@ public class SparkParquetReadersFlatDataBenchmark {
.set("spark.sql.parquet.int96AsTimestamp", "false")
.set("spark.sql.caseSensitive", "false")
.set("spark.sql.parquet.fieldId.write.enabled", "false")
+ .set("spark.sql.legacy.parquet.nanosAsLong", "false")
.callInit()
.build()) {
@@ -214,6 +215,7 @@ public class SparkParquetReadersFlatDataBenchmark {
.set("spark.sql.parquet.binaryAsString", "false")
.set("spark.sql.parquet.int96AsTimestamp", "false")
.set("spark.sql.caseSensitive", "false")
+ .set("spark.sql.legacy.parquet.nanosAsLong", "false")
.callInit()
.build()) {
diff --git a/spark/v3.3/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersNestedDataBenchmark.java b/spark/v3.3/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersNestedDataBenchmark.java
index 7a47d7ca53..aeadc43de9 100644
--- a/spark/v3.3/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersNestedDataBenchmark.java
+++ b/spark/v3.3/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersNestedDataBenchmark.java
@@ -154,6 +154,7 @@ public class SparkParquetReadersNestedDataBenchmark {
.set("spark.sql.parquet.int96AsTimestamp", "false")
.set("spark.sql.caseSensitive", "false")
.set("spark.sql.parquet.fieldId.write.enabled", "false")
+ .set("spark.sql.legacy.parquet.nanosAsLong", "false")
.callInit()
.build()) {
@@ -212,6 +213,7 @@ public class SparkParquetReadersNestedDataBenchmark {
.set("spark.sql.parquet.binaryAsString", "false")
.set("spark.sql.parquet.int96AsTimestamp", "false")
.set("spark.sql.caseSensitive", "false")
+ .set("spark.sql.legacy.parquet.nanosAsLong", "false")
.callInit()
.build()) {
diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersFlatDataBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersFlatDataBenchmark.java
index 63f111a37d..e65e2fb576 100644
--- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersFlatDataBenchmark.java
+++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersFlatDataBenchmark.java
@@ -156,6 +156,8 @@ public class SparkParquetReadersFlatDataBenchmark {
.set("spark.sql.parquet.int96AsTimestamp", "false")
.set("spark.sql.caseSensitive", "false")
.set("spark.sql.parquet.fieldId.write.enabled", "false")
+ .set("spark.sql.parquet.inferTimestampNTZ.enabled", "false")
+ .set("spark.sql.legacy.parquet.nanosAsLong", "false")
.callInit()
.build()) {
@@ -214,6 +216,8 @@ public class SparkParquetReadersFlatDataBenchmark {
.set("spark.sql.parquet.binaryAsString", "false")
.set("spark.sql.parquet.int96AsTimestamp", "false")
.set("spark.sql.caseSensitive", "false")
+ .set("spark.sql.parquet.inferTimestampNTZ.enabled", "false")
+ .set("spark.sql.legacy.parquet.nanosAsLong", "false")
.callInit()
.build()) {
diff --git a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersNestedDataBenchmark.java b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersNestedDataBenchmark.java
index 7a47d7ca53..3520c9b474 100644
--- a/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersNestedDataBenchmark.java
+++ b/spark/v3.4/spark/src/jmh/java/org/apache/iceberg/spark/data/parquet/SparkParquetReadersNestedDataBenchmark.java
@@ -154,6 +154,8 @@ public class SparkParquetReadersNestedDataBenchmark {
.set("spark.sql.parquet.int96AsTimestamp", "false")
.set("spark.sql.caseSensitive", "false")
.set("spark.sql.parquet.fieldId.write.enabled", "false")
+ .set("spark.sql.parquet.inferTimestampNTZ.enabled", "false")
+ .set("spark.sql.legacy.parquet.nanosAsLong", "false")
.callInit()
.build()) {
@@ -212,6 +214,8 @@ public class SparkParquetReadersNestedDataBenchmark {
.set("spark.sql.parquet.binaryAsString", "false")
.set("spark.sql.parquet.int96AsTimestamp", "false")
.set("spark.sql.caseSensitive", "false")
+ .set("spark.sql.parquet.inferTimestampNTZ.enabled", "false")
+ .set("spark.sql.legacy.parquet.nanosAsLong", "false")
.callInit()
.build()) {