You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ao...@apache.org on 2023/09/27 20:08:47 UTC

[iceberg] branch master updated: Spark 3.5: Increase default advisory partition size for writes (#8660)

This is an automated email from the ASF dual-hosted git repository.

aokolnychyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/master by this push:
     new 0b1b6241dd Spark 3.5: Increase default advisory partition size for writes (#8660)
0b1b6241dd is described below

commit 0b1b6241dd334d6e25ee83cd6c0bbde7b337da7f
Author: Anton Okolnychyi <ao...@apple.com>
AuthorDate: Wed Sep 27 13:08:41 2023 -0700

    Spark 3.5: Increase default advisory partition size for writes (#8660)
---
 .../java/org/apache/iceberg/TableProperties.java   |   3 +
 .../iceberg/spark/extensions/TestDelete.java       |  30 +++-
 .../apache/iceberg/spark/extensions/TestMerge.java |  36 +++--
 .../iceberg/spark/extensions/TestUpdate.java       |  30 +++-
 .../apache/iceberg/spark/SparkCompressionUtil.java | 148 +++++++++++++++++++
 .../org/apache/iceberg/spark/SparkConfParser.java  |   6 +-
 .../apache/iceberg/spark/SparkSQLProperties.java   |   3 +
 .../org/apache/iceberg/spark/SparkWriteConf.java   | 144 ++++++++++++------
 .../apache/iceberg/spark/SparkWriteOptions.java    |   3 +
 .../iceberg/spark/SparkWriteRequirements.java      |  13 +-
 .../org/apache/iceberg/spark/SparkWriteUtil.java   |  24 ++-
 .../spark/source/SparkPositionDeltaWrite.java      |  15 +-
 .../apache/iceberg/spark/source/SparkWrite.java    |  15 +-
 .../iceberg/spark/TestSparkCompressionUtil.java    | 162 +++++++++++++++++++++
 14 files changed, 543 insertions(+), 89 deletions(-)

diff --git a/core/src/main/java/org/apache/iceberg/TableProperties.java b/core/src/main/java/org/apache/iceberg/TableProperties.java
index af90303a86..f315a0dfc9 100644
--- a/core/src/main/java/org/apache/iceberg/TableProperties.java
+++ b/core/src/main/java/org/apache/iceberg/TableProperties.java
@@ -304,6 +304,9 @@ public class TableProperties {
   public static final String SPARK_WRITE_ACCEPT_ANY_SCHEMA = "write.spark.accept-any-schema";
   public static final boolean SPARK_WRITE_ACCEPT_ANY_SCHEMA_DEFAULT = false;
 
+  public static final String SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES =
+      "write.spark.advisory-partition-size-bytes";
+
   public static final String SNAPSHOT_ID_INHERITANCE_ENABLED =
       "compatibility.snapshot-id-inheritance.enabled";
   public static final boolean SNAPSHOT_ID_INHERITANCE_ENABLED_DEFAULT = false;
diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestDelete.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestDelete.java
index 741b9de31a..a5f62d73e9 100644
--- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestDelete.java
+++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestDelete.java
@@ -177,12 +177,19 @@ public abstract class TestDelete extends SparkRowLevelOperationsTestBase {
 
     // enable AQE and set the advisory partition size big enough to trigger combining
     // set the number of shuffle partitions to 200 to distribute the work across reducers
+    // set the advisory partition size for shuffles small enough to ensure writes override it
     withSQLConf(
         ImmutableMap.of(
-            SQLConf.SHUFFLE_PARTITIONS().key(), "200",
-            SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), "true",
-            SQLConf.COALESCE_PARTITIONS_ENABLED().key(), "true",
-            SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES().key(), "256MB"),
+            SQLConf.SHUFFLE_PARTITIONS().key(),
+            "200",
+            SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(),
+            "true",
+            SQLConf.COALESCE_PARTITIONS_ENABLED().key(),
+            "true",
+            SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES().key(),
+            "100",
+            SparkSQLProperties.ADVISORY_PARTITION_SIZE,
+            String.valueOf(256 * 1024 * 1024)),
         () -> {
           SparkPlan plan =
               executeAndKeepPlan("DELETE FROM %s WHERE mod(id, 2) = 0", commitTarget());
@@ -238,12 +245,19 @@ public abstract class TestDelete extends SparkRowLevelOperationsTestBase {
 
     // enable AQE and set the advisory partition size small enough to trigger a split
     // set the number of shuffle partitions to 2 to only have 2 reducers
+    // set the advisory partition size for shuffles big enough to ensure writes override it
     withSQLConf(
         ImmutableMap.of(
-            SQLConf.SHUFFLE_PARTITIONS().key(), "2",
-            SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), "true",
-            SQLConf.ADAPTIVE_OPTIMIZE_SKEWS_IN_REBALANCE_PARTITIONS_ENABLED().key(), "true",
-            SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES().key(), "100"),
+            SQLConf.SHUFFLE_PARTITIONS().key(),
+            "2",
+            SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(),
+            "true",
+            SQLConf.ADAPTIVE_OPTIMIZE_SKEWS_IN_REBALANCE_PARTITIONS_ENABLED().key(),
+            "true",
+            SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES().key(),
+            "256MB",
+            SparkSQLProperties.ADVISORY_PARTITION_SIZE,
+            "100"),
         () -> {
           SparkPlan plan =
               executeAndKeepPlan("DELETE FROM %s WHERE mod(id, 2) = 0", commitTarget());
diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMerge.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMerge.java
index 3de4856e1a..b8439a399f 100644
--- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMerge.java
+++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMerge.java
@@ -282,13 +282,21 @@ public abstract class TestMerge extends SparkRowLevelOperationsTestBase {
     // enable AQE and set the advisory partition big enough to trigger combining
     // set the number of shuffle partitions to 200 to distribute the work across reducers
     // disable broadcast joins to make sure the join triggers a shuffle
+    // set the advisory partition size for shuffles small enough to ensure writes override it
     withSQLConf(
         ImmutableMap.of(
-            SQLConf.SHUFFLE_PARTITIONS().key(), "200",
-            SQLConf.AUTO_BROADCASTJOIN_THRESHOLD().key(), "-1",
-            SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), "true",
-            SQLConf.COALESCE_PARTITIONS_ENABLED().key(), "true",
-            SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES().key(), "256MB"),
+            SQLConf.SHUFFLE_PARTITIONS().key(),
+            "200",
+            SQLConf.AUTO_BROADCASTJOIN_THRESHOLD().key(),
+            "-1",
+            SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(),
+            "true",
+            SQLConf.COALESCE_PARTITIONS_ENABLED().key(),
+            "true",
+            SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES().key(),
+            "100",
+            SparkSQLProperties.ADVISORY_PARTITION_SIZE,
+            String.valueOf(256 * 1024 * 1024)),
         () -> {
           sql(
               "MERGE INTO %s t USING source "
@@ -352,13 +360,21 @@ public abstract class TestMerge extends SparkRowLevelOperationsTestBase {
     // enable AQE and set the advisory partition size small enough to trigger a split
     // set the number of shuffle partitions to 2 to only have 2 reducers
     // set the min coalesce partition size small enough to avoid coalescing
+    // set the advisory partition size for shuffles big enough to ensure writes override it
     withSQLConf(
         ImmutableMap.of(
-            SQLConf.SHUFFLE_PARTITIONS().key(), "4",
-            SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_SIZE().key(), "100",
-            SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), "true",
-            SQLConf.ADAPTIVE_OPTIMIZE_SKEWS_IN_REBALANCE_PARTITIONS_ENABLED().key(), "true",
-            SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES().key(), "100"),
+            SQLConf.SHUFFLE_PARTITIONS().key(),
+            "4",
+            SQLConf.COALESCE_PARTITIONS_MIN_PARTITION_SIZE().key(),
+            "100",
+            SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(),
+            "true",
+            SQLConf.ADAPTIVE_OPTIMIZE_SKEWS_IN_REBALANCE_PARTITIONS_ENABLED().key(),
+            "true",
+            SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES().key(),
+            "256MB",
+            SparkSQLProperties.ADVISORY_PARTITION_SIZE,
+            "100"),
         () -> {
           SparkPlan plan =
               executeAndKeepPlan(
diff --git a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestUpdate.java b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestUpdate.java
index 8e43947fd2..5633b30ee5 100644
--- a/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestUpdate.java
+++ b/spark/v3.5/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestUpdate.java
@@ -162,12 +162,19 @@ public abstract class TestUpdate extends SparkRowLevelOperationsTestBase {
 
     // enable AQE and set the advisory partition size big enough to trigger combining
     // set the number of shuffle partitions to 200 to distribute the work across reducers
+    // set the advisory partition size for shuffles small enough to ensure writes override it
     withSQLConf(
         ImmutableMap.of(
-            SQLConf.SHUFFLE_PARTITIONS().key(), "200",
-            SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), "true",
-            SQLConf.COALESCE_PARTITIONS_ENABLED().key(), "true",
-            SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES().key(), "256MB"),
+            SQLConf.SHUFFLE_PARTITIONS().key(),
+            "200",
+            SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(),
+            "true",
+            SQLConf.COALESCE_PARTITIONS_ENABLED().key(),
+            "true",
+            SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES().key(),
+            "100",
+            SparkSQLProperties.ADVISORY_PARTITION_SIZE,
+            String.valueOf(256 * 1024 * 1024)),
         () -> {
           SparkPlan plan =
               executeAndKeepPlan("UPDATE %s SET id = -1 WHERE mod(id, 2) = 0", commitTarget());
@@ -226,12 +233,19 @@ public abstract class TestUpdate extends SparkRowLevelOperationsTestBase {
 
     // enable AQE and set the advisory partition size small enough to trigger a split
     // set the number of shuffle partitions to 2 to only have 2 reducers
+    // set the advisory partition size for shuffles big enough to ensure writes override it
     withSQLConf(
         ImmutableMap.of(
-            SQLConf.SHUFFLE_PARTITIONS().key(), "2",
-            SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(), "true",
-            SQLConf.ADAPTIVE_OPTIMIZE_SKEWS_IN_REBALANCE_PARTITIONS_ENABLED().key(), "true",
-            SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES().key(), "100"),
+            SQLConf.SHUFFLE_PARTITIONS().key(),
+            "2",
+            SQLConf.ADAPTIVE_EXECUTION_ENABLED().key(),
+            "true",
+            SQLConf.ADAPTIVE_OPTIMIZE_SKEWS_IN_REBALANCE_PARTITIONS_ENABLED().key(),
+            "true",
+            SQLConf.ADVISORY_PARTITION_SIZE_IN_BYTES().key(),
+            "256MB",
+            SparkSQLProperties.ADVISORY_PARTITION_SIZE,
+            "100"),
         () -> {
           SparkPlan plan =
               executeAndKeepPlan("UPDATE %s SET id = -1 WHERE mod(id, 2) = 0", commitTarget());
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCompressionUtil.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCompressionUtil.java
new file mode 100644
index 0000000000..8f00b7f830
--- /dev/null
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkCompressionUtil.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark;
+
+import java.util.Locale;
+import java.util.Map;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.apache.iceberg.util.Pair;
+import org.apache.spark.SparkConf;
+import org.apache.spark.sql.SparkSession;
+
+class SparkCompressionUtil {
+
+  private static final String LZ4 = "lz4";
+  private static final String ZSTD = "zstd";
+  private static final String GZIP = "gzip";
+  private static final String ZLIB = "zlib";
+  private static final String SNAPPY = "snappy";
+  private static final String NONE = "none";
+
+  // an internal Spark config that controls whether shuffle data is compressed
+  private static final String SHUFFLE_COMPRESSION_ENABLED = "spark.shuffle.compress";
+  private static final boolean SHUFFLE_COMPRESSION_ENABLED_DEFAULT = true;
+
+  // an internal Spark config that controls what compression codec is used
+  private static final String SPARK_COMPRESSION_CODEC = "spark.io.compression.codec";
+  private static final String SPARK_COMPRESSION_CODEC_DEFAULT = "lz4";
+
+  private static final double DEFAULT_COLUMNAR_COMPRESSION = 2;
+  private static final Map<Pair<String, String>, Double> COLUMNAR_COMPRESSIONS =
+      initColumnarCompressions();
+
+  private static final double DEFAULT_ROW_BASED_COMPRESSION = 1;
+  private static final Map<Pair<String, String>, Double> ROW_BASED_COMPRESSIONS =
+      initRowBasedCompressions();
+
+  private SparkCompressionUtil() {}
+
+  /**
+   * Estimates how much the data in shuffle map files will compress once it is written to disk using
+   * a particular file format and codec.
+   */
+  public static double shuffleCompressionRatio(
+      SparkSession spark, FileFormat outputFileFormat, String outputCodec) {
+    if (outputFileFormat == FileFormat.ORC || outputFileFormat == FileFormat.PARQUET) {
+      return columnarCompression(shuffleCodec(spark), outputCodec);
+    } else if (outputFileFormat == FileFormat.AVRO) {
+      return rowBasedCompression(shuffleCodec(spark), outputCodec);
+    } else {
+      return 1.0;
+    }
+  }
+
+  private static String shuffleCodec(SparkSession spark) {
+    SparkConf sparkConf = spark.sparkContext().conf();
+    return shuffleCompressionEnabled(sparkConf) ? sparkCodec(sparkConf) : NONE;
+  }
+
+  private static boolean shuffleCompressionEnabled(SparkConf sparkConf) {
+    return sparkConf.getBoolean(SHUFFLE_COMPRESSION_ENABLED, SHUFFLE_COMPRESSION_ENABLED_DEFAULT);
+  }
+
+  private static String sparkCodec(SparkConf sparkConf) {
+    return sparkConf.get(SPARK_COMPRESSION_CODEC, SPARK_COMPRESSION_CODEC_DEFAULT);
+  }
+
+  private static double columnarCompression(String shuffleCodec, String outputCodec) {
+    Pair<String, String> key = Pair.of(normalize(shuffleCodec), normalize(outputCodec));
+    return COLUMNAR_COMPRESSIONS.getOrDefault(key, DEFAULT_COLUMNAR_COMPRESSION);
+  }
+
+  private static double rowBasedCompression(String shuffleCodec, String outputCodec) {
+    Pair<String, String> key = Pair.of(normalize(shuffleCodec), normalize(outputCodec));
+    return ROW_BASED_COMPRESSIONS.getOrDefault(key, DEFAULT_ROW_BASED_COMPRESSION);
+  }
+
+  private static String normalize(String value) {
+    return value != null ? value.toLowerCase(Locale.ROOT) : null;
+  }
+
+  private static Map<Pair<String, String>, Double> initColumnarCompressions() {
+    Map<Pair<String, String>, Double> compressions = Maps.newHashMap();
+
+    compressions.put(Pair.of(NONE, ZSTD), 4.0);
+    compressions.put(Pair.of(NONE, GZIP), 4.0);
+    compressions.put(Pair.of(NONE, ZLIB), 4.0);
+    compressions.put(Pair.of(NONE, SNAPPY), 3.0);
+    compressions.put(Pair.of(NONE, LZ4), 3.0);
+
+    compressions.put(Pair.of(ZSTD, ZSTD), 2.0);
+    compressions.put(Pair.of(ZSTD, GZIP), 2.0);
+    compressions.put(Pair.of(ZSTD, ZLIB), 2.0);
+    compressions.put(Pair.of(ZSTD, SNAPPY), 1.5);
+    compressions.put(Pair.of(ZSTD, LZ4), 1.5);
+
+    compressions.put(Pair.of(SNAPPY, ZSTD), 3.0);
+    compressions.put(Pair.of(SNAPPY, GZIP), 3.0);
+    compressions.put(Pair.of(SNAPPY, ZLIB), 3.0);
+    compressions.put(Pair.of(SNAPPY, SNAPPY), 2.0);
+    compressions.put(Pair.of(SNAPPY, LZ4), 2.);
+
+    compressions.put(Pair.of(LZ4, ZSTD), 3.0);
+    compressions.put(Pair.of(LZ4, GZIP), 3.0);
+    compressions.put(Pair.of(LZ4, ZLIB), 3.0);
+    compressions.put(Pair.of(LZ4, SNAPPY), 2.0);
+    compressions.put(Pair.of(LZ4, LZ4), 2.0);
+
+    return compressions;
+  }
+
+  private static Map<Pair<String, String>, Double> initRowBasedCompressions() {
+    Map<Pair<String, String>, Double> compressions = Maps.newHashMap();
+
+    compressions.put(Pair.of(NONE, ZSTD), 2.0);
+    compressions.put(Pair.of(NONE, GZIP), 2.0);
+    compressions.put(Pair.of(NONE, ZLIB), 2.0);
+
+    compressions.put(Pair.of(ZSTD, SNAPPY), 0.5);
+    compressions.put(Pair.of(ZSTD, LZ4), 0.5);
+
+    compressions.put(Pair.of(SNAPPY, ZSTD), 1.5);
+    compressions.put(Pair.of(SNAPPY, GZIP), 1.5);
+    compressions.put(Pair.of(SNAPPY, ZLIB), 1.5);
+
+    compressions.put(Pair.of(LZ4, ZSTD), 1.5);
+    compressions.put(Pair.of(LZ4, GZIP), 1.5);
+    compressions.put(Pair.of(LZ4, ZLIB), 1.5);
+
+    return compressions;
+  }
+}
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkConfParser.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkConfParser.java
index eaa301524c..d86246b1e6 100644
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkConfParser.java
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkConfParser.java
@@ -106,7 +106,7 @@ class SparkConfParser {
     }
 
     public Integer parseOptional() {
-      return parse(Integer::parseInt, null);
+      return parse(Integer::parseInt, defaultValue);
     }
   }
 
@@ -129,7 +129,7 @@ class SparkConfParser {
     }
 
     public Long parseOptional() {
-      return parse(Long::parseLong, null);
+      return parse(Long::parseLong, defaultValue);
     }
   }
 
@@ -152,7 +152,7 @@ class SparkConfParser {
     }
 
     public String parseOptional() {
-      return parse(Function.identity(), null);
+      return parse(Function.identity(), defaultValue);
     }
   }
 
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java
index 49ab6e4c4e..11d9cc296c 100644
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkSQLProperties.java
@@ -64,4 +64,7 @@ public class SparkSQLProperties {
 
   // Overrides the delete planning mode
   public static final String DELETE_PLANNING_MODE = "spark.sql.iceberg.delete-planning-mode";
+
+  // Overrides the advisory partition size
+  public static final String ADVISORY_PARTITION_SIZE = "spark.sql.iceberg.advisory-partition-size";
 }
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java
index 834e839874..27ed07fcac 100644
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteConf.java
@@ -33,6 +33,7 @@ import static org.apache.iceberg.TableProperties.ORC_COMPRESSION;
 import static org.apache.iceberg.TableProperties.ORC_COMPRESSION_STRATEGY;
 import static org.apache.iceberg.TableProperties.PARQUET_COMPRESSION;
 import static org.apache.iceberg.TableProperties.PARQUET_COMPRESSION_LEVEL;
+import static org.apache.spark.sql.connector.write.RowLevelOperation.Command.DELETE;
 
 import java.util.Locale;
 import java.util.Map;
@@ -75,6 +76,10 @@ public class SparkWriteConf {
 
   private static final Logger LOG = LoggerFactory.getLogger(SparkWriteConf.class);
 
+  private static final long DATA_FILE_SIZE = 128 * 1024 * 1024; // 128 MB
+  private static final long DELETE_FILE_SIZE = 32 * 1024 * 1024; // 32 MB
+
+  private final SparkSession spark;
   private final Table table;
   private final String branch;
   private final RuntimeConfig sessionConf;
@@ -87,6 +92,7 @@ public class SparkWriteConf {
 
   public SparkWriteConf(
       SparkSession spark, Table table, String branch, Map<String, String> writeOptions) {
+    this.spark = spark;
     this.table = table;
     this.branch = branch;
     this.sessionConf = spark.conf();
@@ -163,6 +169,19 @@ public class SparkWriteConf {
     return FileFormat.fromString(valueAsString);
   }
 
+  private String dataCompressionCodec() {
+    switch (dataFileFormat()) {
+      case PARQUET:
+        return parquetCompressionCodec();
+      case AVRO:
+        return avroCompressionCodec();
+      case ORC:
+        return orcCompressionCodec();
+      default:
+        return null;
+    }
+  }
+
   public long targetDataFileSize() {
     return confParser
         .longConf()
@@ -200,6 +219,19 @@ public class SparkWriteConf {
     return valueAsString != null ? FileFormat.fromString(valueAsString) : dataFileFormat();
   }
 
+  private String deleteCompressionCodec() {
+    switch (deleteFileFormat()) {
+      case PARQUET:
+        return deleteParquetCompressionCodec();
+      case AVRO:
+        return deleteAvroCompressionCodec();
+      case ORC:
+        return deleteOrcCompressionCodec();
+      default:
+        return null;
+    }
+  }
+
   public long targetDeleteFileSize() {
     return confParser
         .longConf()
@@ -236,7 +268,8 @@ public class SparkWriteConf {
       return SparkWriteRequirements.EMPTY;
     }
 
-    return SparkWriteUtil.writeRequirements(table, distributionMode(), fanoutWriterEnabled());
+    return SparkWriteUtil.writeRequirements(
+        table, distributionMode(), fanoutWriterEnabled(), dataAdvisoryPartitionSize());
   }
 
   @VisibleForTesting
@@ -284,7 +317,11 @@ public class SparkWriteConf {
     }
 
     return SparkWriteUtil.copyOnWriteRequirements(
-        table, command, copyOnWriteDistributionMode(command), fanoutWriterEnabled());
+        table,
+        command,
+        copyOnWriteDistributionMode(command),
+        fanoutWriterEnabled(),
+        dataAdvisoryPartitionSize());
   }
 
   @VisibleForTesting
@@ -308,7 +345,11 @@ public class SparkWriteConf {
     }
 
     return SparkWriteUtil.positionDeltaRequirements(
-        table, command, positionDeltaDistributionMode(command), fanoutWriterEnabled());
+        table,
+        command,
+        positionDeltaDistributionMode(command),
+        fanoutWriterEnabled(),
+        command == DELETE ? deleteAdvisoryPartitionSize() : dataAdvisoryPartitionSize());
   }
 
   @VisibleForTesting
@@ -484,42 +525,24 @@ public class SparkWriteConf {
 
     switch (deleteFormat) {
       case PARQUET:
-        setWritePropertyWithFallback(
-            writeProperties,
-            DELETE_PARQUET_COMPRESSION,
-            deleteParquetCompressionCodec(),
-            parquetCompressionCodec());
-        setWritePropertyWithFallback(
-            writeProperties,
-            DELETE_PARQUET_COMPRESSION_LEVEL,
-            deleteParquetCompressionLevel(),
-            parquetCompressionLevel());
+        writeProperties.put(DELETE_PARQUET_COMPRESSION, deleteParquetCompressionCodec());
+        String deleteParquetCompressionLevel = deleteParquetCompressionLevel();
+        if (deleteParquetCompressionLevel != null) {
+          writeProperties.put(DELETE_PARQUET_COMPRESSION_LEVEL, deleteParquetCompressionLevel);
+        }
         break;
 
       case AVRO:
-        setWritePropertyWithFallback(
-            writeProperties,
-            DELETE_AVRO_COMPRESSION,
-            deleteAvroCompressionCodec(),
-            avroCompressionCodec());
-        setWritePropertyWithFallback(
-            writeProperties,
-            DELETE_AVRO_COMPRESSION_LEVEL,
-            deleteAvroCompressionLevel(),
-            avroCompressionLevel());
+        writeProperties.put(DELETE_AVRO_COMPRESSION, deleteAvroCompressionCodec());
+        String deleteAvroCompressionLevel = deleteAvroCompressionLevel();
+        if (deleteAvroCompressionLevel != null) {
+          writeProperties.put(DELETE_AVRO_COMPRESSION_LEVEL, deleteAvroCompressionLevel);
+        }
         break;
 
       case ORC:
-        setWritePropertyWithFallback(
-            writeProperties,
-            DELETE_ORC_COMPRESSION,
-            deleteOrcCompressionCodec(),
-            orcCompressionCodec());
-        setWritePropertyWithFallback(
-            writeProperties,
-            DELETE_ORC_COMPRESSION_STRATEGY,
-            deleteOrcCompressionStrategy(),
-            orcCompressionStrategy());
+        writeProperties.put(DELETE_ORC_COMPRESSION, deleteOrcCompressionCodec());
+        writeProperties.put(DELETE_ORC_COMPRESSION_STRATEGY, deleteOrcCompressionStrategy());
         break;
 
       default:
@@ -529,15 +552,6 @@ public class SparkWriteConf {
     return writeProperties;
   }
 
-  private void setWritePropertyWithFallback(
-      Map<String, String> writeProperties, String key, String value, String fallbackValue) {
-    if (value != null) {
-      writeProperties.put(key, value);
-    } else if (fallbackValue != null) {
-      writeProperties.put(key, fallbackValue);
-    }
-  }
-
   private String parquetCompressionCodec() {
     return confParser
         .stringConf()
@@ -554,7 +568,8 @@ public class SparkWriteConf {
         .option(SparkWriteOptions.COMPRESSION_CODEC)
         .sessionConf(SparkSQLProperties.COMPRESSION_CODEC)
         .tableProperty(DELETE_PARQUET_COMPRESSION)
-        .parseOptional();
+        .defaultValue(parquetCompressionCodec())
+        .parse();
   }
 
   private String parquetCompressionLevel() {
@@ -573,6 +588,7 @@ public class SparkWriteConf {
         .option(SparkWriteOptions.COMPRESSION_LEVEL)
         .sessionConf(SparkSQLProperties.COMPRESSION_LEVEL)
         .tableProperty(DELETE_PARQUET_COMPRESSION_LEVEL)
+        .defaultValue(parquetCompressionLevel())
         .parseOptional();
   }
 
@@ -592,7 +608,8 @@ public class SparkWriteConf {
         .option(SparkWriteOptions.COMPRESSION_CODEC)
         .sessionConf(SparkSQLProperties.COMPRESSION_CODEC)
         .tableProperty(DELETE_AVRO_COMPRESSION)
-        .parseOptional();
+        .defaultValue(avroCompressionCodec())
+        .parse();
   }
 
   private String avroCompressionLevel() {
@@ -611,6 +628,7 @@ public class SparkWriteConf {
         .option(SparkWriteOptions.COMPRESSION_LEVEL)
         .sessionConf(SparkSQLProperties.COMPRESSION_LEVEL)
         .tableProperty(DELETE_AVRO_COMPRESSION_LEVEL)
+        .defaultValue(avroCompressionLevel())
         .parseOptional();
   }
 
@@ -630,7 +648,8 @@ public class SparkWriteConf {
         .option(SparkWriteOptions.COMPRESSION_CODEC)
         .sessionConf(SparkSQLProperties.COMPRESSION_CODEC)
         .tableProperty(DELETE_ORC_COMPRESSION)
-        .parseOptional();
+        .defaultValue(orcCompressionCodec())
+        .parse();
   }
 
   private String orcCompressionStrategy() {
@@ -649,6 +668,39 @@ public class SparkWriteConf {
         .option(SparkWriteOptions.COMPRESSION_STRATEGY)
         .sessionConf(SparkSQLProperties.COMPRESSION_STRATEGY)
         .tableProperty(DELETE_ORC_COMPRESSION_STRATEGY)
-        .parseOptional();
+        .defaultValue(orcCompressionStrategy())
+        .parse();
+  }
+
+  private long dataAdvisoryPartitionSize() {
+    long defaultValue =
+        advisoryPartitionSize(DATA_FILE_SIZE, dataFileFormat(), dataCompressionCodec());
+    return advisoryPartitionSize(defaultValue);
+  }
+
+  private long deleteAdvisoryPartitionSize() {
+    long defaultValue =
+        advisoryPartitionSize(DELETE_FILE_SIZE, deleteFileFormat(), deleteCompressionCodec());
+    return advisoryPartitionSize(defaultValue);
+  }
+
+  private long advisoryPartitionSize(long defaultValue) {
+    return confParser
+        .longConf()
+        .option(SparkWriteOptions.ADVISORY_PARTITION_SIZE)
+        .sessionConf(SparkSQLProperties.ADVISORY_PARTITION_SIZE)
+        .tableProperty(TableProperties.SPARK_WRITE_ADVISORY_PARTITION_SIZE_BYTES)
+        .defaultValue(defaultValue)
+        .parse();
+  }
+
+  private long advisoryPartitionSize(
+      long expectedFileSize, FileFormat outputFileFormat, String outputCodec) {
+    double shuffleCompressionRatio = shuffleCompressionRatio(outputFileFormat, outputCodec);
+    return (long) (expectedFileSize * shuffleCompressionRatio);
+  }
+
+  private double shuffleCompressionRatio(FileFormat outputFileFormat, String outputCodec) {
+    return SparkCompressionUtil.shuffleCompressionRatio(spark, outputFileFormat, outputCodec);
   }
 }
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
index 4e5e420643..48dfa44c91 100644
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteOptions.java
@@ -80,4 +80,7 @@ public class SparkWriteOptions {
   public static final String COMPRESSION_CODEC = "compression-codec";
   public static final String COMPRESSION_LEVEL = "compression-level";
   public static final String COMPRESSION_STRATEGY = "compression-strategy";
+
+  // Overrides the advisory partition size
+  public static final String ADVISORY_PARTITION_SIZE = "advisory-partition-size";
 }
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteRequirements.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteRequirements.java
index bfdc2fe8f8..833e0e44e3 100644
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteRequirements.java
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteRequirements.java
@@ -20,20 +20,24 @@ package org.apache.iceberg.spark;
 
 import org.apache.spark.sql.connector.distributions.Distribution;
 import org.apache.spark.sql.connector.distributions.Distributions;
+import org.apache.spark.sql.connector.distributions.UnspecifiedDistribution;
 import org.apache.spark.sql.connector.expressions.SortOrder;
 
 /** A set of requirements such as distribution and ordering reported to Spark during writes. */
 public class SparkWriteRequirements {
 
   public static final SparkWriteRequirements EMPTY =
-      new SparkWriteRequirements(Distributions.unspecified(), new SortOrder[0]);
+      new SparkWriteRequirements(Distributions.unspecified(), new SortOrder[0], 0);
 
   private final Distribution distribution;
   private final SortOrder[] ordering;
+  private final long advisoryPartitionSize;
 
-  SparkWriteRequirements(Distribution distribution, SortOrder[] ordering) {
+  SparkWriteRequirements(
+      Distribution distribution, SortOrder[] ordering, long advisoryPartitionSize) {
     this.distribution = distribution;
     this.ordering = ordering;
+    this.advisoryPartitionSize = advisoryPartitionSize;
   }
 
   public Distribution distribution() {
@@ -47,4 +51,9 @@ public class SparkWriteRequirements {
   public boolean hasOrdering() {
     return ordering.length != 0;
   }
+
+  public long advisoryPartitionSize() {
+    // Spark prohibits requesting a particular advisory partition size without distribution
+    return distribution instanceof UnspecifiedDistribution ? 0 : advisoryPartitionSize;
+  }
 }
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteUtil.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteUtil.java
index 8f9c39de99..0d68a0d8cd 100644
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteUtil.java
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/SparkWriteUtil.java
@@ -66,11 +66,11 @@ public class SparkWriteUtil {
 
   /** Builds requirements for batch and micro-batch writes such as append or overwrite. */
   public static SparkWriteRequirements writeRequirements(
-      Table table, DistributionMode mode, boolean fanoutEnabled) {
+      Table table, DistributionMode mode, boolean fanoutEnabled, long advisoryPartitionSize) {
 
     Distribution distribution = writeDistribution(table, mode);
     SortOrder[] ordering = writeOrdering(table, fanoutEnabled);
-    return new SparkWriteRequirements(distribution, ordering);
+    return new SparkWriteRequirements(distribution, ordering, advisoryPartitionSize);
   }
 
   private static Distribution writeDistribution(Table table, DistributionMode mode) {
@@ -91,14 +91,18 @@ public class SparkWriteUtil {
 
   /** Builds requirements for copy-on-write DELETE, UPDATE, MERGE operations. */
   public static SparkWriteRequirements copyOnWriteRequirements(
-      Table table, Command command, DistributionMode mode, boolean fanoutEnabled) {
+      Table table,
+      Command command,
+      DistributionMode mode,
+      boolean fanoutEnabled,
+      long advisoryPartitionSize) {
 
     if (command == DELETE || command == UPDATE) {
       Distribution distribution = copyOnWriteDeleteUpdateDistribution(table, mode);
       SortOrder[] ordering = writeOrdering(table, fanoutEnabled);
-      return new SparkWriteRequirements(distribution, ordering);
+      return new SparkWriteRequirements(distribution, ordering, advisoryPartitionSize);
     } else {
-      return writeRequirements(table, mode, fanoutEnabled);
+      return writeRequirements(table, mode, fanoutEnabled, advisoryPartitionSize);
     }
   }
 
@@ -130,16 +134,20 @@ public class SparkWriteUtil {
 
   /** Builds requirements for merge-on-read DELETE, UPDATE, MERGE operations. */
   public static SparkWriteRequirements positionDeltaRequirements(
-      Table table, Command command, DistributionMode mode, boolean fanoutEnabled) {
+      Table table,
+      Command command,
+      DistributionMode mode,
+      boolean fanoutEnabled,
+      long advisoryPartitionSize) {
 
     if (command == UPDATE || command == MERGE) {
       Distribution distribution = positionDeltaUpdateMergeDistribution(table, mode);
       SortOrder[] ordering = positionDeltaUpdateMergeOrdering(table, fanoutEnabled);
-      return new SparkWriteRequirements(distribution, ordering);
+      return new SparkWriteRequirements(distribution, ordering, advisoryPartitionSize);
     } else {
       Distribution distribution = positionDeltaDeleteDistribution(table, mode);
       SortOrder[] ordering = fanoutEnabled ? EMPTY_ORDERING : POSITION_DELETE_ORDERING;
-      return new SparkWriteRequirements(distribution, ordering);
+      return new SparkWriteRequirements(distribution, ordering, advisoryPartitionSize);
     }
   }
 
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeltaWrite.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeltaWrite.java
index a8145c2aba..6c0fc591ff 100644
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeltaWrite.java
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeltaWrite.java
@@ -132,7 +132,9 @@ class SparkPositionDeltaWrite implements DeltaWrite, RequiresDistributionAndOrde
 
   @Override
   public Distribution requiredDistribution() {
-    return writeRequirements.distribution();
+    Distribution distribution = writeRequirements.distribution();
+    LOG.info("Requesting {} as write distribution for table {}", distribution, table.name());
+    return distribution;
   }
 
   @Override
@@ -142,7 +144,16 @@ class SparkPositionDeltaWrite implements DeltaWrite, RequiresDistributionAndOrde
 
   @Override
   public SortOrder[] requiredOrdering() {
-    return writeRequirements.ordering();
+    SortOrder[] ordering = writeRequirements.ordering();
+    LOG.info("Requesting {} as write ordering for table {}", ordering, table.name());
+    return ordering;
+  }
+
+  @Override
+  public long advisoryPartitionSizeInBytes() {
+    long size = writeRequirements.advisoryPartitionSize();
+    LOG.info("Requesting {} bytes advisory partition size for table {}", size, table.name());
+    return size;
   }
 
   @Override
diff --git a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
index 802c789ce8..97359d0a3a 100644
--- a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
+++ b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
@@ -134,7 +134,9 @@ abstract class SparkWrite implements Write, RequiresDistributionAndOrdering {
 
   @Override
   public Distribution requiredDistribution() {
-    return writeRequirements.distribution();
+    Distribution distribution = writeRequirements.distribution();
+    LOG.info("Requesting {} as write distribution for table {}", distribution, table.name());
+    return distribution;
   }
 
   @Override
@@ -144,7 +146,16 @@ abstract class SparkWrite implements Write, RequiresDistributionAndOrdering {
 
   @Override
   public SortOrder[] requiredOrdering() {
-    return writeRequirements.ordering();
+    SortOrder[] ordering = writeRequirements.ordering();
+    LOG.info("Requesting {} as write ordering for table {}", ordering, table.name());
+    return ordering;
+  }
+
+  @Override
+  public long advisoryPartitionSizeInBytes() {
+    long size = writeRequirements.advisoryPartitionSize();
+    LOG.info("Requesting {} bytes advisory partition size for table {}", size, table.name());
+    return size;
   }
 
   BatchWrite asBatchAppend() {
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkCompressionUtil.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkCompressionUtil.java
new file mode 100644
index 0000000000..a2b48edffa
--- /dev/null
+++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/TestSparkCompressionUtil.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.spark;
+
+import static org.apache.iceberg.FileFormat.AVRO;
+import static org.apache.iceberg.FileFormat.METADATA;
+import static org.apache.iceberg.FileFormat.ORC;
+import static org.apache.iceberg.FileFormat.PARQUET;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import org.apache.iceberg.FileFormat;
+import org.apache.spark.SparkConf;
+import org.apache.spark.SparkContext;
+import org.apache.spark.internal.config.package$;
+import org.apache.spark.sql.SparkSession;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestSparkCompressionUtil {
+
+  private SparkSession spark;
+  private SparkConf sparkConf;
+
+  @Before
+  public void initSpark() {
+    this.spark = mock(SparkSession.class);
+    this.sparkConf = mock(SparkConf.class);
+
+    SparkContext sparkContext = mock(SparkContext.class);
+
+    when(spark.sparkContext()).thenReturn(sparkContext);
+    when(sparkContext.conf()).thenReturn(sparkConf);
+  }
+
+  @Test
+  public void testParquetCompressionRatios() {
+    configureShuffle("lz4", true);
+
+    double ratio1 = shuffleCompressionRatio(PARQUET, "zstd");
+    assertThat(ratio1).isEqualTo(3.0);
+
+    double ratio2 = shuffleCompressionRatio(PARQUET, "gzip");
+    assertThat(ratio2).isEqualTo(3.0);
+
+    double ratio3 = shuffleCompressionRatio(PARQUET, "snappy");
+    assertThat(ratio3).isEqualTo(2.0);
+  }
+
+  @Test
+  public void testOrcCompressionRatios() {
+    configureShuffle("lz4", true);
+
+    double ratio1 = shuffleCompressionRatio(ORC, "zlib");
+    assertThat(ratio1).isEqualTo(3.0);
+
+    double ratio2 = shuffleCompressionRatio(ORC, "lz4");
+    assertThat(ratio2).isEqualTo(2.0);
+  }
+
+  @Test
+  public void testAvroCompressionRatios() {
+    configureShuffle("lz4", true);
+
+    double ratio1 = shuffleCompressionRatio(AVRO, "gzip");
+    assertThat(ratio1).isEqualTo(1.5);
+
+    double ratio2 = shuffleCompressionRatio(AVRO, "zstd");
+    assertThat(ratio2).isEqualTo(1.5);
+  }
+
+  @Test
+  public void testCodecNameNormalization() {
+    configureShuffle("zStD", true);
+    double ratio = shuffleCompressionRatio(PARQUET, "ZstD");
+    assertThat(ratio).isEqualTo(2.0);
+  }
+
+  @Test
+  public void testUnknownCodecNames() {
+    configureShuffle("SOME_SPARK_CODEC", true);
+
+    double ratio1 = shuffleCompressionRatio(PARQUET, "SOME_PARQUET_CODEC");
+    assertThat(ratio1).isEqualTo(2.0);
+
+    double ratio2 = shuffleCompressionRatio(ORC, "SOME_ORC_CODEC");
+    assertThat(ratio2).isEqualTo(2.0);
+
+    double ratio3 = shuffleCompressionRatio(AVRO, "SOME_AVRO_CODEC");
+    assertThat(ratio3).isEqualTo(1.0);
+  }
+
+  @Test
+  public void testOtherFileFormats() {
+    configureShuffle("lz4", true);
+    double ratio = shuffleCompressionRatio(METADATA, "zstd");
+    assertThat(ratio).isEqualTo(1.0);
+  }
+
+  @Test
+  public void testNullFileCodec() {
+    configureShuffle("lz4", true);
+
+    double ratio1 = shuffleCompressionRatio(PARQUET, null);
+    assertThat(ratio1).isEqualTo(2.0);
+
+    double ratio2 = shuffleCompressionRatio(ORC, null);
+    assertThat(ratio2).isEqualTo(2.0);
+
+    double ratio3 = shuffleCompressionRatio(AVRO, null);
+    assertThat(ratio3).isEqualTo(1.0);
+  }
+
+  @Test
+  public void testUncompressedShuffles() {
+    configureShuffle("zstd", false);
+
+    double ratio1 = shuffleCompressionRatio(PARQUET, "zstd");
+    assertThat(ratio1).isEqualTo(4.0);
+
+    double ratio2 = shuffleCompressionRatio(ORC, "zlib");
+    assertThat(ratio2).isEqualTo(4.0);
+
+    double ratio3 = shuffleCompressionRatio(AVRO, "gzip");
+    assertThat(ratio3).isEqualTo(2.0);
+  }
+
+  @Test
+  public void testSparkDefaults() {
+    assertThat(package$.MODULE$.SHUFFLE_COMPRESS().defaultValueString()).isEqualTo("true");
+    assertThat(package$.MODULE$.IO_COMPRESSION_CODEC().defaultValueString()).isEqualTo("lz4");
+  }
+
+  private void configureShuffle(String codec, boolean compress) {
+    when(sparkConf.getBoolean(eq("spark.shuffle.compress"), anyBoolean())).thenReturn(compress);
+    when(sparkConf.get(eq("spark.io.compression.codec"), anyString())).thenReturn(codec);
+  }
+
+  private double shuffleCompressionRatio(FileFormat fileFormat, String codec) {
+    return SparkCompressionUtil.shuffleCompressionRatio(spark, fileFormat, codec);
+  }
+}