You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by pv...@apache.org on 2026/07/06 06:45:03 UTC
(iceberg) branch main updated: Core: Extend org.apache.iceberg.hadoop.Configurable in HadoopConfigurable (#16736)
This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 05c5d77fde Core: Extend org.apache.iceberg.hadoop.Configurable in HadoopConfigurable (#16736)
05c5d77fde is described below
commit 05c5d77fde4117a6b9474b4d1191a3bf4e6a6145
Author: Yuya Ebihara <eb...@gmail.com>
AuthorDate: Mon Jul 6 15:44:57 2026 +0900
Core: Extend org.apache.iceberg.hadoop.Configurable in HadoopConfigurable (#16736)
---
.../apache/iceberg/hadoop/HadoopConfigurable.java | 26 ++++++++++++++++++----
.../source/TestSparkCatalogHadoopOverrides.java | 10 ++++-----
.../source/TestSparkCatalogHadoopOverrides.java | 10 ++++-----
.../source/TestSparkCatalogHadoopOverrides.java | 10 ++++-----
4 files changed, 37 insertions(+), 19 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/hadoop/HadoopConfigurable.java b/core/src/main/java/org/apache/iceberg/hadoop/HadoopConfigurable.java
index d2ca1d9756..6427834eae 100644
--- a/core/src/main/java/org/apache/iceberg/hadoop/HadoopConfigurable.java
+++ b/core/src/main/java/org/apache/iceberg/hadoop/HadoopConfigurable.java
@@ -19,19 +19,18 @@
package org.apache.iceberg.hadoop;
import java.util.function.Function;
-import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.util.SerializableSupplier;
/**
- * An interface that extends the Hadoop {@link Configurable} interface to offer better serialization
- * support for customizable Iceberg objects such as {@link org.apache.iceberg.io.FileIO}.
+ * An interface that extends {@link Configurable} to offer better serialization support for
+ * customizable Iceberg objects such as {@link org.apache.iceberg.io.FileIO}.
*
* <p>If an object is serialized and needs to use Hadoop configuration, it is recommended for the
* object to implement this interface so that a serializable supplier of configuration can be
* provided instead of an actual Hadoop configuration which is not serializable.
*/
-public interface HadoopConfigurable extends Configurable {
+public interface HadoopConfigurable extends Configurable<Configuration> {
/**
* Take a function that serializes Hadoop configuration into a supplier. An implementation is
@@ -43,4 +42,23 @@ public interface HadoopConfigurable extends Configurable {
*/
void serializeConfWith(
Function<Configuration, SerializableSupplier<Configuration>> confSerializer);
+
+ /**
+ * Set the configuration to be used by this object.
+ *
+ * @param conf configuration to be used
+ */
+ @Override
+ default void setConf(Configuration conf) {
+ throw new UnsupportedOperationException("setConf is not implemented");
+ }
+
+ /**
+ * Return the configuration used by this object.
+ *
+ * @return Configuration
+ */
+ default Configuration getConf() {
+ throw new UnsupportedOperationException("getConf is not implemented");
+ }
}
diff --git a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java
index fd155a6bca..a7f467ead2 100644
--- a/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java
+++ b/spark/v3.5/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java
@@ -20,13 +20,13 @@ package org.apache.iceberg.spark.source;
import static org.assertj.core.api.Assertions.assertThat;
-import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.KryoHelpers;
import org.apache.iceberg.ParameterizedTestExtension;
import org.apache.iceberg.Parameters;
import org.apache.iceberg.Table;
import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.hadoop.HadoopConfigurable;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.spark.CatalogTestBase;
import org.apache.iceberg.spark.SparkCatalog;
@@ -93,7 +93,7 @@ public class TestSparkCatalogHadoopOverrides extends CatalogTestBase {
@TestTemplate
public void testTableFromCatalogHasOverrides() throws Exception {
Table table = getIcebergTableFromSparkCatalog();
- Configuration conf = ((Configurable) table.io()).getConf();
+ Configuration conf = ((HadoopConfigurable) table.io()).getConf();
String actualCatalogOverride = conf.get(CONFIG_TO_OVERRIDE, "/whammies");
assertThat(actualCatalogOverride)
.as(
@@ -104,7 +104,7 @@ public class TestSparkCatalogHadoopOverrides extends CatalogTestBase {
@TestTemplate
public void ensureRoundTripSerializedTableRetainsHadoopConfig() throws Exception {
Table table = getIcebergTableFromSparkCatalog();
- Configuration originalConf = ((Configurable) table.io()).getConf();
+ Configuration originalConf = ((HadoopConfigurable) table.io()).getConf();
String actualCatalogOverride = originalConf.get(CONFIG_TO_OVERRIDE, "/whammies");
assertThat(actualCatalogOverride)
.as(
@@ -115,7 +115,7 @@ public class TestSparkCatalogHadoopOverrides extends CatalogTestBase {
Table serializableTable = SerializableTableWithSize.copyOf(table);
Table kryoSerializedTable =
KryoHelpers.roundTripSerialize(SerializableTableWithSize.copyOf(table));
- Configuration configFromKryoSerde = ((Configurable) kryoSerializedTable.io()).getConf();
+ Configuration configFromKryoSerde = ((HadoopConfigurable) kryoSerializedTable.io()).getConf();
String kryoSerializedCatalogOverride = configFromKryoSerde.get(CONFIG_TO_OVERRIDE, "/whammies");
assertThat(kryoSerializedCatalogOverride)
.as(
@@ -124,7 +124,7 @@ public class TestSparkCatalogHadoopOverrides extends CatalogTestBase {
// Do the same for Java based serde
Table javaSerializedTable = TestHelpers.roundTripSerialize(serializableTable);
- Configuration configFromJavaSerde = ((Configurable) javaSerializedTable.io()).getConf();
+ Configuration configFromJavaSerde = ((HadoopConfigurable) javaSerializedTable.io()).getConf();
String javaSerializedCatalogOverride = configFromJavaSerde.get(CONFIG_TO_OVERRIDE, "/whammies");
assertThat(javaSerializedCatalogOverride)
.as(
diff --git a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java
index fd155a6bca..a7f467ead2 100644
--- a/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java
+++ b/spark/v4.0/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java
@@ -20,13 +20,13 @@ package org.apache.iceberg.spark.source;
import static org.assertj.core.api.Assertions.assertThat;
-import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.KryoHelpers;
import org.apache.iceberg.ParameterizedTestExtension;
import org.apache.iceberg.Parameters;
import org.apache.iceberg.Table;
import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.hadoop.HadoopConfigurable;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.spark.CatalogTestBase;
import org.apache.iceberg.spark.SparkCatalog;
@@ -93,7 +93,7 @@ public class TestSparkCatalogHadoopOverrides extends CatalogTestBase {
@TestTemplate
public void testTableFromCatalogHasOverrides() throws Exception {
Table table = getIcebergTableFromSparkCatalog();
- Configuration conf = ((Configurable) table.io()).getConf();
+ Configuration conf = ((HadoopConfigurable) table.io()).getConf();
String actualCatalogOverride = conf.get(CONFIG_TO_OVERRIDE, "/whammies");
assertThat(actualCatalogOverride)
.as(
@@ -104,7 +104,7 @@ public class TestSparkCatalogHadoopOverrides extends CatalogTestBase {
@TestTemplate
public void ensureRoundTripSerializedTableRetainsHadoopConfig() throws Exception {
Table table = getIcebergTableFromSparkCatalog();
- Configuration originalConf = ((Configurable) table.io()).getConf();
+ Configuration originalConf = ((HadoopConfigurable) table.io()).getConf();
String actualCatalogOverride = originalConf.get(CONFIG_TO_OVERRIDE, "/whammies");
assertThat(actualCatalogOverride)
.as(
@@ -115,7 +115,7 @@ public class TestSparkCatalogHadoopOverrides extends CatalogTestBase {
Table serializableTable = SerializableTableWithSize.copyOf(table);
Table kryoSerializedTable =
KryoHelpers.roundTripSerialize(SerializableTableWithSize.copyOf(table));
- Configuration configFromKryoSerde = ((Configurable) kryoSerializedTable.io()).getConf();
+ Configuration configFromKryoSerde = ((HadoopConfigurable) kryoSerializedTable.io()).getConf();
String kryoSerializedCatalogOverride = configFromKryoSerde.get(CONFIG_TO_OVERRIDE, "/whammies");
assertThat(kryoSerializedCatalogOverride)
.as(
@@ -124,7 +124,7 @@ public class TestSparkCatalogHadoopOverrides extends CatalogTestBase {
// Do the same for Java based serde
Table javaSerializedTable = TestHelpers.roundTripSerialize(serializableTable);
- Configuration configFromJavaSerde = ((Configurable) javaSerializedTable.io()).getConf();
+ Configuration configFromJavaSerde = ((HadoopConfigurable) javaSerializedTable.io()).getConf();
String javaSerializedCatalogOverride = configFromJavaSerde.get(CONFIG_TO_OVERRIDE, "/whammies");
assertThat(javaSerializedCatalogOverride)
.as(
diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java
index fd155a6bca..a7f467ead2 100644
--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java
+++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkCatalogHadoopOverrides.java
@@ -20,13 +20,13 @@ package org.apache.iceberg.spark.source;
import static org.assertj.core.api.Assertions.assertThat;
-import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.KryoHelpers;
import org.apache.iceberg.ParameterizedTestExtension;
import org.apache.iceberg.Parameters;
import org.apache.iceberg.Table;
import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.hadoop.HadoopConfigurable;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.spark.CatalogTestBase;
import org.apache.iceberg.spark.SparkCatalog;
@@ -93,7 +93,7 @@ public class TestSparkCatalogHadoopOverrides extends CatalogTestBase {
@TestTemplate
public void testTableFromCatalogHasOverrides() throws Exception {
Table table = getIcebergTableFromSparkCatalog();
- Configuration conf = ((Configurable) table.io()).getConf();
+ Configuration conf = ((HadoopConfigurable) table.io()).getConf();
String actualCatalogOverride = conf.get(CONFIG_TO_OVERRIDE, "/whammies");
assertThat(actualCatalogOverride)
.as(
@@ -104,7 +104,7 @@ public class TestSparkCatalogHadoopOverrides extends CatalogTestBase {
@TestTemplate
public void ensureRoundTripSerializedTableRetainsHadoopConfig() throws Exception {
Table table = getIcebergTableFromSparkCatalog();
- Configuration originalConf = ((Configurable) table.io()).getConf();
+ Configuration originalConf = ((HadoopConfigurable) table.io()).getConf();
String actualCatalogOverride = originalConf.get(CONFIG_TO_OVERRIDE, "/whammies");
assertThat(actualCatalogOverride)
.as(
@@ -115,7 +115,7 @@ public class TestSparkCatalogHadoopOverrides extends CatalogTestBase {
Table serializableTable = SerializableTableWithSize.copyOf(table);
Table kryoSerializedTable =
KryoHelpers.roundTripSerialize(SerializableTableWithSize.copyOf(table));
- Configuration configFromKryoSerde = ((Configurable) kryoSerializedTable.io()).getConf();
+ Configuration configFromKryoSerde = ((HadoopConfigurable) kryoSerializedTable.io()).getConf();
String kryoSerializedCatalogOverride = configFromKryoSerde.get(CONFIG_TO_OVERRIDE, "/whammies");
assertThat(kryoSerializedCatalogOverride)
.as(
@@ -124,7 +124,7 @@ public class TestSparkCatalogHadoopOverrides extends CatalogTestBase {
// Do the same for Java based serde
Table javaSerializedTable = TestHelpers.roundTripSerialize(serializableTable);
- Configuration configFromJavaSerde = ((Configurable) javaSerializedTable.io()).getConf();
+ Configuration configFromJavaSerde = ((HadoopConfigurable) javaSerializedTable.io()).getConf();
String javaSerializedCatalogOverride = configFromJavaSerde.get(CONFIG_TO_OVERRIDE, "/whammies");
assertThat(javaSerializedCatalogOverride)
.as(