You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by mx...@apache.org on 2026/08/10 10:57:17 UTC

(iceberg) branch main updated: Flink: Reuse the catalog in the dynamic sink serializer cache (#17437)

This is an automated email from the ASF dual-hosted git repository.

mxm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/main by this push:
     new 6105d39d9b Flink: Reuse the catalog in the dynamic sink serializer cache (#17437)
6105d39d9b is described below

commit 6105d39d9bf1f80081d753f63823df66e4f77b71
Author: vishnu prakash <nv...@gmail.com>
AuthorDate: Mon Aug 10 16:27:10 2026 +0530

    Flink: Reuse the catalog in the dynamic sink serializer cache (#17437)
---
 .../flink/sink/dynamic/TableSerializerCache.java   | 10 ++++-
 .../sink/dynamic/TestTableSerializerCache.java     | 43 ++++++++++++++++++++++
 2 files changed, 52 insertions(+), 1 deletion(-)

diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/sink/dynamic/TableSerializerCache.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/sink/dynamic/TableSerializerCache.java
index 84d0ed9be5..13b4fa56c9 100644
--- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/sink/dynamic/TableSerializerCache.java
+++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/sink/dynamic/TableSerializerCache.java
@@ -28,6 +28,7 @@ import org.apache.flink.table.runtime.typeutils.RowDataSerializer;
 import org.apache.iceberg.PartitionSpec;
 import org.apache.iceberg.Schema;
 import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
 import org.apache.iceberg.catalog.TableIdentifier;
 import org.apache.iceberg.flink.CatalogLoader;
 import org.apache.iceberg.flink.FlinkSchemaUtil;
@@ -50,6 +51,9 @@ class TableSerializerCache implements Serializable {
 
   private final CatalogLoader catalogLoader;
   private final int maximumSize;
+
+  // Intentionally not closed; the catalog is reused for the serializer's lifetime.
+  private transient Catalog catalog;
   private transient Map<String, SerializerInfo> serializers;
 
   TableSerializerCache(CatalogLoader catalogLoader, int maximumSize) {
@@ -120,7 +124,11 @@ class TableSerializerCache implements Serializable {
     }
 
     private void update() {
-      Table table = catalogLoader.loadCatalog().loadTable(TableIdentifier.parse(tableName));
+      if (catalog == null) {
+        catalog = catalogLoader.loadCatalog();
+      }
+
+      Table table = catalog.loadTable(TableIdentifier.parse(tableName));
       schemas = table.schemas();
       specs = table.specs();
     }
diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/sink/dynamic/TestTableSerializerCache.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/sink/dynamic/TestTableSerializerCache.java
index 1cf2c8bae0..bbfb39618f 100644
--- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/sink/dynamic/TestTableSerializerCache.java
+++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/sink/dynamic/TestTableSerializerCache.java
@@ -121,4 +121,47 @@ class TestTableSerializerCache {
     cache = new TableSerializerCache(CATALOG_EXTENSION.catalogLoader(), 1000);
     assertThat(cache.maximumSize()).isEqualTo(1000);
   }
+
+  @Test
+  void testReusesCatalogAcrossLookups() {
+    Table table = CATALOG_EXTENSION.catalog().createTable(TableIdentifier.of("table"), schema1);
+    Table table2 = CATALOG_EXTENSION.catalog().createTable(TableIdentifier.of("table2"), schema2);
+
+    LoadCountingCatalogLoader catalogLoader =
+        new LoadCountingCatalogLoader(CATALOG_EXTENSION.catalogLoader());
+    cache = new TableSerializerCache(catalogLoader, 10);
+
+    // schema/spec ids are unknown, so both lookups miss the cache and need the catalog
+    cache.serializerWithSchemaAndSpec(
+        "table", table.schema().schemaId(), PartitionSpec.unpartitioned().specId());
+    cache.serializerWithSchemaAndSpec(
+        "table2", table2.schema().schemaId(), PartitionSpec.unpartitioned().specId());
+
+    assertThat(catalogLoader.loadCount()).isEqualTo(1);
+  }
+
+  private static class LoadCountingCatalogLoader implements CatalogLoader {
+    private final CatalogLoader delegate;
+    private int loadCount = 0;
+
+    private LoadCountingCatalogLoader(CatalogLoader delegate) {
+      this.delegate = delegate;
+    }
+
+    private int loadCount() {
+      return loadCount;
+    }
+
+    @Override
+    public Catalog loadCatalog() {
+      loadCount += 1;
+      return delegate.loadCatalog();
+    }
+
+    @Override
+    @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
+    public CatalogLoader clone() {
+      return this;
+    }
+  }
 }