You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by mx...@apache.org on 2026/08/10 10:57:17 UTC
(iceberg) branch main updated: Flink: Reuse the catalog in the dynamic sink serializer cache (#17437)
This is an automated email from the ASF dual-hosted git repository.
mxm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 6105d39d9b Flink: Reuse the catalog in the dynamic sink serializer cache (#17437)
6105d39d9b is described below
commit 6105d39d9bf1f80081d753f63823df66e4f77b71
Author: vishnu prakash <nv...@gmail.com>
AuthorDate: Mon Aug 10 16:27:10 2026 +0530
Flink: Reuse the catalog in the dynamic sink serializer cache (#17437)
---
.../flink/sink/dynamic/TableSerializerCache.java | 10 ++++-
.../sink/dynamic/TestTableSerializerCache.java | 43 ++++++++++++++++++++++
2 files changed, 52 insertions(+), 1 deletion(-)
diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/sink/dynamic/TableSerializerCache.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/sink/dynamic/TableSerializerCache.java
index 84d0ed9be5..13b4fa56c9 100644
--- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/sink/dynamic/TableSerializerCache.java
+++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/sink/dynamic/TableSerializerCache.java
@@ -28,6 +28,7 @@ import org.apache.flink.table.runtime.typeutils.RowDataSerializer;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.flink.CatalogLoader;
import org.apache.iceberg.flink.FlinkSchemaUtil;
@@ -50,6 +51,9 @@ class TableSerializerCache implements Serializable {
private final CatalogLoader catalogLoader;
private final int maximumSize;
+
+ // Intentionally not closed; the catalog is reused for the serializer's lifetime.
+ private transient Catalog catalog;
private transient Map<String, SerializerInfo> serializers;
TableSerializerCache(CatalogLoader catalogLoader, int maximumSize) {
@@ -120,7 +124,11 @@ class TableSerializerCache implements Serializable {
}
private void update() {
- Table table = catalogLoader.loadCatalog().loadTable(TableIdentifier.parse(tableName));
+ if (catalog == null) {
+ catalog = catalogLoader.loadCatalog();
+ }
+
+ Table table = catalog.loadTable(TableIdentifier.parse(tableName));
schemas = table.schemas();
specs = table.specs();
}
diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/sink/dynamic/TestTableSerializerCache.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/sink/dynamic/TestTableSerializerCache.java
index 1cf2c8bae0..bbfb39618f 100644
--- a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/sink/dynamic/TestTableSerializerCache.java
+++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/sink/dynamic/TestTableSerializerCache.java
@@ -121,4 +121,47 @@ class TestTableSerializerCache {
cache = new TableSerializerCache(CATALOG_EXTENSION.catalogLoader(), 1000);
assertThat(cache.maximumSize()).isEqualTo(1000);
}
+
+ @Test
+ void testReusesCatalogAcrossLookups() {
+ Table table = CATALOG_EXTENSION.catalog().createTable(TableIdentifier.of("table"), schema1);
+ Table table2 = CATALOG_EXTENSION.catalog().createTable(TableIdentifier.of("table2"), schema2);
+
+ LoadCountingCatalogLoader catalogLoader =
+ new LoadCountingCatalogLoader(CATALOG_EXTENSION.catalogLoader());
+ cache = new TableSerializerCache(catalogLoader, 10);
+
+ // schema/spec ids are unknown, so both lookups miss the cache and need the catalog
+ cache.serializerWithSchemaAndSpec(
+ "table", table.schema().schemaId(), PartitionSpec.unpartitioned().specId());
+ cache.serializerWithSchemaAndSpec(
+ "table2", table2.schema().schemaId(), PartitionSpec.unpartitioned().specId());
+
+ assertThat(catalogLoader.loadCount()).isEqualTo(1);
+ }
+
+ private static class LoadCountingCatalogLoader implements CatalogLoader {
+ private final CatalogLoader delegate;
+ private int loadCount = 0;
+
+ private LoadCountingCatalogLoader(CatalogLoader delegate) {
+ this.delegate = delegate;
+ }
+
+ private int loadCount() {
+ return loadCount;
+ }
+
+ @Override
+ public Catalog loadCatalog() {
+ loadCount += 1;
+ return delegate.loadCatalog();
+ }
+
+ @Override
+ @SuppressWarnings({"checkstyle:NoClone", "checkstyle:SuperClone"})
+ public CatalogLoader clone() {
+ return this;
+ }
+ }
}