You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by si...@apache.org on 2026/09/15 15:42:53 UTC
(iceberg) branch main updated: Flink: Backport view-incapable catalog handling to 2.2, 2.1 and 1.20 (#18128)
This is an automated email from the ASF dual-hosted git repository.
singhpk234 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 522763940b Flink: Backport view-incapable catalog handling to 2.2, 2.1 and 1.20 (#18128)
522763940b is described below
commit 522763940bec734b118206c8f6540aaebac5bd83
Author: Talat UYARER <ta...@apache.org>
AuthorDate: Tue Sep 15 08:42:47 2026 -0700
Flink: Backport view-incapable catalog handling to 2.2, 2.1 and 1.20 (#18128)
Backport view-incapable catalog handling to 2.2, 2.1 and 1.20
---
.../org/apache/iceberg/flink/FlinkCatalog.java | 34 +++++++-
.../TestFlinkCatalogViewIncapableCatalog.java | 90 ++++++++++++++++++++++
.../org/apache/iceberg/flink/FlinkCatalog.java | 34 +++++++-
.../TestFlinkCatalogViewIncapableCatalog.java | 90 ++++++++++++++++++++++
.../org/apache/iceberg/flink/FlinkCatalog.java | 34 +++++++-
.../TestFlinkCatalogViewIncapableCatalog.java | 90 ++++++++++++++++++++++
6 files changed, 360 insertions(+), 12 deletions(-)
diff --git a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java
index e88da598bf..6fcc688be8 100644
--- a/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java
+++ b/flink/v1.20/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java
@@ -88,6 +88,8 @@ import org.apache.iceberg.view.SQLViewRepresentation;
import org.apache.iceberg.view.View;
import org.apache.iceberg.view.ViewProperties;
import org.apache.iceberg.view.ViewVersion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A Flink Catalog implementation that wraps an Iceberg {@link Catalog}.
@@ -102,6 +104,7 @@ import org.apache.iceberg.view.ViewVersion;
*/
@Internal
public class FlinkCatalog extends AbstractCatalog {
+ private static final Logger LOG = LoggerFactory.getLogger(FlinkCatalog.class);
private static final String FLINK_DIALECT = "flink";
private final CatalogLoader catalogLoader;
@@ -366,6 +369,10 @@ public class FlinkCatalog extends AbstractCatalog {
.toList();
} catch (NoSuchNamespaceException e) {
throw new DatabaseNotExistException(getName(), databaseName, e);
+ } catch (UnsupportedOperationException e) {
+ // the catalog rejects view operations at runtime, e.g. a JDBC catalog with a V0 schema
+ LOG.warn("Catalog {} rejects view operations; assuming no view support", getName(), e);
+ return Collections.emptyList();
}
}
@@ -380,13 +387,19 @@ public class FlinkCatalog extends AbstractCatalog {
throw e;
}
+ View view;
try {
- View view = asViewCatalog.loadView(toIdentifier(tablePath));
- return toCatalogView(tablePath, view);
+ view = asViewCatalog.loadView(toIdentifier(tablePath));
} catch (NoSuchViewException viewException) {
e.addSuppressed(viewException);
throw e;
+ } catch (UnsupportedOperationException viewException) {
+ // the catalog rejects view operations at runtime, e.g. a JDBC catalog with a V0 schema
+ e.addSuppressed(viewException);
+ throw e;
}
+
+ return toCatalogView(tablePath, view);
}
// Flink's CREATE TABLE LIKE clause relies on properties sent back here to create new table.
@@ -433,8 +446,21 @@ public class FlinkCatalog extends AbstractCatalog {
@Override
public boolean tableExists(ObjectPath tablePath) throws CatalogException {
TableIdentifier identifier = toIdentifier(tablePath);
- return icebergCatalog.tableExists(identifier)
- || (canBeView(tablePath) && asViewCatalog.viewExists(identifier));
+ if (icebergCatalog.tableExists(identifier)) {
+ return true;
+ }
+
+ if (!canBeView(tablePath)) {
+ return false;
+ }
+
+ try {
+ return asViewCatalog.viewExists(identifier);
+ } catch (UnsupportedOperationException e) {
+ // the catalog rejects view operations at runtime, e.g. a JDBC catalog with a V0 schema
+ LOG.warn("Catalog {} rejects view operations; assuming no view support", getName(), e);
+ return false;
+ }
}
@Override
diff --git a/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogViewIncapableCatalog.java b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogViewIncapableCatalog.java
new file mode 100644
index 0000000000..a986959961
--- /dev/null
+++ b/flink/v1.20/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogViewIncapableCatalog.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import org.apache.flink.table.api.ValidationException;
+import org.apache.flink.table.catalog.Catalog;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.types.Row;
+import org.apache.iceberg.catalog.ViewCatalog;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * A catalog can implement {@link ViewCatalog} but still reject view operations at runtime: a JDBC
+ * catalog with the V0 schema throws {@link UnsupportedOperationException} from every view method.
+ * Table operations through such a catalog must keep working as if views were not supported.
+ */
+class TestFlinkCatalogViewIncapableCatalog extends TestBase {
+
+ private static final String CATALOG_NAME = "jdbc_v0";
+ private static final String DATABASE = "db_v0";
+ private static final String TABLE_NAME = "t";
+
+ @BeforeEach
+ void createCatalogAndTable() {
+ sql(
+ "CREATE CATALOG %s WITH ("
+ + "'type'='iceberg', "
+ + "'catalog-impl'='org.apache.iceberg.jdbc.JdbcCatalog', "
+ + "'uri'='jdbc:sqlite:%s/catalog.db', "
+ + "'warehouse'='file://%s/warehouse')",
+ CATALOG_NAME, temporaryDirectory, temporaryDirectory);
+ sql("USE CATALOG %s", CATALOG_NAME);
+ sql("CREATE DATABASE %s", DATABASE);
+ sql("USE %s", DATABASE);
+ sql("CREATE TABLE %s (id BIGINT)", TABLE_NAME);
+ sql("INSERT INTO %s VALUES (1)", TABLE_NAME);
+ }
+
+ @AfterEach
+ void cleanCatalog() {
+ dropCatalog(CATALOG_NAME, true);
+ }
+
+ @Test
+ void listViews() {
+ assertThat(sql("SHOW VIEWS")).isEmpty();
+ }
+
+ @Test
+ void listTables() {
+ assertThat(sql("SHOW TABLES")).containsExactly(Row.of(TABLE_NAME));
+ }
+
+ @Test
+ void tableExists() {
+ Catalog flinkCatalog = getTableEnv().getCatalog(CATALOG_NAME).get();
+ assertThat(flinkCatalog.tableExists(new ObjectPath(DATABASE, TABLE_NAME))).isTrue();
+ assertThat(flinkCatalog.tableExists(new ObjectPath(DATABASE, "missing"))).isFalse();
+ }
+
+ @Test
+ void getTable() {
+ assertSameElements(Lists.newArrayList(Row.of(1L)), sql("SELECT * FROM %s", TABLE_NAME));
+ assertThatThrownBy(() -> sql("SELECT * FROM missing"))
+ .isInstanceOf(ValidationException.class)
+ .hasMessageContaining("Object 'missing' not found");
+ }
+}
diff --git a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java
index e88da598bf..6fcc688be8 100644
--- a/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java
+++ b/flink/v2.1/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java
@@ -88,6 +88,8 @@ import org.apache.iceberg.view.SQLViewRepresentation;
import org.apache.iceberg.view.View;
import org.apache.iceberg.view.ViewProperties;
import org.apache.iceberg.view.ViewVersion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A Flink Catalog implementation that wraps an Iceberg {@link Catalog}.
@@ -102,6 +104,7 @@ import org.apache.iceberg.view.ViewVersion;
*/
@Internal
public class FlinkCatalog extends AbstractCatalog {
+ private static final Logger LOG = LoggerFactory.getLogger(FlinkCatalog.class);
private static final String FLINK_DIALECT = "flink";
private final CatalogLoader catalogLoader;
@@ -366,6 +369,10 @@ public class FlinkCatalog extends AbstractCatalog {
.toList();
} catch (NoSuchNamespaceException e) {
throw new DatabaseNotExistException(getName(), databaseName, e);
+ } catch (UnsupportedOperationException e) {
+ // the catalog rejects view operations at runtime, e.g. a JDBC catalog with a V0 schema
+ LOG.warn("Catalog {} rejects view operations; assuming no view support", getName(), e);
+ return Collections.emptyList();
}
}
@@ -380,13 +387,19 @@ public class FlinkCatalog extends AbstractCatalog {
throw e;
}
+ View view;
try {
- View view = asViewCatalog.loadView(toIdentifier(tablePath));
- return toCatalogView(tablePath, view);
+ view = asViewCatalog.loadView(toIdentifier(tablePath));
} catch (NoSuchViewException viewException) {
e.addSuppressed(viewException);
throw e;
+ } catch (UnsupportedOperationException viewException) {
+ // the catalog rejects view operations at runtime, e.g. a JDBC catalog with a V0 schema
+ e.addSuppressed(viewException);
+ throw e;
}
+
+ return toCatalogView(tablePath, view);
}
// Flink's CREATE TABLE LIKE clause relies on properties sent back here to create new table.
@@ -433,8 +446,21 @@ public class FlinkCatalog extends AbstractCatalog {
@Override
public boolean tableExists(ObjectPath tablePath) throws CatalogException {
TableIdentifier identifier = toIdentifier(tablePath);
- return icebergCatalog.tableExists(identifier)
- || (canBeView(tablePath) && asViewCatalog.viewExists(identifier));
+ if (icebergCatalog.tableExists(identifier)) {
+ return true;
+ }
+
+ if (!canBeView(tablePath)) {
+ return false;
+ }
+
+ try {
+ return asViewCatalog.viewExists(identifier);
+ } catch (UnsupportedOperationException e) {
+ // the catalog rejects view operations at runtime, e.g. a JDBC catalog with a V0 schema
+ LOG.warn("Catalog {} rejects view operations; assuming no view support", getName(), e);
+ return false;
+ }
}
@Override
diff --git a/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogViewIncapableCatalog.java b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogViewIncapableCatalog.java
new file mode 100644
index 0000000000..a986959961
--- /dev/null
+++ b/flink/v2.1/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogViewIncapableCatalog.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import org.apache.flink.table.api.ValidationException;
+import org.apache.flink.table.catalog.Catalog;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.types.Row;
+import org.apache.iceberg.catalog.ViewCatalog;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * A catalog can implement {@link ViewCatalog} but still reject view operations at runtime: a JDBC
+ * catalog with the V0 schema throws {@link UnsupportedOperationException} from every view method.
+ * Table operations through such a catalog must keep working as if views were not supported.
+ */
+class TestFlinkCatalogViewIncapableCatalog extends TestBase {
+
+ private static final String CATALOG_NAME = "jdbc_v0";
+ private static final String DATABASE = "db_v0";
+ private static final String TABLE_NAME = "t";
+
+ @BeforeEach
+ void createCatalogAndTable() {
+ sql(
+ "CREATE CATALOG %s WITH ("
+ + "'type'='iceberg', "
+ + "'catalog-impl'='org.apache.iceberg.jdbc.JdbcCatalog', "
+ + "'uri'='jdbc:sqlite:%s/catalog.db', "
+ + "'warehouse'='file://%s/warehouse')",
+ CATALOG_NAME, temporaryDirectory, temporaryDirectory);
+ sql("USE CATALOG %s", CATALOG_NAME);
+ sql("CREATE DATABASE %s", DATABASE);
+ sql("USE %s", DATABASE);
+ sql("CREATE TABLE %s (id BIGINT)", TABLE_NAME);
+ sql("INSERT INTO %s VALUES (1)", TABLE_NAME);
+ }
+
+ @AfterEach
+ void cleanCatalog() {
+ dropCatalog(CATALOG_NAME, true);
+ }
+
+ @Test
+ void listViews() {
+ assertThat(sql("SHOW VIEWS")).isEmpty();
+ }
+
+ @Test
+ void listTables() {
+ assertThat(sql("SHOW TABLES")).containsExactly(Row.of(TABLE_NAME));
+ }
+
+ @Test
+ void tableExists() {
+ Catalog flinkCatalog = getTableEnv().getCatalog(CATALOG_NAME).get();
+ assertThat(flinkCatalog.tableExists(new ObjectPath(DATABASE, TABLE_NAME))).isTrue();
+ assertThat(flinkCatalog.tableExists(new ObjectPath(DATABASE, "missing"))).isFalse();
+ }
+
+ @Test
+ void getTable() {
+ assertSameElements(Lists.newArrayList(Row.of(1L)), sql("SELECT * FROM %s", TABLE_NAME));
+ assertThatThrownBy(() -> sql("SELECT * FROM missing"))
+ .isInstanceOf(ValidationException.class)
+ .hasMessageContaining("Object 'missing' not found");
+ }
+}
diff --git a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java
index e88da598bf..6fcc688be8 100644
--- a/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java
+++ b/flink/v2.2/flink/src/main/java/org/apache/iceberg/flink/FlinkCatalog.java
@@ -88,6 +88,8 @@ import org.apache.iceberg.view.SQLViewRepresentation;
import org.apache.iceberg.view.View;
import org.apache.iceberg.view.ViewProperties;
import org.apache.iceberg.view.ViewVersion;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* A Flink Catalog implementation that wraps an Iceberg {@link Catalog}.
@@ -102,6 +104,7 @@ import org.apache.iceberg.view.ViewVersion;
*/
@Internal
public class FlinkCatalog extends AbstractCatalog {
+ private static final Logger LOG = LoggerFactory.getLogger(FlinkCatalog.class);
private static final String FLINK_DIALECT = "flink";
private final CatalogLoader catalogLoader;
@@ -366,6 +369,10 @@ public class FlinkCatalog extends AbstractCatalog {
.toList();
} catch (NoSuchNamespaceException e) {
throw new DatabaseNotExistException(getName(), databaseName, e);
+ } catch (UnsupportedOperationException e) {
+ // the catalog rejects view operations at runtime, e.g. a JDBC catalog with a V0 schema
+ LOG.warn("Catalog {} rejects view operations; assuming no view support", getName(), e);
+ return Collections.emptyList();
}
}
@@ -380,13 +387,19 @@ public class FlinkCatalog extends AbstractCatalog {
throw e;
}
+ View view;
try {
- View view = asViewCatalog.loadView(toIdentifier(tablePath));
- return toCatalogView(tablePath, view);
+ view = asViewCatalog.loadView(toIdentifier(tablePath));
} catch (NoSuchViewException viewException) {
e.addSuppressed(viewException);
throw e;
+ } catch (UnsupportedOperationException viewException) {
+ // the catalog rejects view operations at runtime, e.g. a JDBC catalog with a V0 schema
+ e.addSuppressed(viewException);
+ throw e;
}
+
+ return toCatalogView(tablePath, view);
}
// Flink's CREATE TABLE LIKE clause relies on properties sent back here to create new table.
@@ -433,8 +446,21 @@ public class FlinkCatalog extends AbstractCatalog {
@Override
public boolean tableExists(ObjectPath tablePath) throws CatalogException {
TableIdentifier identifier = toIdentifier(tablePath);
- return icebergCatalog.tableExists(identifier)
- || (canBeView(tablePath) && asViewCatalog.viewExists(identifier));
+ if (icebergCatalog.tableExists(identifier)) {
+ return true;
+ }
+
+ if (!canBeView(tablePath)) {
+ return false;
+ }
+
+ try {
+ return asViewCatalog.viewExists(identifier);
+ } catch (UnsupportedOperationException e) {
+ // the catalog rejects view operations at runtime, e.g. a JDBC catalog with a V0 schema
+ LOG.warn("Catalog {} rejects view operations; assuming no view support", getName(), e);
+ return false;
+ }
}
@Override
diff --git a/flink/v2.2/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogViewIncapableCatalog.java b/flink/v2.2/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogViewIncapableCatalog.java
new file mode 100644
index 0000000000..a986959961
--- /dev/null
+++ b/flink/v2.2/flink/src/test/java/org/apache/iceberg/flink/TestFlinkCatalogViewIncapableCatalog.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.flink;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
+
+import org.apache.flink.table.api.ValidationException;
+import org.apache.flink.table.catalog.Catalog;
+import org.apache.flink.table.catalog.ObjectPath;
+import org.apache.flink.types.Row;
+import org.apache.iceberg.catalog.ViewCatalog;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * A catalog can implement {@link ViewCatalog} but still reject view operations at runtime: a JDBC
+ * catalog with the V0 schema throws {@link UnsupportedOperationException} from every view method.
+ * Table operations through such a catalog must keep working as if views were not supported.
+ */
+class TestFlinkCatalogViewIncapableCatalog extends TestBase {
+
+ private static final String CATALOG_NAME = "jdbc_v0";
+ private static final String DATABASE = "db_v0";
+ private static final String TABLE_NAME = "t";
+
+ @BeforeEach
+ void createCatalogAndTable() {
+ sql(
+ "CREATE CATALOG %s WITH ("
+ + "'type'='iceberg', "
+ + "'catalog-impl'='org.apache.iceberg.jdbc.JdbcCatalog', "
+ + "'uri'='jdbc:sqlite:%s/catalog.db', "
+ + "'warehouse'='file://%s/warehouse')",
+ CATALOG_NAME, temporaryDirectory, temporaryDirectory);
+ sql("USE CATALOG %s", CATALOG_NAME);
+ sql("CREATE DATABASE %s", DATABASE);
+ sql("USE %s", DATABASE);
+ sql("CREATE TABLE %s (id BIGINT)", TABLE_NAME);
+ sql("INSERT INTO %s VALUES (1)", TABLE_NAME);
+ }
+
+ @AfterEach
+ void cleanCatalog() {
+ dropCatalog(CATALOG_NAME, true);
+ }
+
+ @Test
+ void listViews() {
+ assertThat(sql("SHOW VIEWS")).isEmpty();
+ }
+
+ @Test
+ void listTables() {
+ assertThat(sql("SHOW TABLES")).containsExactly(Row.of(TABLE_NAME));
+ }
+
+ @Test
+ void tableExists() {
+ Catalog flinkCatalog = getTableEnv().getCatalog(CATALOG_NAME).get();
+ assertThat(flinkCatalog.tableExists(new ObjectPath(DATABASE, TABLE_NAME))).isTrue();
+ assertThat(flinkCatalog.tableExists(new ObjectPath(DATABASE, "missing"))).isFalse();
+ }
+
+ @Test
+ void getTable() {
+ assertSameElements(Lists.newArrayList(Row.of(1L)), sql("SELECT * FROM %s", TABLE_NAME));
+ assertThatThrownBy(() -> sql("SELECT * FROM missing"))
+ .isInstanceOf(ValidationException.class)
+ .hasMessageContaining("Object 'missing' not found");
+ }
+}