You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2026/04/29 14:25:28 UTC
(paimon) branch master updated: [core] [iceberg] Support syncing Iceberg metadata to multiple Hive databases (#7696)
This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new c40420fdf2 [core] [iceberg] Support syncing Iceberg metadata to multiple Hive databases (#7696)
c40420fdf2 is described below
commit c40420fdf2fb7b6fef301905aec2134b132790b9
Author: Nick Del Nano <nd...@yelp.com>
AuthorDate: Wed Apr 29 07:25:21 2026 -0700
[core] [iceberg] Support syncing Iceberg metadata to multiple Hive databases (#7696)
Issue: https://github.com/apache/paimon/issues/7694
> Paimon's Iceberg compatibility supports a single Iceberg table in the
catalog with metadata.iceberg.database and metadata.iceberg.table
values. I'd like it to support the option of multiple db.table
references so that I can register the Iceberg metadata to both a legacy
db.table value and a new one. This will allow me to migrate my platform
onto this feature without changing existing queries while also using my
new db.table naming standard.
> I don't have a view implementation available in my environment so I
cannot use views to solve it.
Support registering Iceberg metadata in multiple Hive databases by
allowing semicolon-delimited values in metadata.iceberg.database (e.g.
'db1;db2;db3').
---
.../generated/iceberg_configuration.html | 2 +-
.../org/apache/paimon/iceberg/IcebergOptions.java | 18 ++++-
.../apache/paimon/iceberg/IcebergOptionsTest.java | 82 ++++++++++++++++++++++
.../iceberg/IcebergHiveMetadataCommitter.java | 65 +++++++++--------
.../IcebergHiveMetadataCommitterITCaseBase.java | 76 ++++++++++++++++++++
5 files changed, 212 insertions(+), 31 deletions(-)
diff --git a/docs/layouts/shortcodes/generated/iceberg_configuration.html b/docs/layouts/shortcodes/generated/iceberg_configuration.html
index cf57763779..3961343242 100644
--- a/docs/layouts/shortcodes/generated/iceberg_configuration.html
+++ b/docs/layouts/shortcodes/generated/iceberg_configuration.html
@@ -42,7 +42,7 @@ under the License.
<td><h5>metadata.iceberg.database</h5></td>
<td style="word-wrap: break-word;">(none)</td>
<td>String</td>
- <td>Metastore database name for Iceberg Catalog. Set this as an iceberg database alias if using a centralized Catalog.</td>
+ <td>Metastore database name for Iceberg Catalog. Set this as an iceberg database alias if using a centralized Catalog. Multiple databases can be specified with semicolons, e.g. 'db1;db2'. The table will be registered in each database.</td>
</tr>
<tr>
<td><h5>metadata.iceberg.delete-after-commit.enabled</h5></td>
diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergOptions.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergOptions.java
index be4f6152ef..530acd85eb 100644
--- a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergOptions.java
+++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergOptions.java
@@ -28,6 +28,7 @@ import org.apache.paimon.utils.Preconditions;
import java.lang.reflect.Field;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -136,7 +137,9 @@ public class IcebergOptions {
.noDefaultValue()
.withDescription(
"Metastore database name for Iceberg Catalog. "
- + "Set this as an iceberg database alias if using a centralized Catalog.");
+ + "Set this as an iceberg database alias if using a centralized Catalog. "
+ + "Multiple databases can be specified with semicolons, "
+ + "e.g. 'db1;db2'. The table will be registered in each database.");
public static final ConfigOption<String> METASTORE_TABLE =
key("metadata.iceberg.table")
@@ -259,6 +262,19 @@ public class IcebergOptions {
}
}
+ public static List<String> metastoreDatabases(Options options, String fallbackDatabase) {
+ String dbValue = options.get(METASTORE_DATABASE);
+ if (dbValue == null || dbValue.isEmpty()) {
+ return Collections.singletonList(fallbackDatabase);
+ }
+ String[] parts = dbValue.split(";");
+ List<String> databases = new ArrayList<>(parts.length);
+ for (String part : parts) {
+ databases.add(part.trim());
+ }
+ return databases;
+ }
+
/**
* Returns all ConfigOption fields defined in this class. This method uses reflection to
* dynamically discover all ConfigOption fields, ensuring that new options are automatically
diff --git a/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergOptionsTest.java
new file mode 100644
index 0000000000..2d57c3ac0e
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergOptionsTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.iceberg;
+
+import org.apache.paimon.options.Options;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link IcebergOptions#metastoreDatabases}. */
+public class IcebergOptionsTest {
+
+ private static final String FALLBACK_DB = "default_db";
+
+ @Test
+ public void testSingleDatabase() {
+ Options options = new Options();
+ options.set(IcebergOptions.METASTORE_DATABASE, "mydb");
+
+ List<String> databases = IcebergOptions.metastoreDatabases(options, FALLBACK_DB);
+
+ assertThat(databases).containsExactly("mydb");
+ }
+
+ @Test
+ public void testMultipleDatabases() {
+ Options options = new Options();
+ options.set(IcebergOptions.METASTORE_DATABASE, "db1;db2;db3");
+
+ List<String> databases = IcebergOptions.metastoreDatabases(options, FALLBACK_DB);
+
+ assertThat(databases).containsExactly("db1", "db2", "db3");
+ }
+
+ @Test
+ public void testNoDatabaseConfigFallsBack() {
+ Options options = new Options();
+
+ List<String> databases = IcebergOptions.metastoreDatabases(options, FALLBACK_DB);
+
+ assertThat(databases).containsExactly("default_db");
+ }
+
+ @Test
+ public void testEmptyDatabaseStringFallsBack() {
+ Options options = new Options();
+ options.set(IcebergOptions.METASTORE_DATABASE, "");
+
+ List<String> databases = IcebergOptions.metastoreDatabases(options, FALLBACK_DB);
+
+ assertThat(databases).containsExactly("default_db");
+ }
+
+ @Test
+ public void testWhitespaceAroundSemicolon() {
+ Options options = new Options();
+ options.set(IcebergOptions.METASTORE_DATABASE, " db1 ; db2 ; db3 ");
+
+ List<String> databases = IcebergOptions.metastoreDatabases(options, FALLBACK_DB);
+
+ assertThat(databases).containsExactly("db1", "db2", "db3");
+ }
+}
diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java
index 0e7e3adb05..37ce56a140 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java
@@ -49,6 +49,7 @@ import javax.annotation.Nullable;
import java.util.Collections;
import java.util.HashMap;
+import java.util.List;
import java.util.stream.Collectors;
import static org.apache.paimon.iceberg.IcebergCommitCallback.catalogDatabasePath;
@@ -62,14 +63,14 @@ public class IcebergHiveMetadataCommitter implements IcebergMetadataCommitter {
private static final Logger LOG = LoggerFactory.getLogger(IcebergHiveMetadataCommitter.class);
private final FileStoreTable table;
- private final Identifier identifier;
private final ClientPool<IMetaStoreClient, TException> clients;
- private final String icebergHiveDatabase;
- private final String icebergHiveTable;
+ // Supports syncing to multiple Iceberg databases via semicolon-delimited config
+ private final List<String> icebergDatabases;
+ private final String icebergTableName;
public IcebergHiveMetadataCommitter(FileStoreTable table) {
this.table = table;
- this.identifier =
+ Identifier identifier =
Preconditions.checkNotNull(
table.catalogEnvironment().identifier(),
"If you want to sync Paimon Iceberg compatible metadata to Hive, "
@@ -83,8 +84,6 @@ public class IcebergHiveMetadataCommitter implements IcebergMetadataCommitter {
String uri = options.get(IcebergOptions.URI);
String hiveConfDir = options.get(IcebergOptions.HIVE_CONF_DIR);
String hadoopConfDir = options.get(IcebergOptions.HADOOP_CONF_DIR);
- String icebergDatabase = options.get(IcebergOptions.METASTORE_DATABASE);
- String icebergTable = options.get(IcebergOptions.METASTORE_TABLE);
Configuration hadoopConf = new Configuration();
hadoopConf.setClassLoader(IcebergHiveMetadataCommitter.class.getClassLoader());
HiveConf hiveConf = HiveCatalog.createHiveConf(hiveConfDir, hadoopConfDir, hadoopConf);
@@ -104,14 +103,13 @@ public class IcebergHiveMetadataCommitter implements IcebergMetadataCommitter {
IcebergOptions.URI.key());
}
- this.icebergHiveDatabase =
- icebergDatabase != null && !icebergDatabase.isEmpty()
- ? icebergDatabase
- : this.identifier.getDatabaseName();
- this.icebergHiveTable =
- icebergTable != null && !icebergTable.isEmpty()
- ? icebergTable
- : this.identifier.getTableName();
+ this.icebergDatabases =
+ IcebergOptions.metastoreDatabases(options, identifier.getDatabaseName());
+ String tableValue = options.get(IcebergOptions.METASTORE_TABLE);
+ this.icebergTableName =
+ tableValue != null && !tableValue.isEmpty()
+ ? tableValue
+ : identifier.getTableName();
this.clients =
new CachedClientPool(
@@ -140,16 +138,27 @@ public class IcebergHiveMetadataCommitter implements IcebergMetadataCommitter {
private void commitMetadataImpl(Path newMetadataPath, @Nullable Path baseMetadataPath)
throws Exception {
- if (!databaseExists(icebergHiveDatabase)) {
- createDatabase(icebergHiveDatabase);
+ for (String databaseName : icebergDatabases) {
+ commitMetadataForTarget(
+ databaseName, icebergTableName, newMetadataPath, baseMetadataPath);
+ }
+ }
+
+ private void commitMetadataForTarget(
+ String databaseName,
+ String tableName,
+ Path newMetadataPath,
+ @Nullable Path baseMetadataPath)
+ throws Exception {
+ if (!databaseExists(databaseName)) {
+ createDatabase(databaseName);
}
Table hiveTable;
- if (tableExists()) {
- hiveTable =
- clients.run(client -> client.getTable(icebergHiveDatabase, icebergHiveTable));
+ if (tableExists(databaseName, tableName)) {
+ hiveTable = clients.run(client -> client.getTable(databaseName, tableName));
} else {
- hiveTable = createTable(newMetadataPath);
+ hiveTable = createTable(databaseName, tableName, newMetadataPath);
}
hiveTable.getParameters().put("metadata_location", newMetadataPath.toString());
@@ -180,10 +189,7 @@ public class IcebergHiveMetadataCommitter implements IcebergMetadataCommitter {
clients.execute(
client ->
client.alter_table_with_environmentContext(
- icebergHiveDatabase,
- icebergHiveTable,
- hiveTable,
- environmentContext));
+ databaseName, tableName, hiveTable, environmentContext));
}
private boolean databaseExists(String databaseName) throws Exception {
@@ -202,16 +208,17 @@ public class IcebergHiveMetadataCommitter implements IcebergMetadataCommitter {
clients.execute(client -> client.createDatabase(database));
}
- private boolean tableExists() throws Exception {
- return clients.run(client -> client.tableExists(icebergHiveDatabase, icebergHiveTable));
+ private boolean tableExists(String databaseName, String tableName) throws Exception {
+ return clients.run(client -> client.tableExists(databaseName, tableName));
}
- private Table createTable(Path metadataPath) throws Exception {
+ private Table createTable(String databaseName, String tableName, Path metadataPath)
+ throws Exception {
long currentTimeMillis = System.currentTimeMillis();
Table hiveTable =
new Table(
- icebergHiveTable,
- icebergHiveDatabase,
+ tableName,
+ databaseName,
// current linux user
System.getProperty("user.name"),
(int) (currentTimeMillis / 1000),
diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterITCaseBase.java
index 80d7dcc16e..7278f6f9b1 100644
--- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterITCaseBase.java
+++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterITCaseBase.java
@@ -63,6 +63,8 @@ public abstract class IcebergHiveMetadataCommitterITCaseBase {
public void after() {
hiveShell.execute("DROP DATABASE IF EXISTS test_db CASCADE");
hiveShell.execute("DROP DATABASE IF EXISTS test_db_iceberg CASCADE");
+ hiveShell.execute("DROP DATABASE IF EXISTS iceberg_db1 CASCADE");
+ hiveShell.execute("DROP DATABASE IF EXISTS iceberg_db2 CASCADE");
}
@Test
@@ -245,6 +247,80 @@ public abstract class IcebergHiveMetadataCommitterITCaseBase {
"SELECT data, id, pt FROM my_iceberg.test_db_iceberg.t1_iceberg WHERE id > 1 ORDER BY pt, id")));
}
+ @Test
+ public void testMultipleDatabases() throws Exception {
+ TableEnvironment tEnv =
+ TableEnvironmentImpl.create(
+ EnvironmentSettings.newInstance().inBatchMode().build());
+ tEnv.executeSql(
+ "CREATE CATALOG my_paimon WITH ( 'type' = 'paimon', 'warehouse' = '"
+ + path
+ + "' )");
+ tEnv.executeSql("CREATE DATABASE my_paimon.test_db");
+ tEnv.executeSql(
+ "CREATE TABLE my_paimon.test_db.t2 ( pt INT, id INT, data STRING, PRIMARY KEY (pt, id) NOT ENFORCED ) "
+ + "PARTITIONED BY (pt) WITH "
+ + "( 'metadata.iceberg.storage' = 'hive-catalog', 'metadata.iceberg.uri' = '', 'file.format' = 'avro', "
+ + " 'metadata.iceberg.database' = 'iceberg_db1;iceberg_db2',"
+ + " 'full-compaction.delta-commits' = '1' )");
+ tEnv.executeSql(
+ "INSERT INTO my_paimon.test_db.t2 VALUES "
+ + "(1, 1, 'apple'), (1, 2, 'pear'), (2, 1, 'cat'), (2, 2, 'dog')")
+ .await();
+
+ tEnv.executeSql(
+ "CREATE CATALOG my_iceberg WITH "
+ + "( 'type' = 'iceberg', 'catalog-type' = 'hive', 'uri' = '', 'warehouse' = '"
+ + path
+ + "', 'cache-enabled' = 'false' )");
+
+ // data visible in first database
+ assertEquals(
+ Arrays.asList(
+ Row.of(1, 1, "apple"),
+ Row.of(1, 2, "pear"),
+ Row.of(2, 1, "cat"),
+ Row.of(2, 2, "dog")),
+ collect(
+ tEnv.executeSql(
+ "SELECT * FROM my_iceberg.iceberg_db1.t2 ORDER BY pt, id")));
+
+ // data visible in second database
+ assertEquals(
+ Arrays.asList(
+ Row.of(1, 1, "apple"),
+ Row.of(1, 2, "pear"),
+ Row.of(2, 1, "cat"),
+ Row.of(2, 2, "dog")),
+ collect(
+ tEnv.executeSql(
+ "SELECT * FROM my_iceberg.iceberg_db2.t2 ORDER BY pt, id")));
+
+ // updates propagate to both databases
+ tEnv.executeSql(
+ "INSERT INTO my_paimon.test_db.t2 VALUES "
+ + "(1, 1, 'cherry'), (2, 2, 'elephant')")
+ .await();
+ assertEquals(
+ Arrays.asList(
+ Row.of(1, 1, "cherry"),
+ Row.of(1, 2, "pear"),
+ Row.of(2, 1, "cat"),
+ Row.of(2, 2, "elephant")),
+ collect(
+ tEnv.executeSql(
+ "SELECT * FROM my_iceberg.iceberg_db1.t2 ORDER BY pt, id")));
+ assertEquals(
+ Arrays.asList(
+ Row.of(1, 1, "cherry"),
+ Row.of(1, 2, "pear"),
+ Row.of(2, 1, "cat"),
+ Row.of(2, 2, "elephant")),
+ collect(
+ tEnv.executeSql(
+ "SELECT * FROM my_iceberg.iceberg_db2.t2 ORDER BY pt, id")));
+ }
+
@Test
public void testCustomMetastoreClass() {
TableEnvironment tEnv =