You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2026/04/29 14:25:28 UTC

(paimon) branch master updated: [core] [iceberg] Support syncing Iceberg metadata to multiple Hive databases (#7696)

This is an automated email from the ASF dual-hosted git repository.

JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git


The following commit(s) were added to refs/heads/master by this push:
     new c40420fdf2 [core] [iceberg] Support syncing Iceberg metadata to multiple Hive databases (#7696)
c40420fdf2 is described below

commit c40420fdf2fb7b6fef301905aec2134b132790b9
Author: Nick Del Nano <nd...@yelp.com>
AuthorDate: Wed Apr 29 07:25:21 2026 -0700

    [core] [iceberg] Support syncing Iceberg metadata to multiple Hive databases (#7696)
    
    Issue:  https://github.com/apache/paimon/issues/7694
    
    > Paimon's Iceberg compatibility supports a single Iceberg table in the
    catalog with metadata.iceberg.database and metadata.iceberg.table
    values. I'd like it to support the option of multiple db.table
    references so that I can register the Iceberg metadata to both a legacy
    db.table value and a new one. This will allow me to migrate my platform
    onto this feature without changing existing queries while also using my
    new db.table naming standard.
    
    > I don't have a view implementation available in my environment so I
    cannot use views to solve it.
    
    Support registering Iceberg metadata in multiple Hive databases by
    allowing semicolon-delimited values in metadata.iceberg.database (e.g.
    'db1;db2;db3').
---
 .../generated/iceberg_configuration.html           |  2 +-
 .../org/apache/paimon/iceberg/IcebergOptions.java  | 18 ++++-
 .../apache/paimon/iceberg/IcebergOptionsTest.java  | 82 ++++++++++++++++++++++
 .../iceberg/IcebergHiveMetadataCommitter.java      | 65 +++++++++--------
 .../IcebergHiveMetadataCommitterITCaseBase.java    | 76 ++++++++++++++++++++
 5 files changed, 212 insertions(+), 31 deletions(-)

diff --git a/docs/layouts/shortcodes/generated/iceberg_configuration.html b/docs/layouts/shortcodes/generated/iceberg_configuration.html
index cf57763779..3961343242 100644
--- a/docs/layouts/shortcodes/generated/iceberg_configuration.html
+++ b/docs/layouts/shortcodes/generated/iceberg_configuration.html
@@ -42,7 +42,7 @@ under the License.
             <td><h5>metadata.iceberg.database</h5></td>
             <td style="word-wrap: break-word;">(none)</td>
             <td>String</td>
-            <td>Metastore database name for Iceberg Catalog. Set this as an iceberg database alias if using a centralized Catalog.</td>
+            <td>Metastore database name for Iceberg Catalog. Set this as an iceberg database alias if using a centralized Catalog. Multiple databases can be specified with semicolons, e.g. 'db1;db2'. The table will be registered in each database.</td>
         </tr>
         <tr>
             <td><h5>metadata.iceberg.delete-after-commit.enabled</h5></td>
diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergOptions.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergOptions.java
index be4f6152ef..530acd85eb 100644
--- a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergOptions.java
+++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergOptions.java
@@ -28,6 +28,7 @@ import org.apache.paimon.utils.Preconditions;
 
 import java.lang.reflect.Field;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -136,7 +137,9 @@ public class IcebergOptions {
                     .noDefaultValue()
                     .withDescription(
                             "Metastore database name for Iceberg Catalog. "
-                                    + "Set this as an iceberg database alias if using a centralized Catalog.");
+                                    + "Set this as an iceberg database alias if using a centralized Catalog. "
+                                    + "Multiple databases can be specified with semicolons, "
+                                    + "e.g. 'db1;db2'. The table will be registered in each database.");
 
     public static final ConfigOption<String> METASTORE_TABLE =
             key("metadata.iceberg.table")
@@ -259,6 +262,19 @@ public class IcebergOptions {
         }
     }
 
+    public static List<String> metastoreDatabases(Options options, String fallbackDatabase) {
+        String dbValue = options.get(METASTORE_DATABASE);
+        if (dbValue == null || dbValue.isEmpty()) {
+            return Collections.singletonList(fallbackDatabase);
+        }
+        String[] parts = dbValue.split(";");
+        List<String> databases = new ArrayList<>(parts.length);
+        for (String part : parts) {
+            databases.add(part.trim());
+        }
+        return databases;
+    }
+
     /**
      * Returns all ConfigOption fields defined in this class. This method uses reflection to
      * dynamically discover all ConfigOption fields, ensuring that new options are automatically
diff --git a/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergOptionsTest.java b/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergOptionsTest.java
new file mode 100644
index 0000000000..2d57c3ac0e
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergOptionsTest.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.iceberg;
+
+import org.apache.paimon.options.Options;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link IcebergOptions#metastoreDatabases}. */
+public class IcebergOptionsTest {
+
+    private static final String FALLBACK_DB = "default_db";
+
+    @Test
+    public void testSingleDatabase() {
+        Options options = new Options();
+        options.set(IcebergOptions.METASTORE_DATABASE, "mydb");
+
+        List<String> databases = IcebergOptions.metastoreDatabases(options, FALLBACK_DB);
+
+        assertThat(databases).containsExactly("mydb");
+    }
+
+    @Test
+    public void testMultipleDatabases() {
+        Options options = new Options();
+        options.set(IcebergOptions.METASTORE_DATABASE, "db1;db2;db3");
+
+        List<String> databases = IcebergOptions.metastoreDatabases(options, FALLBACK_DB);
+
+        assertThat(databases).containsExactly("db1", "db2", "db3");
+    }
+
+    @Test
+    public void testNoDatabaseConfigFallsBack() {
+        Options options = new Options();
+
+        List<String> databases = IcebergOptions.metastoreDatabases(options, FALLBACK_DB);
+
+        assertThat(databases).containsExactly("default_db");
+    }
+
+    @Test
+    public void testEmptyDatabaseStringFallsBack() {
+        Options options = new Options();
+        options.set(IcebergOptions.METASTORE_DATABASE, "");
+
+        List<String> databases = IcebergOptions.metastoreDatabases(options, FALLBACK_DB);
+
+        assertThat(databases).containsExactly("default_db");
+    }
+
+    @Test
+    public void testWhitespaceAroundSemicolon() {
+        Options options = new Options();
+        options.set(IcebergOptions.METASTORE_DATABASE, " db1 ; db2 ; db3 ");
+
+        List<String> databases = IcebergOptions.metastoreDatabases(options, FALLBACK_DB);
+
+        assertThat(databases).containsExactly("db1", "db2", "db3");
+    }
+}
diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java
index 0e7e3adb05..37ce56a140 100644
--- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java
+++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitter.java
@@ -49,6 +49,7 @@ import javax.annotation.Nullable;
 
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.stream.Collectors;
 
 import static org.apache.paimon.iceberg.IcebergCommitCallback.catalogDatabasePath;
@@ -62,14 +63,14 @@ public class IcebergHiveMetadataCommitter implements IcebergMetadataCommitter {
     private static final Logger LOG = LoggerFactory.getLogger(IcebergHiveMetadataCommitter.class);
 
     private final FileStoreTable table;
-    private final Identifier identifier;
     private final ClientPool<IMetaStoreClient, TException> clients;
-    private final String icebergHiveDatabase;
-    private final String icebergHiveTable;
+    // Supports syncing to multiple Iceberg databases via semicolon-delimited config
+    private final List<String> icebergDatabases;
+    private final String icebergTableName;
 
     public IcebergHiveMetadataCommitter(FileStoreTable table) {
         this.table = table;
-        this.identifier =
+        Identifier identifier =
                 Preconditions.checkNotNull(
                         table.catalogEnvironment().identifier(),
                         "If you want to sync Paimon Iceberg compatible metadata to Hive, "
@@ -83,8 +84,6 @@ public class IcebergHiveMetadataCommitter implements IcebergMetadataCommitter {
         String uri = options.get(IcebergOptions.URI);
         String hiveConfDir = options.get(IcebergOptions.HIVE_CONF_DIR);
         String hadoopConfDir = options.get(IcebergOptions.HADOOP_CONF_DIR);
-        String icebergDatabase = options.get(IcebergOptions.METASTORE_DATABASE);
-        String icebergTable = options.get(IcebergOptions.METASTORE_TABLE);
         Configuration hadoopConf = new Configuration();
         hadoopConf.setClassLoader(IcebergHiveMetadataCommitter.class.getClassLoader());
         HiveConf hiveConf = HiveCatalog.createHiveConf(hiveConfDir, hadoopConfDir, hadoopConf);
@@ -104,14 +103,13 @@ public class IcebergHiveMetadataCommitter implements IcebergMetadataCommitter {
                     IcebergOptions.URI.key());
         }
 
-        this.icebergHiveDatabase =
-                icebergDatabase != null && !icebergDatabase.isEmpty()
-                        ? icebergDatabase
-                        : this.identifier.getDatabaseName();
-        this.icebergHiveTable =
-                icebergTable != null && !icebergTable.isEmpty()
-                        ? icebergTable
-                        : this.identifier.getTableName();
+        this.icebergDatabases =
+                IcebergOptions.metastoreDatabases(options, identifier.getDatabaseName());
+        String tableValue = options.get(IcebergOptions.METASTORE_TABLE);
+        this.icebergTableName =
+                tableValue != null && !tableValue.isEmpty()
+                        ? tableValue
+                        : identifier.getTableName();
 
         this.clients =
                 new CachedClientPool(
@@ -140,16 +138,27 @@ public class IcebergHiveMetadataCommitter implements IcebergMetadataCommitter {
 
     private void commitMetadataImpl(Path newMetadataPath, @Nullable Path baseMetadataPath)
             throws Exception {
-        if (!databaseExists(icebergHiveDatabase)) {
-            createDatabase(icebergHiveDatabase);
+        for (String databaseName : icebergDatabases) {
+            commitMetadataForTarget(
+                    databaseName, icebergTableName, newMetadataPath, baseMetadataPath);
+        }
+    }
+
+    private void commitMetadataForTarget(
+            String databaseName,
+            String tableName,
+            Path newMetadataPath,
+            @Nullable Path baseMetadataPath)
+            throws Exception {
+        if (!databaseExists(databaseName)) {
+            createDatabase(databaseName);
         }
 
         Table hiveTable;
-        if (tableExists()) {
-            hiveTable =
-                    clients.run(client -> client.getTable(icebergHiveDatabase, icebergHiveTable));
+        if (tableExists(databaseName, tableName)) {
+            hiveTable = clients.run(client -> client.getTable(databaseName, tableName));
         } else {
-            hiveTable = createTable(newMetadataPath);
+            hiveTable = createTable(databaseName, tableName, newMetadataPath);
         }
 
         hiveTable.getParameters().put("metadata_location", newMetadataPath.toString());
@@ -180,10 +189,7 @@ public class IcebergHiveMetadataCommitter implements IcebergMetadataCommitter {
         clients.execute(
                 client ->
                         client.alter_table_with_environmentContext(
-                                icebergHiveDatabase,
-                                icebergHiveTable,
-                                hiveTable,
-                                environmentContext));
+                                databaseName, tableName, hiveTable, environmentContext));
     }
 
     private boolean databaseExists(String databaseName) throws Exception {
@@ -202,16 +208,17 @@ public class IcebergHiveMetadataCommitter implements IcebergMetadataCommitter {
         clients.execute(client -> client.createDatabase(database));
     }
 
-    private boolean tableExists() throws Exception {
-        return clients.run(client -> client.tableExists(icebergHiveDatabase, icebergHiveTable));
+    private boolean tableExists(String databaseName, String tableName) throws Exception {
+        return clients.run(client -> client.tableExists(databaseName, tableName));
     }
 
-    private Table createTable(Path metadataPath) throws Exception {
+    private Table createTable(String databaseName, String tableName, Path metadataPath)
+            throws Exception {
         long currentTimeMillis = System.currentTimeMillis();
         Table hiveTable =
                 new Table(
-                        icebergHiveTable,
-                        icebergHiveDatabase,
+                        tableName,
+                        databaseName,
                         // current linux user
                         System.getProperty("user.name"),
                         (int) (currentTimeMillis / 1000),
diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterITCaseBase.java
index 80d7dcc16e..7278f6f9b1 100644
--- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterITCaseBase.java
+++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/iceberg/IcebergHiveMetadataCommitterITCaseBase.java
@@ -63,6 +63,8 @@ public abstract class IcebergHiveMetadataCommitterITCaseBase {
     public void after() {
         hiveShell.execute("DROP DATABASE IF EXISTS test_db CASCADE");
         hiveShell.execute("DROP DATABASE IF EXISTS test_db_iceberg CASCADE");
+        hiveShell.execute("DROP DATABASE IF EXISTS iceberg_db1 CASCADE");
+        hiveShell.execute("DROP DATABASE IF EXISTS iceberg_db2 CASCADE");
     }
 
     @Test
@@ -245,6 +247,80 @@ public abstract class IcebergHiveMetadataCommitterITCaseBase {
                                 "SELECT data, id, pt FROM my_iceberg.test_db_iceberg.t1_iceberg WHERE id > 1 ORDER BY pt, id")));
     }
 
+    @Test
+    public void testMultipleDatabases() throws Exception {
+        TableEnvironment tEnv =
+                TableEnvironmentImpl.create(
+                        EnvironmentSettings.newInstance().inBatchMode().build());
+        tEnv.executeSql(
+                "CREATE CATALOG my_paimon WITH ( 'type' = 'paimon', 'warehouse' = '"
+                        + path
+                        + "' )");
+        tEnv.executeSql("CREATE DATABASE my_paimon.test_db");
+        tEnv.executeSql(
+                "CREATE TABLE my_paimon.test_db.t2 ( pt INT, id INT, data STRING, PRIMARY KEY (pt, id) NOT ENFORCED ) "
+                        + "PARTITIONED BY (pt) WITH "
+                        + "( 'metadata.iceberg.storage' = 'hive-catalog', 'metadata.iceberg.uri' = '', 'file.format' = 'avro', "
+                        + " 'metadata.iceberg.database' = 'iceberg_db1;iceberg_db2',"
+                        + " 'full-compaction.delta-commits' = '1' )");
+        tEnv.executeSql(
+                        "INSERT INTO my_paimon.test_db.t2 VALUES "
+                                + "(1, 1, 'apple'), (1, 2, 'pear'), (2, 1, 'cat'), (2, 2, 'dog')")
+                .await();
+
+        tEnv.executeSql(
+                "CREATE CATALOG my_iceberg WITH "
+                        + "( 'type' = 'iceberg', 'catalog-type' = 'hive', 'uri' = '', 'warehouse' = '"
+                        + path
+                        + "', 'cache-enabled' = 'false' )");
+
+        // data visible in first database
+        assertEquals(
+                Arrays.asList(
+                        Row.of(1, 1, "apple"),
+                        Row.of(1, 2, "pear"),
+                        Row.of(2, 1, "cat"),
+                        Row.of(2, 2, "dog")),
+                collect(
+                        tEnv.executeSql(
+                                "SELECT * FROM my_iceberg.iceberg_db1.t2 ORDER BY pt, id")));
+
+        // data visible in second database
+        assertEquals(
+                Arrays.asList(
+                        Row.of(1, 1, "apple"),
+                        Row.of(1, 2, "pear"),
+                        Row.of(2, 1, "cat"),
+                        Row.of(2, 2, "dog")),
+                collect(
+                        tEnv.executeSql(
+                                "SELECT * FROM my_iceberg.iceberg_db2.t2 ORDER BY pt, id")));
+
+        // updates propagate to both databases
+        tEnv.executeSql(
+                        "INSERT INTO my_paimon.test_db.t2 VALUES "
+                                + "(1, 1, 'cherry'), (2, 2, 'elephant')")
+                .await();
+        assertEquals(
+                Arrays.asList(
+                        Row.of(1, 1, "cherry"),
+                        Row.of(1, 2, "pear"),
+                        Row.of(2, 1, "cat"),
+                        Row.of(2, 2, "elephant")),
+                collect(
+                        tEnv.executeSql(
+                                "SELECT * FROM my_iceberg.iceberg_db1.t2 ORDER BY pt, id")));
+        assertEquals(
+                Arrays.asList(
+                        Row.of(1, 1, "cherry"),
+                        Row.of(1, 2, "pear"),
+                        Row.of(2, 1, "cat"),
+                        Row.of(2, 2, "elephant")),
+                collect(
+                        tEnv.executeSql(
+                                "SELECT * FROM my_iceberg.iceberg_db2.t2 ORDER BY pt, id")));
+    }
+
     @Test
     public void testCustomMetastoreClass() {
         TableEnvironment tEnv =