You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by di...@apache.org on 2026/10/29 20:21:24 UTC
(polaris-tools) branch main updated: Migrate over namespace properties (#283)
This is an automated email from the ASF dual-hosted git repository.
dimas-b pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris-tools.git
The following commit(s) were added to refs/heads/main by this push:
new 95f1505 Migrate over namespace properties (#283)
95f1505 is described below
commit 95f150582c935f7e522a86301024aef9154bcfba
Author: Alex Stephen <13...@users.noreply.github.com>
AuthorDate: Thu Oct 29 13:21:18 2026 -0700
Migrate over namespace properties (#283)
* Migrate over namespace properties
---
.../catalog/migrator/api/CatalogMigrator.java | 48 +++++++++++++++++++++-
.../migrator/api/AbstractTestCatalogMigrator.java | 36 ++++++++++++++++
.../migrator/api/HadoopCatalogMigratorTest.java | 47 +++++++++++++++++++++
3 files changed, 130 insertions(+), 1 deletion(-)
diff --git a/iceberg-catalog-migrator/api/src/main/java/org/apache/polaris/iceberg/catalog/migrator/api/CatalogMigrator.java b/iceberg-catalog-migrator/api/src/main/java/org/apache/polaris/iceberg/catalog/migrator/api/CatalogMigrator.java
index b3b84d4..d752378 100644
--- a/iceberg-catalog-migrator/api/src/main/java/org/apache/polaris/iceberg/catalog/migrator/api/CatalogMigrator.java
+++ b/iceberg-catalog-migrator/api/src/main/java/org/apache/polaris/iceberg/catalog/migrator/api/CatalogMigrator.java
@@ -20,9 +20,11 @@ package org.apache.polaris.iceberg.catalog.migrator.api;
import com.google.common.base.Preconditions;
import java.util.Arrays;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.function.Predicate;
import java.util.regex.Pattern;
@@ -85,6 +87,7 @@ public abstract class CatalogMigrator {
}
private static final Logger LOG = LoggerFactory.getLogger(CatalogMigrator.class);
+ private static final String NAMESPACE_LOCATION_PROPERTY = "location";
private final ImmutableCatalogMigrationResult.Builder resultBuilder =
ImmutableCatalogMigrationResult.builder();
private final Set<Namespace> processedNamespaces = new HashSet<>();
@@ -273,7 +276,7 @@ public abstract class CatalogMigrator {
Namespace namespace = Namespace.of(Arrays.copyOfRange(levels, 0, index + 1));
if (processedNamespaces.add(namespace)) {
try {
- ((SupportsNamespaces) targetCatalog()).createNamespace(namespace);
+ createNamespaceOnTargetCatalog(namespace);
} catch (AlreadyExistsException ex) {
LOG.debug(
"{}.Ignoring the error as forcefully creating the namespace even if it exists to avoid "
@@ -285,6 +288,49 @@ public abstract class CatalogMigrator {
}
}
+ // Namespace properties are copied best effort: failing to copy them must not fail the table.
+ private void createNamespaceOnTargetCatalog(Namespace namespace) {
+ SupportsNamespaces target = (SupportsNamespaces) targetCatalog();
+ try {
+ target.createNamespace(namespace, sourceNamespaceProperties(namespace));
+ } catch (UnsupportedOperationException ex) {
+ LOG.warn(
+ "Target catalog doesn't support namespace properties. "
+ + "Creating namespace {} without properties : {}",
+ namespace,
+ ex.getMessage());
+ target.createNamespace(namespace);
+ }
+ }
+
+ private Map<String, String> sourceNamespaceProperties(Namespace namespace) {
+ try {
+ Map<String, String> properties =
+ new HashMap<>(((SupportsNamespaces) sourceCatalog()).loadNamespaceMetadata(namespace));
+ // location points into the source catalog's storage, so let the target assign its own.
+ properties.remove(NAMESPACE_LOCATION_PROPERTY);
+ return properties;
+ } catch (NoSuchNamespaceException ex) {
+ // implicit namespace, so there's nothing to copy.
+ return Map.of();
+ } catch (Exception ex) {
+ if (enableStacktrace()) {
+ LOG.warn(
+ "Unable to load the properties of namespace {} from source catalog. "
+ + "Creating it in target catalog without properties",
+ namespace,
+ ex);
+ } else {
+ LOG.warn(
+ "Unable to load the properties of namespace {} from source catalog. "
+ + "Creating it in target catalog without properties : {}",
+ namespace,
+ ex.getMessage());
+ }
+ return Map.of();
+ }
+ }
+
protected void getAllNamespacesFromSourceCatalog(Namespace namespace, Set<Namespace> visited) {
if (!visited.add(namespace)) {
return;
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/AbstractTestCatalogMigrator.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/AbstractTestCatalogMigrator.java
index 7a61034..0dfc5d0 100644
--- a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/AbstractTestCatalogMigrator.java
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/AbstractTestCatalogMigrator.java
@@ -25,9 +25,11 @@ import nl.altindag.log.LogCaptor;
import nl.altindag.log.model.LogEvent;
import org.apache.iceberg.catalog.Catalog;
import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.SupportsNamespaces;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.catalog.ViewCatalog;
import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.hadoop.HadoopCatalog;
import org.apache.iceberg.hive.HiveCatalog;
import org.apache.polaris.iceberg.catalog.migrator.api.test.AbstractTest;
import org.assertj.core.api.Assertions;
@@ -351,6 +353,40 @@ public abstract class AbstractTestCatalogMigrator extends AbstractTest {
Assertions.assertThat(targetCatalog.listTables(DB1)).containsExactly(tbl5);
}
+ @Test
+ public void testRegisterWithNewNamespaceCopiesProperties() {
+ Assumptions.assumeFalse(
+ sourceCatalog instanceof HadoopCatalog,
+ "HadoopCatalog doesn't support namespace properties");
+
+ Namespace namespace = Namespace.of("db_props");
+ TableIdentifier identifier = TableIdentifier.of(namespace, "tbl");
+ SupportsNamespaces source = (SupportsNamespaces) sourceCatalog;
+ SupportsNamespaces target = (SupportsNamespaces) targetCatalog;
+ source.createNamespace(namespace, Map.of("comment", "migrated namespace"));
+ sourceCatalog.createTable(identifier, schema);
+ String sourceLocation = source.loadNamespaceMetadata(namespace).get("location");
+
+ CatalogMigrationResult result =
+ catalogMigratorWithDefaultArgs(false).registerTable(identifier).result();
+
+ Assertions.assertThat(result.registeredTableIdentifiers()).containsExactly(identifier);
+ Map<String, String> targetProperties = target.loadNamespaceMetadata(namespace);
+ if (targetCatalog instanceof HadoopCatalog) {
+ Assertions.assertThat(targetProperties).doesNotContainKey("comment");
+ } else {
+ Assertions.assertThat(targetProperties).containsEntry("comment", "migrated namespace");
+ }
+ if (sourceLocation != null) {
+ Assertions.assertThat(targetProperties.get("location")).isNotEqualTo(sourceLocation);
+ }
+
+ targetCatalog.dropTable(identifier, false);
+ sourceCatalog.dropTable(identifier, true);
+ target.dropNamespace(namespace);
+ source.dropNamespace(namespace);
+ }
+
@ParameterizedTest
@CsvSource(value = {"false,false", "false,true", "true,false", "true,true"})
public void testStacktrace(boolean deleteSourceTables, boolean enableStacktrace) {
diff --git a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/HadoopCatalogMigratorTest.java b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/HadoopCatalogMigratorTest.java
index 3242962..bccae6e 100644
--- a/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/HadoopCatalogMigratorTest.java
+++ b/iceberg-catalog-migrator/api/src/test/java/org/apache/polaris/iceberg/catalog/migrator/api/HadoopCatalogMigratorTest.java
@@ -22,13 +22,18 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.SupportsNamespaces;
import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.hadoop.HadoopCatalog;
import org.assertj.core.api.Assertions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
public class HadoopCatalogMigratorTest extends AbstractTestCatalogMigrator {
@@ -38,6 +43,48 @@ public class HadoopCatalogMigratorTest extends AbstractTestCatalogMigrator {
initializeTargetCatalog(CatalogMigrationUtil.CatalogType.HADOOP, Collections.emptyMap());
}
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testRegisterWithNewNamespaceWhenPropertiesCannotBeCopied(
+ boolean sourcePropertiesReadable) {
+ // HadoopCatalog can't store namespace properties, so the source fakes them.
+ HadoopCatalog source =
+ new HadoopCatalog() {
+ @Override
+ public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+ if (!sourcePropertiesReadable) {
+ throw new UnsupportedOperationException("not permitted");
+ }
+ return Map.of("comment", "migrated namespace");
+ }
+ };
+ source.setConf(new Configuration());
+ source.initialize("sourceCatalog", hadoopCatalogProperties(true));
+
+ Namespace namespace = Namespace.of("db_props");
+ TableIdentifier identifier = TableIdentifier.of(namespace, "tbl");
+ source.createNamespace(namespace);
+ source.createTable(identifier, schema);
+
+ CatalogMigrationResult result =
+ ImmutableCatalogMigrator.builder()
+ .sourceCatalog(source)
+ .targetCatalog(targetCatalog)
+ .deleteEntriesFromSourceCatalog(false)
+ .build()
+ .registerTable(identifier)
+ .result();
+
+ Assertions.assertThat(result.registeredTableIdentifiers()).containsExactly(identifier);
+ Assertions.assertThat(result.failedToRegisterTableIdentifiers()).isEmpty();
+ Assertions.assertThat(((SupportsNamespaces) targetCatalog).namespaceExists(namespace)).isTrue();
+
+ targetCatalog.dropTable(identifier, false);
+ source.dropTable(identifier, true);
+ ((SupportsNamespaces) targetCatalog).dropNamespace(namespace);
+ source.dropNamespace(namespace);
+ }
+
@Test
public void testRegisterWithNewNestedNamespaces() {
List<Namespace> namespaceList = Arrays.asList(NS1, NS2, NS3, NS1_NS2, NS1_NS3, NS1_NS2_NS3);