You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by em...@apache.org on 2025/04/18 17:24:42 UTC
(polaris-tools) branch main updated: Add Polaris synchronization and migration tool to polaris-tools. (#4)
This is an automated email from the ASF dual-hosted git repository.
emaynard pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris-tools.git
The following commit(s) were added to refs/heads/main by this push:
new bdda19f Add Polaris synchronization and migration tool to polaris-tools. (#4)
bdda19f is described below
commit bdda19f39619969ce939c541f9775a5e7290da87
Author: Mansehaj Singh <ms...@gmail.com>
AuthorDate: Fri Apr 18 10:24:38 2025 -0700
Add Polaris synchronization and migration tool to polaris-tools. (#4)
* Initial commit of synchronizer code
Addressed comments
Add minimal build configuration
* Added optional principal migrations
* Updated tests to accommodate principal creation
* Added migration of principal roles to principals
* Addressed comments
* Updated tests
* Add generic Polaris entity source and target- not tied to API
* Updated docs
* Remove type in options
* update docs
* Added license headers
* Add hard failure flag
* make flag final
* Added explanation to README.md
* Make ETagManager configurable
* Add configurable oauth server for omnipotent principal
* Set iceberg write access as connection property explicitly outside of factory
* Add external id to aws storage config ignore list
---
polaris-synchronizer/.gitignore | 74 +
polaris-synchronizer/README.md | 173 +++
polaris-synchronizer/api/build.gradle.kts | 154 +++
.../tools/sync/polaris/PolarisSynchronizer.java | 1287 ++++++++++++++++++
.../polaris/access/AccessControlConstants.java | 36 +
.../sync/polaris/access/AccessControlService.java | 289 ++++
.../sync/polaris/catalog/BaseTableWithETag.java | 49 +
.../tools/sync/polaris/catalog/ETagManager.java | 55 +
.../catalog/MetadataNotModifiedException.java | 42 +
.../catalog/MetadataWrapperTableOperations.java | 71 +
.../sync/polaris/catalog/NoOpETagManager.java | 38 +
.../tools/sync/polaris/catalog/PolarisCatalog.java | 174 +++
.../polaris/tools/sync/polaris/http/HttpUtil.java | 39 +
.../tools/sync/polaris/http/OAuth2Util.java | 78 ++
.../planning/AccessControlAwarePlanner.java | 306 +++++
.../sync/polaris/planning/DelegatedPlanner.java | 120 ++
.../polaris/planning/ModificationAwarePlanner.java | 422 ++++++
.../sync/polaris/planning/NoOpSyncPlanner.java | 107 ++
.../SourceParitySynchronizationPlanner.java | 192 +++
.../polaris/planning/SynchronizationPlanner.java | 80 ++
.../sync/polaris/planning/plan/PlannedAction.java | 52 +
.../polaris/planning/plan/SynchronizationPlan.java | 111 ++
.../polaris/service/IcebergCatalogService.java | 54 +
.../tools/sync/polaris/service/PolarisService.java | 86 ++
.../polaris/service/impl/PolarisApiService.java | 287 ++++
.../service/impl/PolarisIcebergCatalogService.java | 163 +++
.../main/resources/polaris-management-service.yml | 1432 ++++++++++++++++++++
.../polaris/AccessControlAwarePlannerTest.java | 212 +++
.../sync/polaris/ModificationAwarePlannerTest.java | 401 ++++++
.../SourceParitySynchronizationPlannerTest.java | 303 +++++
polaris-synchronizer/build.gradle.kts | 24 +
polaris-synchronizer/cli/build.gradle.kts | 73 +
.../polaris/CreateOmnipotentPrincipalCommand.java | 195 +++
.../polaris/tools/sync/polaris/CsvETagManager.java | 138 ++
.../tools/sync/polaris/ETagManagerFactory.java | 67 +
.../tools/sync/polaris/PolarisServiceFactory.java | 56 +
.../tools/sync/polaris/PolarisSynchronizerCLI.java | 46 +
.../tools/sync/polaris/SyncPolarisCommand.java | 171 +++
.../cli/src/main/resources/logback.xml | 36 +
.../tools/sync/polaris/ETagManagerFactoryTest.java | 41 +
.../gradle/wrapper/gradle-wrapper.properties | 7 +
polaris-synchronizer/gradlew | 252 ++++
polaris-synchronizer/gradlew.bat | 94 ++
polaris-synchronizer/settings.gradle.kts | 10 +
44 files changed, 8097 insertions(+)
diff --git a/polaris-synchronizer/.gitignore b/polaris-synchronizer/.gitignore
new file mode 100644
index 0000000..a893541
--- /dev/null
+++ b/polaris-synchronizer/.gitignore
@@ -0,0 +1,74 @@
+
+### Java ###
+# Compiled class file
+*.class
+
+# Log file
+*.log
+
+# BlueJ files
+*.ctxt
+
+# Mobile Tools for Java (J2ME)
+.mtj.tmp/
+
+# Package Files #
+*.jar
+*.war
+*.nar
+*.ear
+*.zip
+*.tar.gz
+*.rar
+
+# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
+hs_err_pid*
+
+#misc
+target/
+dependency-reduced-pom.xml
+*.patch
+*.DS_Store
+.DS_Store
+
+#intellij
+*.iml
+.idea
+*.ipr
+*.iws
+
+# vscode
+.vscode
+
+# node
+node_modules/
+ui/src/generated/
+
+# Eclipse IDE
+.classpath
+.factorypath
+.project
+.settings
+.checkstyle
+out/
+
+# gradle
+.gradle/
+build/
+gradle/wrapper/gradle-wrapper.jar
+version.txt
+
+# Python venv
+venv/
+
+# Maven flatten plugin
+.flattened-pom.xml
+
+# Site
+site/site
+
+# Ignore Gradle project-specific cache directory
+.gradle
+
+# Ignore Gradle build output directory
+build
diff --git a/polaris-synchronizer/README.md b/polaris-synchronizer/README.md
new file mode 100644
index 0000000..fb9e4f6
--- /dev/null
+++ b/polaris-synchronizer/README.md
@@ -0,0 +1,173 @@
+# Objective
+
+To provide users of [Apache Polaris (Incubating)](https://github.com/apache/polaris) a tool to be able to easily and efficiently
+migrate their entities from one Polaris instance to another.
+
+Polaris is a catalog of catalogs. It can become cumbersome to perform catalog-by-catalog migration of each and every catalog contained
+within a Polaris instance. Additionally, while migrating catalog-by-catalog Iceberg entities is achievable using the
+existing generic [iceberg-catalog-migrator](../iceberg-catalog-migrator/README.md), the existing tool will not migrate
+Polaris specific entities, like principal-roles, catalog-roles, grants.
+
+## Use Cases
+* **Migration:** A user may have an active Polaris deployment that they want to migrate to a managed cloud offering like
+ [Snowflake Open Catalog](https://www.snowflake.com/en/product/features/open-catalog/).
+* **Preventing Vendor Lock-In:** A user may currently have a managed Polaris offering and want the freedom to switch providers or to host Polaris themselves.
+* **Backup:** Modern data solutions often require employing redundancy. This tool can be run on a periodic cron to keep snapshots of a Polaris instance.
+
+In the case of migration to/from a cloud offering, access to the Polaris metastore is possibly limited or entirely restricted.
+This tool instead uses the Polaris REST API to perform the migration/synchronization.
+
+The tool currently supports migrating the following Polaris Management entities:
+* Optionally, Principals (with `--sync-principals` flag). Credentials will be different on the target instance.
+* Optionally, assignment of Principal Roles to Principals (with `--sync-principals` flag)
+* Principal roles
+* Catalogs
+ * Catalog Roles
+ * Assignment of Catalog Roles to Principal Roles
+ * Grants
+
+The tool currently supports migrating the following Iceberg entities:
+* Namespaces
+* Tables
+
+# Building the Tool from Source
+
+**Prerequisite:** Must have Java installed in your machine (Java 21 is recommended as the minimum Java version) to use this CLI tool.
+
+```
+gradlew build # build and run tests
+gradlew assemble # build without running tests
+```
+
+The default build location for the built JAR will be `cli/build/libs/`
+
+# Migrating between Polaris Instances
+
+### Step 1: Create a principal with read-only access to catalog internals on the source Polaris instance.
+
+**This step only has to be completed once.**
+
+Polaris is built with a separation between access and metadata management permissions. The `service_admin`
+may have permissions to create access related entities like principal roles, catalog roles, and grants, but may not necessarily
+possess the ability to view Iceberg content of catalogs, like namespaces and tables. We need to create a super user principal
+that has access to all entities on the source Polaris instance in order to migrate them.
+
+To do this, we can use the `create-omnipotent-principal` command to create a principal, principal role,
+and a catalog role per catalog with the appropriate grants to read all entities on the source Polaris instance.
+
+**Example:** Create a **read-only** principal on the source Polaris instance, and replace it if it already exists,
+with 10 concurrent catalog setup threads:
+```
+java -jar cli/build/libs/polaris-synchronizer-cli.jar create-omnipotent-principal \
+--polaris-api-connection-properties base-url=http://localhost:8181 \
+--polaris-api-connection-properties oauth2-server-uri=http://localhost:8181/api/catalog/v1/oauth/tokens \
+--polaris-api-connection-properties client-id=root \
+--polaris-api-connection-properties client-secret=<client_secret> \
+--polaris-api-connection-properties scope=PRINCIPAL_ROLE:ALL \
+--replace \ # replace it if it already exists
+--concurrency 10 # 10 concurrent catalog setup threads
+```
+
+Upon finishing execution, the tool will output the principal name and client credentials for this
+principal. **Make sure to note these down as they will be necessary for the migration step.**
+
+**Example Output:**
+```
+======================================================
+Omnipotent Principal Credentials:
+name = omnipotent-principal-XXXXX
+clientId = ff7s8f9asbX10
+clientSecret = <client-secret>
+======================================================
+```
+
+Additionally, at the end of execution the command will output a list of catalogs for which catalog setup failed.
+**These catalogs may experience failure during migration**.
+
+**Example Output:**
+```
+Encountered issues creating catalog roles for the following catalogs: [catalog-1, catalog-2]
+```
+
+### Step 2: Create a principal with read-write access to catalog internals on the target Polaris instance.
+
+**This step only has to be completed once.**
+
+The same `create-omnipotent-principal` command can also be used to now create a **read-write** principal on the target
+Polaris instance so that the tool can create entities on the target.
+
+To create a read-write principal, we simply specify the `--write-access` option.
+
+**Example:** Create a read-write principal on your target Polaris instance, replacing it if it exists, with 10 concurrent
+catalog setup threads.
+```
+java -jar cli/build/libs/polaris-synchronizer-cli.jar \
+create-omnipotent-principal \
+--polaris-api-connection-properties base-url=http://localhost:8181 \
+--polaris-api-connection-properties oauth2-server-uri=http://localhost:8181/api/catalog/v1/oauth/tokens \
+--polaris-api-connection-properties client-id=root \
+--polaris-api-connection-properties client-secret=<client_secret> \
+--polaris-api-connection-properties scope=PRINCIPAL_ROLE:ALL \
+--replace \ # replace if it already exists
+--concurrency 10 \ # 10 concurrent catalog setup threads
+--write-access # give the principal write access to catalog internals
+```
+
+Similarly to the last step, the tool will output the client credentials and principal name. Again, these need to be noted
+for subsequent steps.
+
+**Example Output:**
+```
+======================================================
+Omnipotent Principal Credentials:
+name = omnipotent-principal-YYYYY
+clientId = 0af20a3a0037a40d
+clientSecret = <client-secret>
+======================================================
+```
+
+> :warning: `service_admin` is not guaranteed to have access management level grants on every catalog. This is usually
+> delegated to the `catalog_admin` role, which is automatically granted to whichever principal role was used to create
+> the catalog. This means that while the tool can detect this catalog when run with `service_admin` level access,
+> it cannot create an omnipotent principal for this catalog. To remedy this, create a catalog-role with `CATALOG_MANAGE_ACCESS`
+> grants for the catalog, and assign it to the principal used to run this tool (presumably, a principal with the `servic_admin`
+> principal role). Then, re-running `create-omnipotent-principal` should be able to create the relevant entities for that catalog.
+
+### Step 3: Running the Migration/Synchronization
+
+Running the synchronization requires minimal reconfiguration, can be run idempotently, and will attempt to only copy over the
+diff between the source and target Polaris instances. This can be achieved using the `sync-polaris` command.
+
+> :warning: If you want to migrate principals and their assignments to principal-roles as well, run the tool with the
+> `--sync-principals` flag. Please note that this will reset the client credentials for that principal on the target
+> Polaris instance. The new credentials will be logged to stdout, ONLY for each newly created or overwritten principal.
+> Please note that this output should be securely managed, client credentials should only ever be stored in a secure vault.
+
+**Example** Running the synchronization between source Polaris instance using an access token, and a target Polaris instance
+using client credentials.
+```
+java -jar cli/build/libs/polaris-synchronizer-cli.jar sync-polaris \
+--source-properties base-url=http://localhost:8181 \
+--source-properties client-id=root \
+--source-properties client-secret=<client_secret> \
+--source-properties oauth2-server-uri=http://localhost:8181/api/catalog/v1/oauth/tokens \
+--source-properties scope=PRINCIPAL_ROLE:ALL \
+--source-properties omnipotent-principal-name=omnipotent-principal-XXXXX \
+--source-properties omnipotent-principal-client-id=589550e8b23d271e \
+--source-properties omnipotent-principal-client-secret=<omni_client_secret> \
+--source-properties omnipotent-principal-oauth2-server-uri=http://localhost:8181/api/catalog/v1/oauth/tokens \
+--target-properties base-url=http://localhost:5858 \
+--target-properties client-id=root \
+--target-properties client-secret=<client_secret> \
+--target-properties oauth2-server-uri=http://localhost:5858/api/catalog/v1/oauth/tokens \
+--target-properties scope=PRINCIPAL_ROLE:ALL \
+--target-properties omnipotent-principal-name=omnipotent-principal-YYYYY \
+--target-properties omnipotent-principal-client-id=9b8ac0f1e4e2e614 \
+--target-properties omnipotent-principal-client-secret=<omni_client_secret> \
+--target-properties omnipotent-principal-oauth2-server-uri=http://localhost:5858/api/catalog/v1/oauth/tokens
+```
+
+> :warning: The tool will not migrate the `service_admin`, `catalog_admin`, nor the omnipotent principals from the source
+> nor remove or modify them or their assignments to principals/principal-roles on the target. This is to accommodate that
+> the tool itself will be running with the permission levels for these principals and roles, and we do not want to modify
+> the tool's permissions at runtime.
diff --git a/polaris-synchronizer/api/build.gradle.kts b/polaris-synchronizer/api/build.gradle.kts
new file mode 100644
index 0000000..a348d21
--- /dev/null
+++ b/polaris-synchronizer/api/build.gradle.kts
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+ `java-library`
+ `maven-publish`
+ signing
+ id("org.openapi.generator") version "7.11.0"
+}
+
+repositories {
+ mavenCentral()
+}
+
+java {
+ toolchain {
+ languageVersion.set(JavaLanguageVersion.of(21)) // Set the compilation JDK to 21
+ }
+}
+
+dependencies {
+ // implementation(libs.openapi.generator)
+ implementation("jakarta.annotation:jakarta.annotation-api:2.1.1")
+ implementation("org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.7.1")
+ implementation("com.fasterxml.jackson.core:jackson-databind:2.18.3")
+ implementation("com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.18.3")
+ implementation("org.slf4j:log4j-over-slf4j:2.0.17")
+
+ implementation("org.apache.hadoop:hadoop-common:2.7.3") {
+ exclude("org.apache.avro", "avro")
+ exclude("org.slf4j", "slf4j-log4j12")
+ exclude("javax.servlet", "servlet-api")
+ exclude("com.google.code.gson", "gson")
+ exclude("commons-beanutils")
+ }
+
+ testImplementation("org.junit.jupiter:junit-jupiter-params:5.10.0")
+ testImplementation("org.junit.jupiter:junit-jupiter-api:5.10.0")
+ testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.10.0")
+ testRuntimeOnly("org.junit.platform:junit-platform-launcher:1.10.0")
+}
+
+tasks.withType<Test>().configureEach { useJUnitPlatform() }
+
+tasks.register<org.openapitools.generator.gradle.plugin.tasks.GenerateTask>(
+ "generatePolarisManagementClient"
+) {
+ inputSpec.set("$projectDir/src/main/resources/polaris-management-service.yml")
+ generatorName.set("java")
+ outputDir.set("${layout.buildDirectory.get()}/generated")
+ apiPackage.set("org.apache.polaris.management.client")
+ modelPackage.set("org.apache.polaris.core.admin.model")
+ removeOperationIdPrefix.set(true)
+
+ globalProperties.set(
+ mapOf(
+ "apis" to "",
+ "models" to "",
+ "supportingFiles" to "",
+ "apiDocs" to "false",
+ "modelTests" to "false",
+ )
+ )
+
+ additionalProperties.set(
+ mapOf(
+ "apiNamePrefix" to "PolarisManagement",
+ "apiNameSuffix" to "Api",
+ "metricsPrefix" to "polaris.management",
+ )
+ )
+
+ configOptions.set(
+ mapOf(
+ "library" to "native",
+ "sourceFolder" to "src/main/java",
+ "useJakartaEe" to "true",
+ "useBeanValidation" to "false",
+ "openApiNullable" to "false",
+ "useRuntimeException" to "true",
+ "supportUrlQuery" to "false",
+ )
+ )
+
+ importMappings.set(
+ mapOf(
+ "AbstractOpenApiSchema" to "org.apache.polaris.core.admin.model.AbstractOpenApiSchema",
+ "AddGrantRequest" to "org.apache.polaris.core.admin.model.AddGrantRequest",
+ "AwsStorageConfigInfo" to "org.apache.polaris.core.admin.model.AwsStorageConfigInfo",
+ "AzureStorageConfigInfo" to "org.apache.polaris.core.admin.model.AzureStorageConfigInfo",
+ "Catalog" to "org.apache.polaris.core.admin.model.Catalog",
+ "CatalogGrant" to "org.apache.polaris.core.admin.model.CatalogGrant",
+ "CatalogPrivilege" to "org.apache.polaris.core.admin.model.CatalogPrivilege",
+ "CatalogProperties" to "org.apache.polaris.core.admin.model.CatalogProperties",
+ "CatalogRole" to "org.apache.polaris.core.admin.model.CatalogRole",
+ "CatalogRoles" to "org.apache.polaris.core.admin.model.CatalogRoles",
+ "Catalogs" to "org.apache.polaris.core.admin.model.Catalogs",
+ "CreateCatalogRequest" to "org.apache.polaris.core.admin.model.CreateCatalogRequest",
+ "CreateCatalogRoleRequest" to "org.apache.polaris.core.admin.model.CreateCatalogRoleRequest",
+ "CreatePrincipalRequest" to "org.apache.polaris.core.admin.model.CreatePrincipalRequest",
+ "CreatePrincipalRoleRequest" to
+ "org.apache.polaris.core.admin.model.CreatePrincipalRoleRequest",
+ "ExternalCatalog" to "org.apache.polaris.core.admin.model.ExternalCatalog",
+ "FileStorageConfigInfo" to "org.apache.polaris.core.admin.model.FileStorageConfigInfo",
+ "GcpStorageConfigInfo" to "org.apache.polaris.core.admin.model.GcpStorageConfigInfo",
+ "GrantCatalogRoleRequest" to "org.apache.polaris.core.admin.model.GrantCatalogRoleRequest",
+ "GrantPrincipalRoleRequest" to
+ "org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest",
+ "GrantResource" to "org.apache.polaris.core.admin.model.GrantResource",
+ "GrantResources" to "org.apache.polaris.core.admin.model.GrantResources",
+ "NamespaceGrant" to "org.apache.polaris.core.admin.model.NamespaceGrant",
+ "NamespacePrivilege" to "org.apache.polaris.core.admin.model.NamespacePrivilege",
+ "PolarisCatalog" to "org.apache.polaris.core.admin.model.PolarisCatalog",
+ "Principal" to "org.apache.polaris.core.admin.model.Principal",
+ "PrincipalRole" to "org.apache.polaris.core.admin.model.PrincipalRole",
+ "PrincipalRoles" to "org.apache.polaris.core.admin.model.PrincipalRoles",
+ "PrincipalWithCredentials" to "org.apache.polaris.core.admin.model.PrincipalWithCredentials",
+ "PrincipalWithCredentialsCredentials" to
+ "org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials",
+ "Principals" to "org.apache.polaris.core.admin.model.Principals",
+ "RevokeGrantRequest" to "org.apache.polaris.core.admin.model.RevokeGrantRequest",
+ "StorageConfigInfo" to "org.apache.polaris.core.admin.model.StorageConfigInfo",
+ "TableGrant" to "org.apache.polaris.core.admin.model.TableGrant",
+ "TablePrivilege" to "org.apache.polaris.core.admin.model.TablePrivilege",
+ "UpdateCatalogRequest" to "org.apache.polaris.core.admin.model.UpdateCatalogRequest",
+ "UpdateCatalogRoleRequest" to "org.apache.polaris.core.admin.model.UpdateCatalogRoleRequest",
+ "UpdatePrincipalRequest" to "org.apache.polaris.core.admin.model.UpdatePrincipalRequest",
+ "UpdatePrincipalRoleRequest" to
+ "org.apache.polaris.core.admin.model.UpdatePrincipalRoleRequest",
+ "ViewGrant" to "org.apache.polaris.core.admin.model.ViewGrant",
+ "ViewPrivilege" to "org.apache.polaris.core.admin.model.ViewPrivilege",
+ )
+ )
+}
+
+tasks.named("compileJava") { dependsOn("generatePolarisManagementClient") }
+
+sourceSets.main { java.srcDir("${layout.buildDirectory.get()}/generated/src/main/java") }
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java
new file mode 100644
index 0000000..20d3534
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizer.java
@@ -0,0 +1,1287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import org.apache.iceberg.BaseTable;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.polaris.core.admin.model.Catalog;
+import org.apache.polaris.core.admin.model.CatalogRole;
+import org.apache.polaris.core.admin.model.GrantResource;
+import org.apache.polaris.core.admin.model.Principal;
+import org.apache.polaris.core.admin.model.PrincipalRole;
+import org.apache.polaris.core.admin.model.PrincipalWithCredentials;
+import org.apache.polaris.tools.sync.polaris.catalog.BaseTableWithETag;
+import org.apache.polaris.tools.sync.polaris.catalog.ETagManager;
+import org.apache.polaris.tools.sync.polaris.catalog.MetadataNotModifiedException;
+import org.apache.polaris.tools.sync.polaris.planning.SynchronizationPlanner;
+import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan;
+import org.apache.polaris.tools.sync.polaris.service.IcebergCatalogService;
+import org.apache.polaris.tools.sync.polaris.service.PolarisService;
+import org.apache.polaris.tools.sync.polaris.service.impl.PolarisIcebergCatalogService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Encapsulates idempotent and failure-safe logic to perform Polaris entity syncs. Performs logging
+ * with configurability and all actions related to the generated sync plans.
+ */
+public class PolarisSynchronizer {
+
+ private final Logger clientLogger;
+
+ private final SynchronizationPlanner syncPlanner;
+
+ private final PolarisService source;
+
+ private final PolarisService target;
+
+ private final ETagManager etagManager;
+
+ private final boolean haltOnFailure;
+
+ public PolarisSynchronizer(
+ Logger clientLogger,
+ boolean haltOnFailure,
+ SynchronizationPlanner synchronizationPlanner,
+ PolarisService source,
+ PolarisService target,
+ ETagManager etagManager) {
+ this.clientLogger =
+ clientLogger == null ? LoggerFactory.getLogger(PolarisSynchronizer.class) : clientLogger;
+ this.haltOnFailure = haltOnFailure;
+ this.syncPlanner = synchronizationPlanner;
+ this.source = source;
+ this.target = target;
+ this.etagManager = etagManager;
+ }
+
+ /**
+ * Calculates the total number of sync tasks to complete.
+ *
+ * @param plan the plan to scan for cahnges
+ * @return the nuber of syncs to perform
+ */
+ private int totalSyncsToComplete(SynchronizationPlan<?> plan) {
+ return plan.entitiesToCreate().size()
+ + plan.entitiesToOverwrite().size()
+ + plan.entitiesToRemove().size();
+ }
+
+ /** Sync principals from source to target. */
+ public void syncPrincipals() {
+ List<Principal> principalsSource;
+
+ try {
+ principalsSource = source.listPrincipals();
+ clientLogger.info("Listed {} principals from source.", principalsSource.size());
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.info("Failed to list principals from source.", e);
+ return;
+ }
+
+ List<Principal> principalsTarget;
+
+ try {
+ principalsTarget = target.listPrincipals();
+ clientLogger.info("Listed {} principals from target.", principalsTarget.size());
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.info("Failed to list principals from target.", e);
+ return;
+ }
+
+ SynchronizationPlan<Principal> principalSyncPlan =
+ syncPlanner.planPrincipalSync(principalsSource, principalsTarget);
+
+ principalSyncPlan
+ .entitiesToSkipAndSkipChildren()
+ .forEach(
+ principal ->
+ clientLogger.info("Skipping principal {}.", principal.getName()));
+
+ principalSyncPlan
+ .entitiesNotModified()
+ .forEach(
+ principal ->
+ clientLogger.info(
+ "No change detected for principal {}, skipping.",
+ principal.getName()));
+
+ int syncsCompleted = 0;
+ final int totalSyncsToComplete = totalSyncsToComplete(principalSyncPlan);
+
+ for (Principal principal : principalSyncPlan.entitiesToCreate()) {
+ try {
+ PrincipalWithCredentials createdPrincipal = target.createPrincipal(principal);
+ clientLogger.info("Created principal {} on target. Target credentials: {}:{} - {}/{}",
+ principal.getName(),
+ createdPrincipal.getCredentials().getClientId(),
+ createdPrincipal.getCredentials().getClientSecret(),
+ ++syncsCompleted,
+ totalSyncsToComplete
+ );
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error("Failed to create principal {} on target. - {}/{}",
+ principal.getName(), ++syncsCompleted, totalSyncsToComplete, e);
+ }
+ }
+
+ for (Principal principal : principalSyncPlan.entitiesToOverwrite()) {
+ try {
+ target.dropPrincipal(principal.getName());
+ PrincipalWithCredentials overwrittenPrincipal = target.createPrincipal(principal);
+ clientLogger.info("Overwrote principal {} on target. Target credentials: {}:{} - {}/{}",
+ principal.getName(),
+ overwrittenPrincipal.getCredentials().getClientId(),
+ overwrittenPrincipal.getCredentials().getClientSecret(),
+ ++syncsCompleted,
+ totalSyncsToComplete
+ );
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error("Failed to overwrite principal {} on target. - {}/{}",
+ principal.getName(), ++syncsCompleted, totalSyncsToComplete, e);
+ }
+ }
+
+ for (Principal principal : principalSyncPlan.entitiesToRemove()) {
+ try {
+ target.dropPrincipal(principal.getName());
+ clientLogger.info("Removed principal {} on target. - {}/{}",
+ principal.getName(), ++syncsCompleted, totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error("Failed to remove principal {} ont target. - {}/{}",
+ principal.getName(), ++syncsCompleted, totalSyncsToComplete, e);
+ }
+ }
+
+ for (Principal principal : principalSyncPlan.entitiesToSyncChildren()) {
+ syncAssignedPrincipalRolesForPrincipal(principal.getName());
+ }
+ }
+
+ /**
+ * Synchronize assigned principal roles for a principal from source to target.
+ * @param principalName the name of the principal to synchronize the principal roles for
+ */
+ public void syncAssignedPrincipalRolesForPrincipal(String principalName) {
+ List<PrincipalRole> assignedPrincipalRolesSource;
+
+ try {
+ assignedPrincipalRolesSource = source.listPrincipalRolesAssigned(principalName);
+ clientLogger.info("Listed {} assigned principal-roles for principal {} from source.",
+ assignedPrincipalRolesSource.size(), principalName);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error("Failed to list assigned principal-roles for principal {} from source.", principalName, e);
+ return;
+ }
+
+ List<PrincipalRole> assignedPrincipalRolesTarget;
+
+ try {
+ assignedPrincipalRolesTarget = target.listPrincipalRolesAssigned(principalName);
+ clientLogger.info("Listed {} assigned principal-roles for principal {} from target.",
+ assignedPrincipalRolesTarget.size(), principalName);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error("Failed to list assigned principal-roles for principal {} from target.", principalName, e);
+ return;
+ }
+
+ SynchronizationPlan<PrincipalRole> assignedPrincipalRoleSyncPlan =
+ syncPlanner.planAssignPrincipalsToPrincipalRolesSync(
+ principalName, assignedPrincipalRolesSource, assignedPrincipalRolesTarget);
+
+ assignedPrincipalRoleSyncPlan
+ .entitiesToSkip()
+ .forEach(
+ principalRole ->
+ clientLogger.info("Skipping assignment of principal-role {} to principal {}.",
+ principalName, principalRole.getName()));
+
+ assignedPrincipalRoleSyncPlan
+ .entitiesNotModified()
+ .forEach(
+ principalRole ->
+ clientLogger.info(
+ "Principal {} is already assigned to principal-role {}, skipping.",
+ principalName, principalRole.getName()));
+
+ int syncsCompleted = 0;
+ final int totalSyncsToComplete = totalSyncsToComplete(assignedPrincipalRoleSyncPlan);
+
+ for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToCreate()) {
+ try {
+ target.assignPrincipalRole(principalName, principalRole.getName());
+ clientLogger.info("Assigned principal-role {} to principal {}. - {}/{}",
+ principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error("Failed to assign principal-role {} to principal {}. - {}/{}",
+ principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete);
+ }
+ }
+
+ for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToOverwrite()) {
+ try {
+ target.assignPrincipalRole(principalName, principalRole.getName());
+ clientLogger.info("Assigned principal-role {} to principal {}. - {}/{}",
+ principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error("Failed to assign principal-role {} to principal {}. - {}/{}",
+ principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete);
+ }
+ }
+
+ for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToRemove()) {
+ try {
+ target.revokePrincipalRole(principalName, principalRole.getName());
+ clientLogger.info("Revoked principal-role {} from principal {}. - {}/{}",
+ principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error("Failed to revoke principal-role {} to principal {}. - {}/{}",
+ principalRole.getName(), principalName, ++syncsCompleted, totalSyncsToComplete);
+ }
+ }
+ }
+
+ /** Sync principal roles from source to target. */
+ public void syncPrincipalRoles() {
+ List<PrincipalRole> principalRolesSource;
+
+ try {
+ principalRolesSource = source.listPrincipalRoles();
+ clientLogger.info("Listed {} principal-roles from source.", principalRolesSource.size());
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error("Failed to list principal-roles from source.", e);
+ return;
+ }
+
+ List<PrincipalRole> principalRolesTarget;
+
+ try {
+ principalRolesTarget = target.listPrincipalRoles();
+ clientLogger.info("Listed {} principal-roles from target.", principalRolesTarget.size());
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error("Failed to list principal-roles from target.", e);
+ return;
+ }
+
+ SynchronizationPlan<PrincipalRole> principalRoleSyncPlan =
+ syncPlanner.planPrincipalRoleSync(principalRolesSource, principalRolesTarget);
+
+ principalRoleSyncPlan
+ .entitiesToSkip()
+ .forEach(
+ principalRole ->
+ clientLogger.info("Skipping principal-role {}.", principalRole.getName()));
+
+ principalRoleSyncPlan
+ .entitiesNotModified()
+ .forEach(
+ principalRole ->
+ clientLogger.info(
+ "No change detected for principal-role {}, skipping.",
+ principalRole.getName()));
+
+ int syncsCompleted = 0;
+ final int totalSyncsToComplete = totalSyncsToComplete(principalRoleSyncPlan);
+
+ for (PrincipalRole principalRole : principalRoleSyncPlan.entitiesToCreate()) {
+ try {
+ target.createPrincipalRole(principalRole);
+ clientLogger.info(
+ "Created principal-role {} on target. - {}/{}",
+ principalRole.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to create principal-role {} on target. - {}/{}",
+ principalRole.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (PrincipalRole principalRole : principalRoleSyncPlan.entitiesToOverwrite()) {
+ try {
+ target.dropPrincipalRole(principalRole.getName());
+ target.createPrincipalRole(principalRole);
+ clientLogger.info(
+ "Overwrote principal-role {} on target. - {}/{}",
+ principalRole.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to overwrite principal-role {} on target. - {}/{}",
+ principalRole.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (PrincipalRole principalRole : principalRoleSyncPlan.entitiesToRemove()) {
+ try {
+ target.dropPrincipalRole(principalRole.getName());
+ clientLogger.info(
+ "Removed principal-role {} on target. - {}/{}",
+ principalRole.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to remove principal-role {} on target. - {}/{}",
+ principalRole.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+ }
+
+ /**
+ * Sync assignments of principal roles to a catalog role.
+ *
+ * @param catalogName the catalog that the catalog role is in
+ * @param catalogRoleName the name of the catalog role
+ */
+ public void syncAssigneePrincipalRolesForCatalogRole(String catalogName, String catalogRoleName) {
+ List<PrincipalRole> principalRolesSource;
+
+ try {
+ principalRolesSource =
+ source.listAssigneePrincipalRolesForCatalogRole(catalogName, catalogRoleName);
+ clientLogger.info(
+ "Listed {} assignee principal-roles for catalog-role {} in catalog {} from source.",
+ principalRolesSource.size(),
+ catalogRoleName,
+ catalogName);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to list assignee principal-roles for catalog-role {} in catalog {} from source.",
+ catalogRoleName,
+ catalogName,
+ e);
+ return;
+ }
+
+ List<PrincipalRole> principalRolesTarget;
+
+ try {
+ principalRolesTarget =
+ target.listAssigneePrincipalRolesForCatalogRole(catalogName, catalogRoleName);
+ clientLogger.info(
+ "Listed {} assignee principal-roles for catalog-role {} in catalog {} from target.",
+ principalRolesTarget.size(),
+ catalogRoleName,
+ catalogName);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to list assignee principal-roles for catalog-role {} in catalog {} from target.",
+ catalogRoleName,
+ catalogName,
+ e);
+ return;
+ }
+
+ SynchronizationPlan<PrincipalRole> assignedPrincipalRoleSyncPlan =
+ syncPlanner.planAssignPrincipalRolesToCatalogRolesSync(
+ catalogName, catalogRoleName, principalRolesSource, principalRolesTarget);
+
+ assignedPrincipalRoleSyncPlan
+ .entitiesToSkip()
+ .forEach(
+ principalRole ->
+ clientLogger.info(
+ "Skipping assignment of principal-role {} to catalog-role {} in catalog {}.",
+ principalRole.getName(),
+ catalogRoleName,
+ catalogName));
+
+ assignedPrincipalRoleSyncPlan
+ .entitiesNotModified()
+ .forEach(
+ principalRole ->
+ clientLogger.info(
+ "Principal-role {} is already assigned to catalog-role {} in catalog {}. Skipping.",
+ principalRole.getName(),
+ catalogRoleName,
+ catalogName));
+
+ int syncsCompleted = 0;
+ int totalSyncsToComplete = totalSyncsToComplete(assignedPrincipalRoleSyncPlan);
+
+ for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToCreate()) {
+ try {
+ target.assignCatalogRole(
+ principalRole.getName(), catalogName, catalogRoleName);
+ clientLogger.info(
+ "Assigned principal-role {} to catalog-role {} in catalog {}. - {}/{}",
+ principalRole.getName(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to assign principal-role {} to catalog-role {} in catalog {}. - {}/{}",
+ principalRole.getName(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToOverwrite()) {
+ try {
+ target.assignCatalogRole(
+ principalRole.getName(), catalogName, catalogRoleName);
+ clientLogger.info(
+ "Assigned principal-role {} to catalog-role {} in catalog {}. - {}/{}",
+ principalRole.getName(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to assign principal-role {} to catalog-role {} in catalog {}. - {}/{}",
+ principalRole.getName(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (PrincipalRole principalRole : assignedPrincipalRoleSyncPlan.entitiesToRemove()) {
+ try {
+ target.revokeCatalogRole(
+ principalRole.getName(), catalogName, catalogRoleName);
+ clientLogger.info(
+ "Revoked principal-role {} from catalog-role {} in catalog {}. - {}/{}",
+ principalRole.getName(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to revoke principal-role {} from catalog-role {} in catalog {}. - {}/{}",
+ principalRole.getName(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+ }
+
+ /** Sync catalogs across the source and target polaris instance. */
+ public void syncCatalogs() {
+ List<Catalog> catalogsSource;
+
+ try {
+ catalogsSource = source.listCatalogs();
+ clientLogger.info("Listed {} catalogs from source.", catalogsSource.size());
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error("Failed to list catalogs from source.", e);
+ return;
+ }
+
+ List<Catalog> catalogsTarget;
+
+ try {
+ catalogsTarget = target.listCatalogs();
+ clientLogger.info("Listed {} catalogs from target.", catalogsTarget.size());
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error("Failed to list catalogs from target.", e);
+ return;
+ }
+
+ SynchronizationPlan<Catalog> catalogSyncPlan =
+ syncPlanner.planCatalogSync(catalogsSource, catalogsTarget);
+
+ catalogSyncPlan
+ .entitiesToSkip()
+ .forEach(catalog -> clientLogger.info("Skipping catalog {}.", catalog.getName()));
+
+ catalogSyncPlan
+ .entitiesToSkipAndSkipChildren()
+ .forEach(
+ catalog ->
+ clientLogger.info(
+ "Skipping catalog {} and all child entities.", catalog.getName()));
+
+ catalogSyncPlan
+ .entitiesNotModified()
+ .forEach(
+ catalog ->
+ clientLogger.info(
+ "No change detected in catalog {}. Skipping.", catalog.getName()));
+
+ int syncsCompleted = 0;
+ int totalSyncsToComplete = totalSyncsToComplete(catalogSyncPlan);
+
+ for (Catalog catalog : catalogSyncPlan.entitiesToCreate()) {
+ try {
+ target.createCatalog(catalog);
+ clientLogger.info(
+ "Created catalog {}. - {}/{}",
+ catalog.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to create catalog {}. - {}/{}",
+ catalog.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (Catalog catalog : catalogSyncPlan.entitiesToOverwrite()) {
+ try {
+ target.dropCatalogCascade(catalog.getName());
+ target.createCatalog(catalog);
+ clientLogger.info(
+ "Overwrote catalog {}. - {}/{}",
+ catalog.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to overwrite catalog {}. - {}/{}",
+ catalog.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (Catalog catalog : catalogSyncPlan.entitiesToRemove()) {
+ try {
+ target.dropCatalogCascade(catalog.getName());
+ clientLogger.info(
+ "Removed catalog {}. - {}/{}",
+ catalog.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to remove catalog {}. - {}/{}",
+ catalog.getName(),
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (Catalog catalog : catalogSyncPlan.entitiesToSyncChildren()) {
+ syncCatalogRoles(catalog.getName());
+
+ IcebergCatalogService sourceIcebergCatalogService;
+
+ try {
+ sourceIcebergCatalogService = source.initializeIcebergCatalogService(catalog.getName());
+ clientLogger.info(
+ "Initialized Iceberg REST catalog for Polaris catalog {} on source.",
+ catalog.getName());
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to initialize Iceberg REST catalog for Polaris catalog {} on source.",
+ catalog.getName(),
+ e);
+ continue;
+ }
+
+ IcebergCatalogService targetIcebergCatalogService;
+
+ try {
+ targetIcebergCatalogService = target.initializeIcebergCatalogService(catalog.getName());
+ clientLogger.info(
+ "Initialized Iceberg REST catalog for Polaris catalog {} on target.",
+ catalog.getName());
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to initialize Iceberg REST catalog for Polaris catalog {} on target.",
+ catalog.getName(),
+ e);
+ continue;
+ }
+
+ syncNamespaces(
+ catalog.getName(), Namespace.empty(), sourceIcebergCatalogService, targetIcebergCatalogService);
+ }
+ }
+
+ /**
+ * Sync catalog roles across the source and polaris instance for a catalog.
+ *
+ * @param catalogName the catalog to sync roles for
+ */
+ public void syncCatalogRoles(String catalogName) {
+ List<CatalogRole> catalogRolesSource;
+
+ try {
+ catalogRolesSource = source.listCatalogRoles(catalogName);
+ clientLogger.info(
+ "Listed {} catalog-roles for catalog {} from source.",
+ catalogRolesSource.size(),
+ catalogName);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to list catalog-roles for catalog {} from source.", catalogName, e);
+ return;
+ }
+
+ List<CatalogRole> catalogRolesTarget;
+
+ try {
+ catalogRolesTarget = target.listCatalogRoles(catalogName);
+ clientLogger.info(
+ "Listed {} catalog-roles for catalog {} from target.",
+ catalogRolesTarget.size(),
+ catalogName);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to list catalog-roles for catalog {} from target.", catalogName, e);
+ return;
+ }
+
+ SynchronizationPlan<CatalogRole> catalogRoleSyncPlan =
+ syncPlanner.planCatalogRoleSync(catalogName, catalogRolesSource, catalogRolesTarget);
+
+ catalogRoleSyncPlan
+ .entitiesToSkip()
+ .forEach(
+ catalogRole ->
+ clientLogger.info(
+ "Skipping catalog-role {} in catalog {}.", catalogRole.getName(), catalogName));
+
+ catalogRoleSyncPlan
+ .entitiesToSkipAndSkipChildren()
+ .forEach(
+ catalogRole ->
+ clientLogger.info(
+ "Skipping catalog-role {} in catalog {} and all child entities.",
+ catalogRole.getName(),
+ catalogName));
+
+ catalogRoleSyncPlan
+ .entitiesNotModified()
+ .forEach(
+ catalogRole ->
+ clientLogger.info(
+ "No change detected in catalog-role {} in catalog {}. Skipping.",
+ catalogRole.getName(),
+ catalogName));
+
+ int syncsCompleted = 0;
+ int totalSyncsToComplete = totalSyncsToComplete(catalogRoleSyncPlan);
+
+ for (CatalogRole catalogRole : catalogRoleSyncPlan.entitiesToCreate()) {
+ try {
+ target.createCatalogRole(catalogName, catalogRole);
+ clientLogger.info(
+ "Created catalog-role {} for catalog {}. - {}/{}",
+ catalogRole.getName(),
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to create catalog-role {} for catalog {}. - {}/{}",
+ catalogRole.getName(),
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (CatalogRole catalogRole : catalogRoleSyncPlan.entitiesToOverwrite()) {
+ try {
+ target.dropCatalogRole(catalogName, catalogRole.getName());
+ target.createCatalogRole(catalogName, catalogRole);
+ clientLogger.info(
+ "Overwrote catalog-role {} for catalog {}. - {}/{}",
+ catalogRole.getName(),
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to overwrite catalog-role {} for catalog {}. - {}/{}",
+ catalogRole.getName(),
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (CatalogRole catalogRole : catalogRoleSyncPlan.entitiesToRemove()) {
+ try {
+ target.dropCatalogRole(catalogName, catalogRole.getName());
+ clientLogger.info(
+ "Removed catalog-role {} for catalog {}. - {}/{}",
+ catalogRole.getName(),
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to remove catalog-role {} for catalog {}. - {}/{}",
+ catalogRole.getName(),
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (CatalogRole catalogRole : catalogRoleSyncPlan.entitiesToSyncChildren()) {
+ syncAssigneePrincipalRolesForCatalogRole(catalogName, catalogRole.getName());
+ syncGrants(catalogName, catalogRole.getName());
+ }
+ }
+
+ /**
+ * Sync grants for a catalog role across the source and the target.
+ *
+ * @param catalogName
+ * @param catalogRoleName
+ */
+ private void syncGrants(String catalogName, String catalogRoleName) {
+ List<GrantResource> grantsSource;
+
+ try {
+ grantsSource = source.listGrants(catalogName, catalogRoleName);
+ clientLogger.info(
+ "Listed {} grants for catalog-role {} in catalog {} from source.",
+ grantsSource.size(),
+ catalogRoleName,
+ catalogName);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to list grants for catalog-role {} in catalog {} from source.",
+ catalogRoleName,
+ catalogName,
+ e);
+ return;
+ }
+
+ List<GrantResource> grantsTarget;
+
+ try {
+ grantsTarget = target.listGrants(catalogName, catalogRoleName);
+ clientLogger.info(
+ "Listed {} grants for catalog-role {} in catalog {} from target.",
+ grantsTarget.size(),
+ catalogRoleName,
+ catalogName);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to list grants for catalog-role {} in catalog {} from target.",
+ catalogRoleName,
+ catalogName,
+ e);
+ return;
+ }
+
+ SynchronizationPlan<GrantResource> grantSyncPlan =
+ syncPlanner.planGrantSync(catalogName, catalogRoleName, grantsSource, grantsTarget);
+
+ grantSyncPlan
+ .entitiesToSkip()
+ .forEach(
+ grant ->
+ clientLogger.info(
+ "Skipping addition of grant {} to catalog-role {} in catalog {}.",
+ grant.getType(),
+ catalogRoleName,
+ catalogName));
+
+ grantSyncPlan
+ .entitiesNotModified()
+ .forEach(
+ grant ->
+ clientLogger.info(
+ "Grant {} was already added to catalog-role {} in catalog {}. Skipping.",
+ grant.getType(),
+ catalogRoleName,
+ catalogName));
+
+ int syncsCompleted = 0;
+ int totalSyncsToComplete = totalSyncsToComplete(grantSyncPlan);
+
+ for (GrantResource grant : grantSyncPlan.entitiesToCreate()) {
+ try {
+ target.addGrant(catalogName, catalogRoleName, grant);
+ clientLogger.info(
+ "Added grant {} to catalog-role {} for catalog {}. - {}/{}",
+ grant.getType(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to add grant {} to catalog-role {} for catalog {}. - {}/{}",
+ grant.getType(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (GrantResource grant : grantSyncPlan.entitiesToOverwrite()) {
+ try {
+ target.addGrant(catalogName, catalogRoleName, grant);
+ clientLogger.info(
+ "Added grant {} to catalog-role {} for catalog {}. - {}/{}",
+ grant.getType(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to add grant {} to catalog-role {} for catalog {}. - {}/{}",
+ grant.getType(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (GrantResource grant : grantSyncPlan.entitiesToRemove()) {
+ try {
+ target.revokeGrant(catalogName, catalogRoleName, grant);
+ clientLogger.info(
+ "Revoked grant {} from catalog-role {} for catalog {}. - {}/{}",
+ grant.getType(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to revoke grant {} from catalog-role {} for catalog {}. - {}/{}",
+ grant.getType(),
+ catalogRoleName,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+ }
+
+ /**
+ * Sync namespaces contained within a parent namespace.
+ *
+ * @param catalogName
+ * @param parentNamespace
+ * @param sourceIcebergCatalogService
+ * @param targetIcebergCatalogService
+ */
+ public void syncNamespaces(
+ String catalogName,
+ Namespace parentNamespace,
+ IcebergCatalogService sourceIcebergCatalogService,
+ IcebergCatalogService targetIcebergCatalogService) {
+ List<Namespace> namespacesSource;
+
+ try {
+ namespacesSource = sourceIcebergCatalogService.listNamespaces(parentNamespace);
+ clientLogger.info(
+ "Listed {} namespaces in namespace {} for catalog {} from source.",
+ namespacesSource.size(),
+ parentNamespace,
+ catalogName);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to list namespaces in namespace {} for catalog {} from source.",
+ parentNamespace,
+ catalogName,
+ e);
+ return;
+ }
+
+ List<Namespace> namespacesTarget;
+
+ try {
+ namespacesTarget = targetIcebergCatalogService.listNamespaces(parentNamespace);
+ clientLogger.info(
+ "Listed {} namespaces in namespace {} for catalog {} from target.",
+ namespacesTarget.size(),
+ parentNamespace,
+ catalogName);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to list namespaces in namespace {} for catalog {} from target.",
+ parentNamespace,
+ catalogName,
+ e);
+ return;
+ }
+
+ SynchronizationPlan<Namespace> namespaceSynchronizationPlan =
+ syncPlanner.planNamespaceSync(
+ catalogName, parentNamespace, namespacesSource, namespacesTarget);
+
+ int syncsCompleted = 0;
+ int totalSyncsToComplete = totalSyncsToComplete(namespaceSynchronizationPlan);
+
+ namespaceSynchronizationPlan
+ .entitiesNotModified()
+ .forEach(
+ namespace ->
+ clientLogger.info(
+ "No change detected for namespace {} in namespace {} for catalog {}, skipping.",
+ namespace,
+ parentNamespace,
+ catalogName));
+
+ for (Namespace namespace : namespaceSynchronizationPlan.entitiesToCreate()) {
+ try {
+ Map<String, String> namespaceMetadata = sourceIcebergCatalogService.loadNamespaceMetadata(namespace);
+ targetIcebergCatalogService.createNamespace(namespace, namespaceMetadata);
+ clientLogger.info(
+ "Created namespace {} in namespace {} for catalog {} - {}/{}",
+ namespace,
+ parentNamespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to create namespace {} in namespace {} for catalog {} - {}/{}",
+ namespace,
+ parentNamespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (Namespace namespace : namespaceSynchronizationPlan.entitiesToOverwrite()) {
+ try {
+ Map<String, String> sourceNamespaceMetadata =
+ sourceIcebergCatalogService.loadNamespaceMetadata(namespace);
+ Map<String, String> targetNamespaceMetadata =
+ targetIcebergCatalogService.loadNamespaceMetadata(namespace);
+
+ if (sourceNamespaceMetadata.equals(targetNamespaceMetadata)) {
+ clientLogger.info(
+ "Namespace metadata for namespace {} in namespace {} for catalog {} was not modified, skipping. - {}/{}",
+ namespace,
+ parentNamespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ continue;
+ }
+
+ targetIcebergCatalogService.setNamespaceProperties(namespace, sourceNamespaceMetadata);
+
+ clientLogger.info(
+ "Overwrote namespace metadata {} in namespace {} for catalog {} - {}/{}",
+ namespace,
+ parentNamespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to overwrite namespace metadata {} in namespace {} for catalog {} - {}/{}",
+ namespace,
+ parentNamespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (Namespace namespace : namespaceSynchronizationPlan.entitiesToRemove()) {
+ try {
+ targetIcebergCatalogService.dropNamespaceCascade(namespace);
+ clientLogger.info(
+ "Removed namespace {} in namespace {} for catalog {} - {}/{}",
+ namespace,
+ parentNamespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to remove namespace {} in namespace {} for catalog {} - {}/{}",
+ namespace,
+ parentNamespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (Namespace namespace : namespaceSynchronizationPlan.entitiesToSyncChildren()) {
+ syncTables(catalogName, namespace, sourceIcebergCatalogService, targetIcebergCatalogService);
+ syncNamespaces(catalogName, namespace, sourceIcebergCatalogService, targetIcebergCatalogService);
+ }
+ }
+
+ /**
+ * Sync tables contained within a namespace.
+ *
+ * @param catalogName
+ * @param namespace
+ * @param sourceIcebergCatalogService
+ * @param targetIcebergCatalogService
+ */
+ public void syncTables(
+ String catalogName,
+ Namespace namespace,
+ IcebergCatalogService sourceIcebergCatalogService,
+ IcebergCatalogService targetIcebergCatalogService) {
+ Set<TableIdentifier> sourceTables;
+
+ try {
+ sourceTables = new HashSet<>(sourceIcebergCatalogService.listTables(namespace));
+ clientLogger.info(
+ "Listed {} tables in namespace {} for catalog {} on source.",
+ sourceTables.size(),
+ namespace,
+ catalogName);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to list tables in namespace {} for catalog {} on source.",
+ namespace,
+ catalogName,
+ e);
+ return;
+ }
+
+ Set<TableIdentifier> targetTables;
+
+ try {
+ targetTables = new HashSet<>(targetIcebergCatalogService.listTables(namespace));
+ clientLogger.info(
+ "Listed {} tables in namespace {} for catalog {} on target.",
+ targetTables.size(),
+ namespace,
+ catalogName);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to list tables in namespace {} for catalog {} on target.",
+ namespace,
+ catalogName,
+ e);
+ return;
+ }
+
+ SynchronizationPlan<TableIdentifier> tableSyncPlan =
+ syncPlanner.planTableSync(catalogName, namespace, sourceTables, targetTables);
+
+ tableSyncPlan
+ .entitiesToSkip()
+ .forEach(
+ tableId ->
+ clientLogger.info(
+ "Skipping table {} in namespace {} in catalog {}.",
+ tableId,
+ namespace,
+ catalogName));
+
+ int syncsCompleted = 0;
+ int totalSyncsToComplete = totalSyncsToComplete(tableSyncPlan);
+
+ for (TableIdentifier tableId : tableSyncPlan.entitiesToCreate()) {
+ try {
+ Table table = sourceIcebergCatalogService.loadTable(tableId);
+
+ if (table instanceof BaseTable baseTable) {
+ targetIcebergCatalogService.registerTable(
+ tableId, baseTable.operations().current().metadataFileLocation());
+ } else {
+ throw new IllegalStateException("Cannot register table that does not extend BaseTable.");
+ }
+
+ if (table instanceof BaseTableWithETag tableWithETag) {
+ etagManager.storeETag(catalogName, tableId, tableWithETag.etag());
+ }
+
+ clientLogger.info(
+ "Registered table {} in namespace {} in catalog {}. - {}/{}",
+ tableId,
+ namespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to register table {} in namespace {} in catalog {}. - {}/{}",
+ tableId,
+ namespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (TableIdentifier tableId : tableSyncPlan.entitiesToOverwrite()) {
+ try {
+ Table table;
+
+ if (sourceIcebergCatalogService instanceof PolarisIcebergCatalogService polarisCatalogService) {
+ String etag = etagManager.getETag(catalogName, tableId);
+ table = polarisCatalogService.loadTable(tableId, etag);
+ } else {
+ table = sourceIcebergCatalogService.loadTable(tableId);
+ }
+
+ if (table instanceof BaseTable baseTable) {
+ targetIcebergCatalogService.dropTableWithoutPurge(tableId);
+ targetIcebergCatalogService.registerTable(
+ tableId, baseTable.operations().current().metadataFileLocation());
+ } else {
+ throw new IllegalStateException("Cannot register table that does not extend BaseTable.");
+ }
+
+ if (table instanceof BaseTableWithETag tableWithETag) {
+ etagManager.storeETag(catalogName, tableId, tableWithETag.etag());
+ }
+
+ clientLogger.info(
+ "Dropped and re-registered table {} in namespace {} in catalog {}. - {}/{}",
+ tableId,
+ namespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (MetadataNotModifiedException e) {
+ clientLogger.info(
+ "Table {} in namespace {} in catalog {} with was not modified, not overwriting in target catalog. - {}/{}",
+ tableId,
+ namespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.error(
+ "Failed to drop and re-register table {} in namespace {} in catalog {}. - {}/{}",
+ tableId,
+ namespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+
+ for (TableIdentifier table : tableSyncPlan.entitiesToRemove()) {
+ try {
+ targetIcebergCatalogService.dropTableWithoutPurge(table);
+ clientLogger.info(
+ "Dropped table {} in namespace {} in catalog {}. - {}/{}",
+ table,
+ namespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete);
+ } catch (Exception e) {
+ if (haltOnFailure) throw e;
+ clientLogger.info(
+ "Failed to drop table {} in namespace {} in catalog {}. - {}/{}",
+ table,
+ namespace,
+ catalogName,
+ ++syncsCompleted,
+ totalSyncsToComplete,
+ e);
+ }
+ }
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlConstants.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlConstants.java
new file mode 100644
index 0000000..fc0b978
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlConstants.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.access;
+
+/**
+ * Constants related to access control logic.
+ */
+public class AccessControlConstants {
+
+ /**
+ * Property used to identify the omnipotent principal and all related access control entities
+ * across the source and target.
+ */
+ public static final String OMNIPOTENCE_PROPERTY = "IS_OMNIPOTENT_PRINCIPAL";
+
+ /**
+ * Prefix to specify for naming an entity related to the omnipotent principal.
+ */
+ protected static final String OMNIPOTENT_PRINCIPAL_NAME_PREFIX = "omnipotent-principal-";
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlService.java
new file mode 100644
index 0000000..8e9896a
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/access/AccessControlService.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.access;
+
+import static org.apache.polaris.core.admin.model.CatalogPrivilege.CATALOG_MANAGE_METADATA;
+import static org.apache.polaris.core.admin.model.CatalogPrivilege.CATALOG_READ_PROPERTIES;
+import static org.apache.polaris.core.admin.model.CatalogPrivilege.NAMESPACE_LIST;
+import static org.apache.polaris.core.admin.model.CatalogPrivilege.NAMESPACE_READ_PROPERTIES;
+import static org.apache.polaris.core.admin.model.CatalogPrivilege.TABLE_LIST;
+import static org.apache.polaris.core.admin.model.CatalogPrivilege.TABLE_READ_PROPERTIES;
+import static org.apache.polaris.core.admin.model.CatalogPrivilege.VIEW_LIST;
+import static org.apache.polaris.core.admin.model.CatalogPrivilege.VIEW_READ_PROPERTIES;
+import static org.apache.polaris.tools.sync.polaris.access.AccessControlConstants.OMNIPOTENCE_PROPERTY;
+import static org.apache.polaris.tools.sync.polaris.access.AccessControlConstants.OMNIPOTENT_PRINCIPAL_NAME_PREFIX;
+
+import java.util.List;
+import java.util.NoSuchElementException;
+import org.apache.polaris.core.admin.model.CatalogGrant;
+import org.apache.polaris.core.admin.model.CatalogRole;
+import org.apache.polaris.core.admin.model.GrantResource;
+import org.apache.polaris.core.admin.model.Principal;
+import org.apache.polaris.core.admin.model.PrincipalRole;
+import org.apache.polaris.core.admin.model.PrincipalWithCredentials;
+import org.apache.polaris.tools.sync.polaris.service.impl.PolarisApiService;
+
+/**
+ * Service class to facilitate the access control needs of the synchronization. This involves
+ * setting up principals, principal roles, catalog roles, and grants to allow the tool to be able to
+ * introspect into catalog internals like catalog-roles, tables, grants.
+ */
+public class AccessControlService {
+
+ private final PolarisApiService polaris;
+
+ public AccessControlService(PolarisApiService polaris) {
+ this.polaris = polaris;
+ }
+
+ /**
+ * Creates or replaces the existing omnipotent principal on the provided polaris instance.
+ *
+ * @param replace if true, if an existing omnipotent principal role exists, it will be dropped and
+ * recreated
+ * @return the principal and credentials for the omnipotent principal
+ */
+ public PrincipalWithCredentials createOmnipotentPrincipal(boolean replace) {
+ List<Principal> principals = polaris.listPrincipals();
+
+ Principal omnipotentPrincipalPrototype =
+ new Principal()
+ .name(OMNIPOTENT_PRINCIPAL_NAME_PREFIX + System.currentTimeMillis())
+ .putPropertiesItem(
+ OMNIPOTENCE_PROPERTY, ""); // this property identifies the omnipotent principal
+
+ for (Principal principal : principals) {
+ if (principal.getProperties() != null
+ && principal.getProperties().containsKey(OMNIPOTENCE_PROPERTY)) {
+ if (replace) {
+ // drop existing omnipotent principal in preparation for replacement
+ polaris.dropPrincipal(principal.getName());
+ } else {
+ // we cannot create another omnipotent principal and cannot replace the existing, fail
+ throw new IllegalStateException(
+ "Not permitted to replace existing omnipotent principal, but omnipotent "
+ + "principal with property "
+ + OMNIPOTENCE_PROPERTY
+ + " already exists");
+ }
+ }
+ }
+
+ // existing principal with identifying property does not exist, create a new one
+ return polaris.createPrincipal(omnipotentPrincipalPrototype);
+ }
+
+ /**
+ * Retrieves the omnipotent principal role for the provided principalName.
+ *
+ * @param principalName the principal name to search for roles with
+ * @return the principal role for the provided principal, if exists
+ */
+ public PrincipalRole getOmnipotentPrincipalRoleForPrincipal(String principalName) {
+ List<PrincipalRole> principalRolesAssigned =
+ polaris.listPrincipalRolesAssigned(principalName);
+
+ return principalRolesAssigned.stream()
+ .filter(
+ principalRole ->
+ principalRole.getProperties() != null
+ && principalRole.getProperties().containsKey(OMNIPOTENCE_PROPERTY))
+ .findFirst()
+ .orElseThrow(
+ () ->
+ new NoSuchElementException(
+ "No omnipotent principal role with property "
+ + OMNIPOTENCE_PROPERTY
+ + " exists for principal "
+ + principalName));
+ }
+
+ /**
+ * Creates a principal role for the omnipotent principal and assigns it to the provided omnipotent
+ * principal.
+ *
+ * @param omnipotentPrincipal the principal to create and assign the role for
+ * @param replace if true, drops existing omnipotent principal roles if they exist before creating
+ * the new one
+ * @return the principal role for the omnipotent principal
+ */
+ public PrincipalRole createAndAssignPrincipalRole(
+ PrincipalWithCredentials omnipotentPrincipal, boolean replace) {
+ List<PrincipalRole> principalRoles = polaris.listPrincipalRoles();
+
+ PrincipalRole omnipotentPrincipalRole =
+ new PrincipalRole()
+ .name(omnipotentPrincipal.getPrincipal().getName())
+ .putPropertiesItem(OMNIPOTENCE_PROPERTY, "");
+
+ for (PrincipalRole principalRole : principalRoles) {
+ if (principalRole.getProperties() != null
+ && principalRole.getProperties().containsKey(OMNIPOTENCE_PROPERTY)) {
+ // replace existing principal role if exists
+ if (replace) {
+ polaris.dropPrincipalRole(principalRole.getName());
+ } else {
+ throw new IllegalStateException(
+ "Not permitted to replace existing omnipotent principal role, but omnipotent "
+ + "principal role with property "
+ + OMNIPOTENCE_PROPERTY
+ + " already exists");
+ }
+ }
+ }
+
+ polaris.createPrincipalRole(omnipotentPrincipalRole);
+ polaris.assignPrincipalRole(omnipotentPrincipal.getPrincipal().getName(), omnipotentPrincipalRole.getName());
+ return omnipotentPrincipalRole;
+ }
+
+ /**
+ * Creates an omnipotent catalog role for a catalog and assigns it to the provided omnipotent
+ * principal role.
+ *
+ * @param catalogName the catalog to create the catalog role for
+ * @param omnipotentPrincipalRole the omnipotent principal role to assign the created catalog role
+ * to
+ * @param replace if true, drops and recreates the existing omnipotent catalog role
+ * @return the created omnipotent catalog role
+ */
+ public CatalogRole createAndAssignCatalogRole(
+ String catalogName, PrincipalRole omnipotentPrincipalRole, boolean replace) {
+ List<CatalogRole> catalogRoles = polaris.listCatalogRoles(catalogName);
+
+ for (CatalogRole catalogRole : catalogRoles) {
+ if (catalogRole.getProperties() != null
+ && catalogRole.getProperties().containsKey(OMNIPOTENCE_PROPERTY)) {
+ if (replace) {
+ polaris.dropCatalogRole(catalogName, catalogRole.getName());
+ } else {
+ throw new IllegalStateException(
+ "Not permitted to replace existing omnipotent catalog role for catalog "
+ + catalogName
+ + ", but omnipotent principal with property "
+ + OMNIPOTENCE_PROPERTY
+ + " already exists");
+ }
+ }
+ }
+
+ CatalogRole omnipotentCatalogRole =
+ new CatalogRole()
+ .name(omnipotentPrincipalRole.getName())
+ .putPropertiesItem(OMNIPOTENCE_PROPERTY, "");
+
+ polaris.createCatalogRole(catalogName, omnipotentCatalogRole);
+ polaris.assignCatalogRole(omnipotentPrincipalRole.getName(), catalogName, omnipotentCatalogRole.getName());
+ return omnipotentCatalogRole;
+ }
+
+ /**
+ * Adds grants for privilege level desired on the omnipotent catalog role.
+ *
+ * @param catalogName the catalog to identify the role in
+ * @param catalogRoleName the name of the catalog role to assign the grants tpo
+ * @param withWriteAccess if the catalog role should be given write access to the catalog
+ * internals
+ * @return the grants that were added to the catalog role
+ */
+ public List<CatalogGrant> addGrantsToCatalogRole(
+ String catalogName, String catalogRoleName, boolean withWriteAccess) {
+ if (withWriteAccess) {
+ // write access only requires CATALOG_MANAGE_METADATA
+ CatalogGrant catalogManageMetadata =
+ new CatalogGrant()
+ .type(GrantResource.TypeEnum.CATALOG)
+ .privilege(CATALOG_MANAGE_METADATA);
+
+ polaris.addGrant(catalogName, catalogRoleName, catalogManageMetadata);
+ return List.of(catalogManageMetadata);
+ } else {
+ // read access requires reading properties and listing entities for each entity type
+ CatalogGrant catalogReadProperties =
+ new CatalogGrant()
+ .type(GrantResource.TypeEnum.CATALOG)
+ .privilege(CATALOG_READ_PROPERTIES);
+
+ CatalogGrant namespaceReadProperties =
+ new CatalogGrant()
+ .type(GrantResource.TypeEnum.CATALOG)
+ .privilege(NAMESPACE_READ_PROPERTIES);
+
+ CatalogGrant namespaceList =
+ new CatalogGrant().type(GrantResource.TypeEnum.CATALOG).privilege(NAMESPACE_LIST);
+
+ CatalogGrant tableReadProperties =
+ new CatalogGrant().type(GrantResource.TypeEnum.CATALOG).privilege(TABLE_READ_PROPERTIES);
+
+ CatalogGrant tableList =
+ new CatalogGrant().type(GrantResource.TypeEnum.CATALOG).privilege(TABLE_LIST);
+
+ CatalogGrant viewReadProperties =
+ new CatalogGrant().type(GrantResource.TypeEnum.CATALOG).privilege(VIEW_READ_PROPERTIES);
+
+ CatalogGrant viewList =
+ new CatalogGrant().type(GrantResource.TypeEnum.CATALOG).privilege(VIEW_LIST);
+
+ polaris.addGrant(catalogName, catalogRoleName, catalogReadProperties);
+ polaris.addGrant(catalogName, catalogRoleName, namespaceReadProperties);
+ polaris.addGrant(catalogName, catalogRoleName, namespaceList);
+ polaris.addGrant(catalogName, catalogRoleName, tableReadProperties);
+ polaris.addGrant(catalogName, catalogRoleName, tableList);
+ polaris.addGrant(catalogName, catalogRoleName, viewReadProperties);
+ polaris.addGrant(catalogName, catalogRoleName, viewList);
+ return List.of(
+ catalogReadProperties, namespaceReadProperties, tableReadProperties, viewReadProperties);
+ }
+ }
+
+ /**
+ * Determines if an omnipotent catalog role already exists for this catalog.
+ *
+ * @param catalogName the catalog to search in
+ * @return true if exists, false otherwise
+ */
+ public boolean omnipotentCatalogRoleExists(String catalogName) {
+ List<CatalogRole> catalogRoles = polaris.listCatalogRoles(catalogName);
+
+ return catalogRoles.stream()
+ .anyMatch(
+ catalogRole ->
+ catalogRole.getProperties() != null
+ && catalogRole.getProperties().containsKey(OMNIPOTENCE_PROPERTY));
+ }
+
+ /**
+ * Creates catalog role for catalog, assigns it to provided principal role, and assigns grants
+ * with appropriate privilege level.
+ *
+ * @param catalogName the catalog to create the role for
+ * @param omnipotentPrincipalRole the principal role to assign the catalog role to
+ * @param replace if true, drops the existing catalog role if it exists
+ * @param withWriteAccess gives write access to the catalog role
+ */
+ public void setupOmnipotentRoleForCatalog(
+ String catalogName,
+ PrincipalRole omnipotentPrincipalRole,
+ boolean replace,
+ boolean withWriteAccess) {
+ CatalogRole omniPotentCatalogRole =
+ createAndAssignCatalogRole(catalogName, omnipotentPrincipalRole, replace);
+ addGrantsToCatalogRole(catalogName, omniPotentCatalogRole.getName(), withWriteAccess);
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/BaseTableWithETag.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/BaseTableWithETag.java
new file mode 100644
index 0000000..c4bd498
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/BaseTableWithETag.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.catalog;
+
+import org.apache.iceberg.BaseTable;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.metrics.MetricsReporter;
+
+/**
+ * Wrapper around {@link BaseTable} that contains the latest ETag for the table.
+ *
+ * TODO: Remove this class once Iceberg gets first class support for ETags.
+ * in the canonical response types.
+*/
+public class BaseTableWithETag extends BaseTable {
+
+ private final String etag;
+
+ public BaseTableWithETag(TableOperations ops, String name, String etag) {
+ super(ops, name);
+ this.etag = etag;
+ }
+
+ public BaseTableWithETag(
+ TableOperations ops, String name, MetricsReporter reporter, String etag) {
+ super(ops, name, reporter);
+ this.etag = etag;
+ }
+
+ public String etag() {
+ return etag;
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagManager.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagManager.java
new file mode 100644
index 0000000..832d75c
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/ETagManager.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.catalog;
+
+import org.apache.iceberg.catalog.TableIdentifier;
+
+import java.util.Map;
+
+/**
+ * Generic interface to provide and store ETags for tables within catalogs. This allows the storage
+ * of the ETag to be completely independent from the tool.
+ */
+public interface ETagManager {
+
+ /**
+ * Used to initialize the instance for use. Should be called prior to
+ * calling any methods.
+ * @param properties properties to configure instance with
+ */
+ void initialize(Map<String, String> properties);
+
+ /**
+ * Retrieves the ETag for the table.
+ *
+ * @param catalogName the catalog the table is in
+ * @param tableIdentifier the table identifier
+ * @return The ETag for the last known metadata for the table
+ */
+ String getETag(String catalogName, TableIdentifier tableIdentifier);
+
+ /**
+ * After table loading, stores the fetched ETag.
+ *
+ * @param catalogName the catalog the table is in
+ * @param tableIdentifier the table identifier
+ * @param etag the ETag that was provided by the Iceberg REST api
+ */
+ void storeETag(String catalogName, TableIdentifier tableIdentifier, String etag);
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataNotModifiedException.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataNotModifiedException.java
new file mode 100644
index 0000000..1ca85aa
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataNotModifiedException.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.catalog;
+
+import org.apache.iceberg.catalog.TableIdentifier;
+
+/**
+ * Thrown when the metadata for a particular table was not modified from the version
+ * identified by a provided ETag.
+ *
+ * TODO: Remove once Iceberg has first class support for ETags.
+ */
+public class MetadataNotModifiedException extends RuntimeException {
+
+ public MetadataNotModifiedException(TableIdentifier tableIdentifier) {
+ super("Table " + tableIdentifier + " was not modified.");
+ }
+
+ public MetadataNotModifiedException(String message) {
+ super(message);
+ }
+
+ public MetadataNotModifiedException(String message, Throwable cause) {
+ super(message, cause);
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataWrapperTableOperations.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataWrapperTableOperations.java
new file mode 100644
index 0000000..909cd8c
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/MetadataWrapperTableOperations.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.catalog;
+
+import java.util.NoSuchElementException;
+import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableOperations;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.io.LocationProvider;
+
+/**
+ * Wrapper table operations class that just allows fetching a provided table metadata. Used to build
+ * a {@link org.apache.iceberg.BaseTable} without having to expose a full-fledged operations class.
+ *
+ * TODO: Remove this class once Iceberg gets first class support for ETags.
+ * in the canonical response types.
+ */
+public class MetadataWrapperTableOperations implements TableOperations {
+
+ private final TableMetadata tableMetadata;
+
+ public MetadataWrapperTableOperations(TableMetadata tableMetadata) {
+ this.tableMetadata = tableMetadata;
+ }
+
+ @Override
+ public TableMetadata current() {
+ return this.tableMetadata;
+ }
+
+ @Override
+ public TableMetadata refresh() {
+ return this.tableMetadata;
+ }
+
+ @Override
+ public void commit(TableMetadata tableMetadata, TableMetadata tableMetadata1) {
+ throw new UnsupportedOperationException("Cannot perform commit.");
+ }
+
+ @Override
+ public FileIO io() {
+ throw new NoSuchElementException("Does not possess file io.");
+ }
+
+ @Override
+ public String metadataFileLocation(String s) {
+ return this.tableMetadata.metadataFileLocation();
+ }
+
+ @Override
+ public LocationProvider locationProvider() {
+ throw new NoSuchElementException("Does not possess location provider.");
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagManager.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagManager.java
new file mode 100644
index 0000000..1bd1499
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/NoOpETagManager.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.catalog;
+
+import org.apache.iceberg.catalog.TableIdentifier;
+
+import java.util.Map;
+
+/** Implementation that returns nothing and stores no ETags. */
+public class NoOpETagManager implements ETagManager {
+
+ @Override
+ public void initialize(Map<String, String> properties) {}
+
+ @Override
+ public String getETag(String catalogName, TableIdentifier tableIdentifier) {
+ return null;
+ }
+
+ @Override
+ public void storeETag(String catalogName, TableIdentifier tableIdentifier, String etag) {}
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/PolarisCatalog.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/PolarisCatalog.java
new file mode 100644
index 0000000..5c489ae
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/catalog/PolarisCatalog.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.catalog;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.util.Map;
+import org.apache.http.HttpHeaders;
+import org.apache.http.HttpStatus;
+import org.apache.iceberg.BaseTable;
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Catalog;
+import org.apache.iceberg.catalog.SupportsNamespaces;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.catalog.ViewCatalog;
+import org.apache.iceberg.hadoop.Configurable;
+import org.apache.iceberg.rest.RESTCatalog;
+import org.apache.iceberg.rest.ResourcePaths;
+import org.apache.iceberg.rest.responses.LoadTableResponse;
+import org.apache.iceberg.rest.responses.LoadTableResponseParser;
+import org.apache.polaris.tools.sync.polaris.http.OAuth2Util;
+
+/**
+ * Overrides loadTable default implementation to issue a custom loadTable request to the Polaris
+ * Iceberg REST Api and build the table metadata. This is necessary since the existing {@link
+ * RESTCatalog} does not provide a way to capture response headers to retrieve the ETag on a
+ * loadTable request.
+ *
+ * TODO: Remove this class once Iceberg gets first class support for ETags.
+ * in the canonical response types.
+ */
+public class PolarisCatalog extends RESTCatalog
+ implements Catalog, ViewCatalog, SupportsNamespaces, Configurable<Object>, Closeable {
+
+ private String name = null;
+
+ private Map<String, String> properties = null;
+
+ private String accessToken = null;
+
+ private HttpClient httpClient = null;
+
+ private ObjectMapper objectMapper = null;
+
+ private ResourcePaths resourcePaths = null;
+
+ public PolarisCatalog() {
+ super();
+ }
+
+ @Override
+ public void initialize(String name, Map<String, String> props) {
+ this.name = name;
+ this.properties = props;
+
+ if (resourcePaths == null) {
+ this.properties.put("prefix", props.get("warehouse"));
+ resourcePaths = ResourcePaths.forCatalogProperties(this.properties);
+ }
+
+ super.initialize(name, props);
+
+ if (accessToken == null || httpClient == null || this.objectMapper == null) {
+ String oauth2ServerUri = props.get("uri") + "/v1/oauth/tokens";
+ String credential = props.get("credential");
+
+ String clientId = credential.split(":")[0];
+ String clientSecret = credential.split(":")[1];
+
+ String scope = props.get("scope");
+
+ // TODO: Add token refresh
+ try {
+ this.accessToken = OAuth2Util.fetchToken(oauth2ServerUri, clientId, clientSecret, scope);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ this.httpClient = HttpClient.newBuilder().build();
+ this.objectMapper = new ObjectMapper();
+ }
+ }
+
+ @Override
+ public Table loadTable(TableIdentifier ident) {
+ return loadTable(ident, null);
+ }
+
+ /**
+ * Perform a loadTable with a specified ETag in the If-None-Match header. TODO: Remove this once
+ * ETag is officially supported in Iceberg
+ *
+ * @param ident the identifier of the table
+ * @param etag the etag
+ * @return a {@link BaseTable} if no ETag was found in the response headers. A {@link
+ * BaseTableWithETag} if an ETag was included in the response headers.
+ * @throws MetadataNotModifiedException if the Iceberg REST catalog responded with 304 NOT MODIFIED
+ */
+ public Table loadTable(TableIdentifier ident, String etag) {
+ String catalogName = this.properties.get("warehouse");
+
+ String tablePath =
+ String.format("%s/%s", this.properties.get("uri"), resourcePaths.table(ident));
+
+ HttpRequest.Builder requestBuilder =
+ HttpRequest.newBuilder()
+ .uri(URI.create(tablePath))
+ .header(HttpHeaders.AUTHORIZATION, "Bearer " + accessToken)
+ .GET();
+
+ // specify last known etag in if-none-match header
+ if (etag != null) {
+ requestBuilder.header(HttpHeaders.IF_NONE_MATCH, etag);
+ }
+
+ HttpRequest request = requestBuilder.build();
+
+ HttpResponse<String> response;
+
+ try {
+ response = this.httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ // api responded with 304 not modified, throw from here to signal
+ if (response.statusCode() == HttpStatus.SC_NOT_MODIFIED) {
+ throw new MetadataNotModifiedException(ident);
+ }
+
+ String body = response.body();
+
+ String newETag = null;
+
+ // if etag header is present in response, store new provided etag
+ if (response.headers().firstValue(HttpHeaders.ETAG).isPresent()) {
+ newETag = response.headers().firstValue(HttpHeaders.ETAG).get();
+ }
+
+ // build custom base table with metadata so that tool can retrieve the
+ // location and register it on the target side
+ LoadTableResponse loadTableResponse = LoadTableResponseParser.fromJson(body);
+ MetadataWrapperTableOperations ops =
+ new MetadataWrapperTableOperations(loadTableResponse.tableMetadata());
+
+ if (newETag != null) {
+ return new BaseTableWithETag(ops, CatalogUtil.fullTableName(catalogName, ident), newETag);
+ }
+
+ return new BaseTable(ops, CatalogUtil.fullTableName(catalogName, ident));
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/http/HttpUtil.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/http/HttpUtil.java
new file mode 100644
index 0000000..50c1428
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/http/HttpUtil.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.http;
+
+import java.net.URLEncoder;
+import java.nio.charset.StandardCharsets;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/** Encapsulates handy http utility methods. */
+public class HttpUtil {
+
+ /** Turn a {@link Map} into an xxx-url-form-encoded compatible String form body. */
+ public static String constructFormEncodedString(Map<String, String> parameters) {
+ return parameters.entrySet().stream()
+ .map(
+ entry ->
+ URLEncoder.encode(entry.getKey(), StandardCharsets.UTF_8)
+ + "="
+ + URLEncoder.encode(entry.getValue(), StandardCharsets.UTF_8))
+ .collect(Collectors.joining("&"));
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/http/OAuth2Util.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/http/OAuth2Util.java
new file mode 100644
index 0000000..98ada78
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/http/OAuth2Util.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.http;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.io.IOException;
+import java.net.URI;
+import java.net.http.HttpClient;
+import java.net.http.HttpRequest;
+import java.net.http.HttpResponse;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import org.apache.http.HttpHeaders;
+import org.apache.http.entity.ContentType;
+
+/** Utility class to manage OAuth2 flow for a Polaris instance. */
+public class OAuth2Util {
+
+ private static final HttpClient httpClient = HttpClient.newHttpClient();
+
+ private static final ObjectMapper objectMapper = new ObjectMapper();
+
+ public static String fetchToken(
+ String oauth2ServerUri, String clientId, String clientSecret, String scope)
+ throws IOException {
+
+ Map<String, String> formBody =
+ Map.of(
+ "grant_type", "client_credentials",
+ "scope", scope,
+ "client_id", clientId,
+ "client_secret", clientSecret);
+
+ String formBodyAsString = HttpUtil.constructFormEncodedString(formBody);
+
+ HttpRequest request =
+ HttpRequest.newBuilder()
+ .uri(URI.create(oauth2ServerUri))
+ .header(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_FORM_URLENCODED.getMimeType())
+ .POST(HttpRequest.BodyPublishers.ofString(formBodyAsString))
+ .build();
+
+ try {
+ HttpResponse<String> response =
+ httpClient.send(request, HttpResponse.BodyHandlers.ofString());
+ Map<String, String> responseBody =
+ objectMapper.readValue(response.body(), new TypeReference<>() {});
+
+ String accessToken = responseBody.getOrDefault("access_token", null);
+
+ if (accessToken != null) {
+ return accessToken;
+ }
+
+ throw new NoSuchElementException(
+ "No field 'access_token' found in response from oauth2-server-uri.");
+ } catch (Exception e) {
+ throw new RuntimeException("Could not fetch access token", e);
+ }
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/AccessControlAwarePlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/AccessControlAwarePlanner.java
new file mode 100644
index 0000000..0e958b1
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/AccessControlAwarePlanner.java
@@ -0,0 +1,306 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.planning;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.polaris.core.admin.model.CatalogRole;
+import org.apache.polaris.core.admin.model.Principal;
+import org.apache.polaris.core.admin.model.PrincipalRole;
+import org.apache.polaris.tools.sync.polaris.access.AccessControlConstants;
+import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan;
+
+/**
+ * Planner that filters out access control entities that should not be modified in the duration of
+ * the sync. This includes the omnipotent roles and principals that we do not want to copy between
+ * the two instances as well as modifications to service_admin or catalog_admin that may disrupt
+ * manage_access permissions.
+ */
+public class AccessControlAwarePlanner extends DelegatedPlanner implements SynchronizationPlanner {
+
+ public AccessControlAwarePlanner(SynchronizationPlanner delegate) {
+ super(delegate);
+ }
+
+ @Override
+ public SynchronizationPlan<Principal> planPrincipalSync(
+ List<Principal> principalsOnSource, List<Principal> principalsOnTarget) {
+ List<Principal> skippedPrincipals = new ArrayList<>();
+ List<Principal> filteredPrincipalsSource = new ArrayList<>();
+ List<Principal> filteredPrincipalsTarget = new ArrayList<>();
+
+ for (Principal principal : principalsOnSource) {
+ // if the principal is the omnipotent principal do not sync it to target
+ if (principal.getProperties() != null
+ && principal.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) {
+ skippedPrincipals.add(principal);
+ continue;
+ }
+
+ // do not modify root principal
+ if (principal.getName().equals("root")) {
+ skippedPrincipals.add(principal);
+ continue;
+ }
+
+ filteredPrincipalsSource.add(principal);
+ }
+
+ for (Principal principal : principalsOnTarget) {
+ // if the principal is the omnipotent principal, ensure it is not modified
+ // on the target
+ if (principal.getProperties() != null
+ && principal.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) {
+ skippedPrincipals.add(principal);
+ continue;
+ }
+
+ // do not modify root principal
+ if (principal.getName().equals("root")) {
+ skippedPrincipals.add(principal);
+ continue;
+ }
+
+ filteredPrincipalsTarget.add(principal);
+ }
+
+ SynchronizationPlan<Principal> delegatedPlan =
+ delegate.planPrincipalSync(filteredPrincipalsSource, filteredPrincipalsTarget);
+
+ for (Principal principal : skippedPrincipals) {
+ delegatedPlan.skipEntityAndSkipChildren(principal);
+ }
+
+ return delegatedPlan;
+ }
+
+ @Override
+ public SynchronizationPlan<PrincipalRole> planAssignPrincipalsToPrincipalRolesSync(
+ String principalName,
+ List<PrincipalRole> assignedPrincipalRolesOnSource,
+ List<PrincipalRole> assignedPrincipalRolesOnTarget
+ ) {
+ List<PrincipalRole> skippedRoles = new ArrayList<>();
+ List<PrincipalRole> filteredRolesSource = new ArrayList<>();
+ List<PrincipalRole> filteredRolesTarget = new ArrayList<>();
+
+ for (PrincipalRole role : assignedPrincipalRolesOnSource) {
+ // filter out assignment to omnipotent principal role
+ if (role.getProperties() != null
+ && role.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) {
+ skippedRoles.add(role);
+ continue;
+ }
+
+ // filter out assignment to service admin
+ if (role.getName().equals("service_admin")) {
+ skippedRoles.add(role);
+ continue;
+ }
+
+ filteredRolesSource.add(role);
+ }
+
+ for (PrincipalRole role : assignedPrincipalRolesOnTarget) {
+ // filer out assignment to omnipotent principal role
+ if (role.getProperties() != null
+ && role.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) {
+ skippedRoles.add(role);
+ continue;
+ }
+
+ // filter out assignment to service admin
+ if (role.getName().equals("service_admin")) {
+ skippedRoles.add(role);
+ continue;
+ }
+
+ filteredRolesTarget.add(role);
+ }
+
+ SynchronizationPlan<PrincipalRole> delegatedPlan =
+ this.delegate.planAssignPrincipalsToPrincipalRolesSync(
+ principalName, filteredRolesSource, filteredRolesTarget);
+
+ for (PrincipalRole role : skippedRoles) {
+ delegatedPlan.skipEntity(role);
+ }
+
+ return delegatedPlan;
+ }
+
+ @Override
+ public SynchronizationPlan<PrincipalRole> planPrincipalRoleSync(
+ List<PrincipalRole> principalRolesOnSource, List<PrincipalRole> principalRolesOnTarget) {
+ List<PrincipalRole> skippedRoles = new ArrayList<>();
+ List<PrincipalRole> filteredRolesSource = new ArrayList<>();
+ List<PrincipalRole> filteredRolesTarget = new ArrayList<>();
+
+ for (PrincipalRole role : principalRolesOnSource) {
+ // filter out omnipotent principal role
+ if (role.getProperties() != null
+ && role.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) {
+ skippedRoles.add(role);
+ continue;
+ }
+
+ // filter out service_admin
+ if (role.getName().equals("service_admin")) {
+ skippedRoles.add(role);
+ continue;
+ }
+
+ filteredRolesSource.add(role);
+ }
+
+ for (PrincipalRole role : principalRolesOnTarget) {
+ // filter out omnipotent principal role
+ if (role.getProperties() != null
+ && role.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) {
+ skippedRoles.add(role);
+ continue;
+ }
+
+ // filter out service admin
+ if (role.getName().equals("service_admin")) {
+ skippedRoles.add(role);
+ continue;
+ }
+
+ filteredRolesTarget.add(role);
+ }
+
+ SynchronizationPlan<PrincipalRole> delegatedPlan =
+ this.delegate.planPrincipalRoleSync(filteredRolesSource, filteredRolesTarget);
+
+ for (PrincipalRole role : skippedRoles) {
+ delegatedPlan.skipEntity(role);
+ }
+
+ return delegatedPlan;
+ }
+
+ @Override
+ public SynchronizationPlan<CatalogRole> planCatalogRoleSync(
+ String catalogName,
+ List<CatalogRole> catalogRolesOnSource,
+ List<CatalogRole> catalogRolesOnTarget) {
+ List<CatalogRole> skippedRoles = new ArrayList<>();
+ List<CatalogRole> filteredRolesSource = new ArrayList<>();
+ List<CatalogRole> filteredRolesTarget = new ArrayList<>();
+
+ for (CatalogRole role : catalogRolesOnSource) {
+ // filter out omnipotent catalog role
+ if (role.getProperties() != null
+ && role.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) {
+ skippedRoles.add(role);
+ continue;
+ }
+
+ // filter out catalog admin
+ if (role.getName().equals("catalog_admin")) {
+ skippedRoles.add(role);
+ continue;
+ }
+
+ filteredRolesSource.add(role);
+ }
+
+ for (CatalogRole role : catalogRolesOnTarget) {
+ // filter out omnipotent catalog role
+ if (role.getProperties() != null
+ && role.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) {
+ skippedRoles.add(role);
+ continue;
+ }
+
+ // filter out catalog admin
+ if (role.getName().equals("catalog_admin")) {
+ skippedRoles.add(role);
+ continue;
+ }
+
+ filteredRolesTarget.add(role);
+ }
+
+ SynchronizationPlan<CatalogRole> delegatedPlan =
+ this.delegate.planCatalogRoleSync(catalogName, filteredRolesSource, filteredRolesTarget);
+
+ for (CatalogRole role : skippedRoles) {
+ delegatedPlan.skipEntityAndSkipChildren(role);
+ }
+
+ return delegatedPlan;
+ }
+
+ @Override
+ public SynchronizationPlan<PrincipalRole> planAssignPrincipalRolesToCatalogRolesSync(
+ String catalogName,
+ String catalogRoleName,
+ List<PrincipalRole> assignedPrincipalRolesOnSource,
+ List<PrincipalRole> assignedPrincipalRolesOnTarget) {
+ List<PrincipalRole> skippedRoles = new ArrayList<>();
+ List<PrincipalRole> filteredRolesSource = new ArrayList<>();
+ List<PrincipalRole> filteredRolesTarget = new ArrayList<>();
+
+ for (PrincipalRole role : assignedPrincipalRolesOnSource) {
+ // filter out assignment to omnipotent catalog role
+ if (role.getProperties() != null
+ && role.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) {
+ skippedRoles.add(role);
+ continue;
+ }
+
+ // filter out assignment to service admin
+ if (role.getName().equals("service_admin")) {
+ skippedRoles.add(role);
+ continue;
+ }
+
+ filteredRolesSource.add(role);
+ }
+
+ for (PrincipalRole role : assignedPrincipalRolesOnTarget) {
+ // filer out assignment to omnipotent principal role
+ if (role.getProperties() != null
+ && role.getProperties().containsKey(AccessControlConstants.OMNIPOTENCE_PROPERTY)) {
+ skippedRoles.add(role);
+ continue;
+ }
+
+ // filter out assignment to service admin
+ if (role.getName().equals("service_admin")) {
+ skippedRoles.add(role);
+ continue;
+ }
+
+ filteredRolesTarget.add(role);
+ }
+
+ SynchronizationPlan<PrincipalRole> delegatedPlan =
+ this.delegate.planAssignPrincipalRolesToCatalogRolesSync(
+ catalogName, catalogRoleName, filteredRolesSource, filteredRolesTarget);
+
+ for (PrincipalRole role : skippedRoles) {
+ delegatedPlan.skipEntity(role);
+ }
+
+ return delegatedPlan;
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/DelegatedPlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/DelegatedPlanner.java
new file mode 100644
index 0000000..2db3015
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/DelegatedPlanner.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.planning;
+
+import java.util.List;
+import java.util.Set;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.polaris.core.admin.model.Catalog;
+import org.apache.polaris.core.admin.model.CatalogRole;
+import org.apache.polaris.core.admin.model.GrantResource;
+import org.apache.polaris.core.admin.model.Principal;
+import org.apache.polaris.core.admin.model.PrincipalRole;
+import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan;
+
+/**
+ * Extend this to delegate planning to another planner, but only override methods for the
+ * functionality needed.
+ */
+public abstract class DelegatedPlanner implements SynchronizationPlanner {
+
+ protected final SynchronizationPlanner delegate;
+
+ public DelegatedPlanner(SynchronizationPlanner delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public SynchronizationPlan<Principal> planPrincipalSync(
+ List<Principal> principalsOnSource, List<Principal> principalsOnTarget) {
+ return delegate.planPrincipalSync(principalsOnSource, principalsOnTarget);
+ }
+
+ @Override
+ public SynchronizationPlan<PrincipalRole> planAssignPrincipalsToPrincipalRolesSync(
+ String principalName,
+ List<PrincipalRole> assignedPrincipalRolesOnSource,
+ List<PrincipalRole> assignedPrincipalRolesOnTarget
+ ) {
+ return delegate.planAssignPrincipalsToPrincipalRolesSync(
+ principalName, assignedPrincipalRolesOnSource, assignedPrincipalRolesOnTarget);
+ }
+
+ @Override
+ public SynchronizationPlan<PrincipalRole> planPrincipalRoleSync(
+ List<PrincipalRole> principalRolesOnSource, List<PrincipalRole> principalRolesOnTarget) {
+ return delegate.planPrincipalRoleSync(principalRolesOnSource, principalRolesOnTarget);
+ }
+
+ @Override
+ public SynchronizationPlan<Catalog> planCatalogSync(
+ List<Catalog> catalogsOnSource, List<Catalog> catalogsOnTarget) {
+ return delegate.planCatalogSync(catalogsOnSource, catalogsOnTarget);
+ }
+
+ @Override
+ public SynchronizationPlan<CatalogRole> planCatalogRoleSync(
+ String catalogName,
+ List<CatalogRole> catalogRolesOnSource,
+ List<CatalogRole> catalogRolesOnTarget) {
+ return delegate.planCatalogRoleSync(catalogName, catalogRolesOnSource, catalogRolesOnTarget);
+ }
+
+ @Override
+ public SynchronizationPlan<GrantResource> planGrantSync(
+ String catalogName,
+ String catalogRoleName,
+ List<GrantResource> grantsOnSource,
+ List<GrantResource> grantsOnTarget) {
+ return delegate.planGrantSync(catalogName, catalogRoleName, grantsOnSource, grantsOnTarget);
+ }
+
+ @Override
+ public SynchronizationPlan<PrincipalRole> planAssignPrincipalRolesToCatalogRolesSync(
+ String catalogName,
+ String catalogRoleName,
+ List<PrincipalRole> assignedPrincipalRolesOnSource,
+ List<PrincipalRole> assignedPrincipalRolesOnTarget) {
+ return delegate.planAssignPrincipalRolesToCatalogRolesSync(
+ catalogName,
+ catalogRoleName,
+ assignedPrincipalRolesOnSource,
+ assignedPrincipalRolesOnTarget);
+ }
+
+ @Override
+ public SynchronizationPlan<Namespace> planNamespaceSync(
+ String catalogName,
+ Namespace namespace,
+ List<Namespace> namespacesOnSource,
+ List<Namespace> namespacesOnTarget) {
+ return delegate.planNamespaceSync(
+ catalogName, namespace, namespacesOnSource, namespacesOnTarget);
+ }
+
+ @Override
+ public SynchronizationPlan<TableIdentifier> planTableSync(
+ String catalogName,
+ Namespace namespace,
+ Set<TableIdentifier> tablesOnSource,
+ Set<TableIdentifier> tablesOnTarget) {
+ return delegate.planTableSync(catalogName, namespace, tablesOnSource, tablesOnTarget);
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java
new file mode 100644
index 0000000..15308ab
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/ModificationAwarePlanner.java
@@ -0,0 +1,422 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.planning;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.function.Supplier;
+
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.polaris.core.admin.model.Catalog;
+import org.apache.polaris.core.admin.model.CatalogRole;
+import org.apache.polaris.core.admin.model.GrantResource;
+import org.apache.polaris.core.admin.model.Principal;
+import org.apache.polaris.core.admin.model.PrincipalRole;
+import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan;
+import org.eclipse.collections.impl.block.procedure.CollectIfProcedure;
+
+/** Planner that checks for modifications and plans to skip entities that have not been modified. */
+public class ModificationAwarePlanner implements SynchronizationPlanner {
+
+ private static final String CREATE_TIMESTAMP = "createTimestamp";
+
+ private static final String LAST_UPDATE_TIMESTAMP = "lastUpdateTimestamp";
+
+ private static final String ENTITY_VERSION = "entityVersion";
+
+ private static final List<String> DEFAULT_KEYS_TO_IGNORE =
+ List.of(CREATE_TIMESTAMP, LAST_UPDATE_TIMESTAMP, ENTITY_VERSION);
+
+ private static final List<String> CATALOG_KEYS_TO_IGNORE =
+ List.of(
+ // defaults
+ CREATE_TIMESTAMP,
+ LAST_UPDATE_TIMESTAMP,
+ ENTITY_VERSION,
+
+ // For certain storageConfigInfo fields, depending on the credentials Polaris was set up
+ // with
+ // to access the storage, some fields will always be different across the source and the
+ // target.
+ // For example, for S3 my source and target Polaris instances may be set up with different
+ // AWS users,
+ // each of which assumes the same role to access the storage
+
+ // S3
+ "storageConfigInfo.userArn",
+ "storageConfigInfo.externalId",
+
+ // AZURE
+ "storageConfigInfo.consentUrl",
+ "storageConfigInfo.multiTenantAppName",
+
+ // GCP
+ "storageConfigInfo.gcsServiceAccount");
+
+ private static final String CLIENT_ID = "clientId";
+
+ private static final String CLIENT_SECRET = "clientSecret";
+
+ private static final List<String> PRINCIPAL_KEYS_TO_IGNORE = List.of(
+ CREATE_TIMESTAMP,
+ LAST_UPDATE_TIMESTAMP,
+ ENTITY_VERSION,
+
+ // client id will never be the same across the instances, ignore it
+ CLIENT_ID
+ );
+
+ private final SynchronizationPlanner delegate;
+
+ private final ObjectMapper objectMapper;
+
+ public ModificationAwarePlanner(SynchronizationPlanner delegate) {
+ this.objectMapper = new ObjectMapper();
+ this.delegate = delegate;
+ }
+
+ /**
+ * Removes keys from the provided map.
+ *
+ * @param map the map to remove the keys from
+ * @param keysToRemove a list of keys, nested keys should be separated by '.' eg. "key1.key2"
+ * @return the map with the keys removed
+ */
+ private Map<String, Object> removeKeys(Map<String, Object> map, List<String> keysToRemove) {
+ Map<String, Object> cleaned =
+ objectMapper.convertValue(map, new TypeReference<Map<String, Object>>() {});
+
+ for (String key : keysToRemove) {
+ // splits key into first part and rest, eg. key1.key2.key3 becomes [key1, key2.key3]
+ String[] separateFirst = key.split("\\.", 2);
+ String primary = separateFirst[0];
+
+ if (separateFirst.length > 1) {
+ // if there are more nested keys, we want to recursively search the sub map if it exists
+ Object valueForPrimary = cleaned.get(primary); // get object for primary key if it exists
+
+ if (valueForPrimary == null) {
+ continue;
+ }
+
+ try {
+ Map<String, Object> subMap =
+ objectMapper.convertValue(valueForPrimary, new TypeReference<>() {});
+ Map<String, Object> cleanedSubMap =
+ removeKeys(subMap, List.of(separateFirst[1])); // remove nested keys from submap
+ cleaned.put(primary, cleanedSubMap); // replace sub-map with key removed
+ } catch (IllegalArgumentException e) {
+ // do nothing because that means the key does not exist, no need to remove it
+ }
+ } else {
+ cleaned.remove(primary); // just remove the key if we have no more nesting
+ }
+ }
+
+ return cleaned;
+ }
+
+ /**
+ * Compares two objects to see if they are the same.
+ *
+ * @param o1
+ * @param o2
+ * @param keysToIgnore list of keys to ignore in the comparison
+ * @return true if they are the same, false otherwise
+ */
+ private boolean areSame(Object o1, Object o2, List<String> keysToIgnore) {
+ Map<String, Object> o1AsMap = objectMapper.convertValue(o1, new TypeReference<>() {});
+ Map<String, Object> o2AsMap = objectMapper.convertValue(o2, new TypeReference<>() {});
+ o1AsMap = removeKeys(o1AsMap, keysToIgnore);
+ o2AsMap = removeKeys(o2AsMap, keysToIgnore);
+ return o1AsMap.equals(o2AsMap);
+ }
+
+ private boolean areSame(Object o1, Object o2) {
+ return areSame(o1, o2, DEFAULT_KEYS_TO_IGNORE);
+ }
+
+ /**
+ * Container to represent the result of filtering out entities that are on both the source and the target
+ * and have not been modified.
+ * @param filteredEntitiesSource entities on the source that are do not exist on the
+ * target or are different from the target for the same identifier
+ * @param filteredEntitiesTarget entities on the target that do not exist on the source or are different from the
+ * source for the same identifier
+ * @param notModifiedEntities entities from the source that have not been modified since migration to the target
+ * @param <T> the entity type
+ */
+ private record FilteredNotModifiedEntityResult<T>(
+ List<T> filteredEntitiesSource, List<T> filteredEntitiesTarget, List<T> notModifiedEntities) {}
+
+ /**
+ * Filter out entities that are the same across the source and target Polaris instance.
+ * @param entitiesOnSource the entities from the source
+ * @param entitiesOnTarget the entities from the target
+ * @param entityIdentifierSupplier supplies an identifier to identify the same entity on the source and target
+ * @param entitiesAreSame compares entity with same identifier on source and target, returns true if they are the same,
+ * false otherwise
+ * @return the entities on the source and target with unmodified entities filtered out
+ * @param <T> the entity type
+ */
+ private <T> FilteredNotModifiedEntityResult<T> filterOutEntitiesNotModified(
+ Collection<T> entitiesOnSource,
+ Collection<T> entitiesOnTarget,
+ Function<T, Object> entityIdentifierSupplier,
+ BiFunction<T, T, Boolean> entitiesAreSame
+ ) {
+ Map<Object, T> sourceEntitiesById = new HashMap<>();
+ Map<Object, T> targetEntitiesById = new HashMap<>();
+
+ List<T> notModifiedEntities = new ArrayList<>();
+
+ entitiesOnSource.forEach(entity -> sourceEntitiesById.put(entityIdentifierSupplier.apply(entity), entity));
+ entitiesOnTarget.forEach(entity -> targetEntitiesById.put(entityIdentifierSupplier.apply(entity), entity));
+
+ for (T sourceEntity : entitiesOnSource) {
+ Object sourceEntityId = entityIdentifierSupplier.apply(sourceEntity);
+ if (targetEntitiesById.containsKey(sourceEntityId)) {
+ T targetEntity = targetEntitiesById.get(sourceEntityId);
+ Object targetEntityId = entityIdentifierSupplier.apply(targetEntity);
+
+ if (entitiesAreSame.apply(sourceEntity, targetEntity)) {
+ notModifiedEntities.add(sourceEntity);
+ sourceEntitiesById.remove(sourceEntityId);
+ targetEntitiesById.remove(targetEntityId);
+ }
+ }
+ }
+
+ return new FilteredNotModifiedEntityResult<>(
+ sourceEntitiesById.values().stream().toList(),
+ targetEntitiesById.values().stream().toList(),
+ notModifiedEntities
+ );
+ }
+
+ @Override
+ public SynchronizationPlan<Principal> planPrincipalSync(List<Principal> principalsOnSource, List<Principal> principalsOnTarget) {
+ FilteredNotModifiedEntityResult<Principal> result = filterOutEntitiesNotModified(
+ principalsOnSource,
+ principalsOnTarget,
+ Principal::getName,
+ (p1, p2) -> areSame(p1, p2, PRINCIPAL_KEYS_TO_IGNORE)
+ );
+
+ SynchronizationPlan<Principal> delegatedPlan = delegate.planPrincipalSync(
+ result.filteredEntitiesSource(),
+ result.filteredEntitiesTarget()
+ );
+
+ for (Principal principal : result.notModifiedEntities()) {
+ delegatedPlan.skipEntityNotModified(principal);
+ }
+
+ return delegatedPlan;
+ }
+
+ @Override
+ public SynchronizationPlan<PrincipalRole> planAssignPrincipalsToPrincipalRolesSync(
+ String principalName,
+ List<PrincipalRole> assignedPrincipalRolesOnSource,
+ List<PrincipalRole> assignedPrincipalRolesOnTarget
+ ) {
+ FilteredNotModifiedEntityResult<PrincipalRole> result = filterOutEntitiesNotModified(
+ assignedPrincipalRolesOnSource,
+ assignedPrincipalRolesOnTarget,
+ PrincipalRole::getName,
+ this::areSame
+ );
+
+ SynchronizationPlan<PrincipalRole> delegatedPlan =
+ delegate.planAssignPrincipalsToPrincipalRolesSync(
+ principalName,
+ result.filteredEntitiesSource(),
+ result.filteredEntitiesTarget());
+
+ for (PrincipalRole principalRole : result.notModifiedEntities()) {
+ delegatedPlan.skipEntityNotModified(principalRole);
+ }
+
+ return delegatedPlan;
+ }
+
+ @Override
+ public SynchronizationPlan<PrincipalRole> planPrincipalRoleSync(
+ List<PrincipalRole> principalRolesOnSource, List<PrincipalRole> principalRolesOnTarget) {
+ FilteredNotModifiedEntityResult<PrincipalRole> result = filterOutEntitiesNotModified(
+ principalRolesOnSource,
+ principalRolesOnTarget,
+ PrincipalRole::getName,
+ this::areSame
+ );
+
+ SynchronizationPlan<PrincipalRole> delegatedPlan =
+ delegate.planPrincipalRoleSync(
+ result.filteredEntitiesSource(),
+ result.filteredEntitiesTarget());
+
+ for (PrincipalRole principalRole : result.notModifiedEntities()) {
+ delegatedPlan.skipEntityNotModified(principalRole);
+ }
+
+ return delegatedPlan;
+ }
+
+ private boolean areSame(Catalog source, Catalog target) {
+ return areSame(source, target, CATALOG_KEYS_TO_IGNORE)
+ // because of the way the jackson serialization works, any class that extends HashMap is
+ // serialized
+ // with just the fields in the map. Unfortunately, CatalogProperties extends HashMap so we
+ // must
+ // manually compare the fields in the catalog properties and cannot automatically
+ // deserialize them
+ // as a map
+ && Objects.equals(source.getProperties(), target.getProperties());
+ }
+
+ @Override
+ public SynchronizationPlan<Catalog> planCatalogSync(
+ List<Catalog> catalogsOnSource, List<Catalog> catalogsOnTarget) {
+ FilteredNotModifiedEntityResult<Catalog> result = filterOutEntitiesNotModified(
+ catalogsOnSource,
+ catalogsOnTarget,
+ Catalog::getName,
+ this::areSame
+ );
+
+ SynchronizationPlan<Catalog> delegatedPlan =
+ delegate.planCatalogSync(
+ result.filteredEntitiesSource(),
+ result.filteredEntitiesTarget());
+
+ for (Catalog catalog : result.notModifiedEntities()) {
+ delegatedPlan.skipEntityNotModified(catalog);
+ }
+
+ return delegatedPlan;
+ }
+
+ @Override
+ public SynchronizationPlan<CatalogRole> planCatalogRoleSync(
+ String catalogName,
+ List<CatalogRole> catalogRolesOnSource,
+ List<CatalogRole> catalogRolesOnTarget) {
+ FilteredNotModifiedEntityResult<CatalogRole> result = filterOutEntitiesNotModified(
+ catalogRolesOnSource,
+ catalogRolesOnTarget,
+ CatalogRole::getName,
+ this::areSame
+ );
+
+ SynchronizationPlan<CatalogRole> delegatedPlan =
+ delegate.planCatalogRoleSync(
+ catalogName,
+ result.filteredEntitiesSource(),
+ result.filteredEntitiesTarget());
+
+ for (CatalogRole catalogRole : result.notModifiedEntities()) {
+ delegatedPlan.skipEntityNotModified(catalogRole);
+ }
+
+ return delegatedPlan;
+ }
+
+ @Override
+ public SynchronizationPlan<GrantResource> planGrantSync(
+ String catalogName,
+ String catalogRoleName,
+ List<GrantResource> grantsOnSource,
+ List<GrantResource> grantsOnTarget) {
+ FilteredNotModifiedEntityResult<GrantResource> result = filterOutEntitiesNotModified(
+ grantsOnSource,
+ grantsOnTarget,
+ grant -> grant,
+ GrantResource::equals
+ );
+
+ SynchronizationPlan<GrantResource> delegatedPlan =
+ delegate.planGrantSync(
+ catalogName,
+ catalogRoleName,
+ result.filteredEntitiesSource(),
+ result.filteredEntitiesTarget());
+
+ for (GrantResource grant : result.notModifiedEntities()) {
+ delegatedPlan.skipEntityNotModified(grant);
+ }
+
+ return delegatedPlan;
+ }
+
+ @Override
+ public SynchronizationPlan<PrincipalRole> planAssignPrincipalRolesToCatalogRolesSync(
+ String catalogName,
+ String catalogRoleName,
+ List<PrincipalRole> assignedPrincipalRolesOnSource,
+ List<PrincipalRole> assignedPrincipalRolesOnTarget) {
+ FilteredNotModifiedEntityResult<PrincipalRole> result = filterOutEntitiesNotModified(
+ assignedPrincipalRolesOnSource,
+ assignedPrincipalRolesOnTarget,
+ PrincipalRole::getName,
+ this::areSame
+ );
+
+ SynchronizationPlan<PrincipalRole> delegatedPlan = delegate.planAssignPrincipalRolesToCatalogRolesSync(
+ catalogName,
+ catalogRoleName,
+ assignedPrincipalRolesOnSource,
+ assignedPrincipalRolesOnTarget);
+
+ for (PrincipalRole principalRole : result.notModifiedEntities()) {
+ delegatedPlan.skipEntityNotModified(principalRole);
+ }
+ return delegatedPlan;
+ }
+
+ @Override
+ public SynchronizationPlan<Namespace> planNamespaceSync(
+ String catalogName,
+ Namespace namespace,
+ List<Namespace> namespacesOnSource,
+ List<Namespace> namespacesOnTarget) {
+ return delegate.planNamespaceSync(
+ catalogName, namespace, namespacesOnSource, namespacesOnTarget);
+ }
+
+ @Override
+ public SynchronizationPlan<TableIdentifier> planTableSync(
+ String catalogName,
+ Namespace namespace,
+ Set<TableIdentifier> tablesOnSource,
+ Set<TableIdentifier> tablesOnTarget) {
+ return delegate.planTableSync(catalogName, namespace, tablesOnSource, tablesOnTarget);
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java
new file mode 100644
index 0000000..0747ce6
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/NoOpSyncPlanner.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.planning;
+
+import java.util.List;
+import java.util.Set;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.polaris.core.admin.model.Catalog;
+import org.apache.polaris.core.admin.model.CatalogRole;
+import org.apache.polaris.core.admin.model.GrantResource;
+import org.apache.polaris.core.admin.model.Principal;
+import org.apache.polaris.core.admin.model.PrincipalRole;
+import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan;
+
+/**
+ * No-op implementation that generates plans that direct no modifications to the target.
+ */
+public class NoOpSyncPlanner implements SynchronizationPlanner {
+
+ @Override
+ public SynchronizationPlan<Principal> planPrincipalSync(
+ List<Principal> principalsOnSource, List<Principal> principalsOnTarget) {
+ return new SynchronizationPlan<>();
+ }
+
+ @Override
+ public SynchronizationPlan<PrincipalRole> planAssignPrincipalsToPrincipalRolesSync(
+ String principalName,
+ List<PrincipalRole> assignedPrincipalRolesOnSource,
+ List<PrincipalRole> assignedPrincipalRolesOnTarget
+ ) {
+ return new SynchronizationPlan<>();
+ }
+
+ @Override
+ public SynchronizationPlan<PrincipalRole> planPrincipalRoleSync(
+ List<PrincipalRole> principalRolesOnSource, List<PrincipalRole> principalRolesOnTarget) {
+ return new SynchronizationPlan<>();
+ }
+
+ @Override
+ public SynchronizationPlan<Catalog> planCatalogSync(
+ List<Catalog> catalogsOnSource, List<Catalog> catalogsOnTarget) {
+ return new SynchronizationPlan<>();
+ }
+
+ @Override
+ public SynchronizationPlan<CatalogRole> planCatalogRoleSync(
+ String catalogName,
+ List<CatalogRole> catalogRolesOnSource,
+ List<CatalogRole> catalogRolesOnTarget) {
+ return new SynchronizationPlan<>();
+ }
+
+ @Override
+ public SynchronizationPlan<GrantResource> planGrantSync(
+ String catalogName,
+ String catalogRoleName,
+ List<GrantResource> grantsOnSource,
+ List<GrantResource> grantsOnTarget) {
+ return new SynchronizationPlan<>();
+ }
+
+ @Override
+ public SynchronizationPlan<PrincipalRole> planAssignPrincipalRolesToCatalogRolesSync(
+ String catalogName,
+ String catalogRoleName,
+ List<PrincipalRole> assignedPrincipalRolesOnSource,
+ List<PrincipalRole> assignedPrincipalRolesOnTarget) {
+ return new SynchronizationPlan<>();
+ }
+
+ @Override
+ public SynchronizationPlan<Namespace> planNamespaceSync(
+ String catalogName,
+ Namespace namespace,
+ List<Namespace> namespacesOnSource,
+ List<Namespace> namespacesOnTarget) {
+ return null;
+ }
+
+ @Override
+ public SynchronizationPlan<TableIdentifier> planTableSync(
+ String catalogName,
+ Namespace namespace,
+ Set<TableIdentifier> tablesOnSource,
+ Set<TableIdentifier> tablesOnTarget) {
+ return new SynchronizationPlan<>();
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SourceParitySynchronizationPlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SourceParitySynchronizationPlanner.java
new file mode 100644
index 0000000..a60fea7
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SourceParitySynchronizationPlanner.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.planning;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.polaris.core.admin.model.Catalog;
+import org.apache.polaris.core.admin.model.CatalogRole;
+import org.apache.polaris.core.admin.model.GrantResource;
+import org.apache.polaris.core.admin.model.Principal;
+import org.apache.polaris.core.admin.model.PrincipalRole;
+import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan;
+
+/**
+ * Sync planner that attempts to create total parity between the source and target Polaris
+ * instances. This involves creating new entities, overwriting entities that exist on both source
+ * and target, and removing entities that exist only on the target.
+ */
+public class SourceParitySynchronizationPlanner implements SynchronizationPlanner {
+
+ /**
+ * Sort entities from the source into create, overwrite, and remove categories
+ * on the basis of which identifiers exist on the source and target Polaris.
+ * Identifiers that are both on the source and target instance will be marked
+ * for overwrite. Entities that are only on the source instance will be marked for
+ * creation. Entities that are only on the target instance will be marked for deletion.
+ * @param entitiesOnSource the entities from the source
+ * @param entitiesOnTarget the entities from the target
+ * @param supportOverwrites true if "overwriting" the entity is necessary. Most grant record entities do not need overwriting.
+ * @param entityIdentifierSupplier consumes an entity and returns an identifying representation of that entity
+ * @return a {@link SynchronizationPlan} with the entities sorted based on the souce parity strategy
+ * @param <T> the type of the entity
+ */
+ private <T> SynchronizationPlan<T> sortOnIdentifier(
+ Collection<T> entitiesOnSource,
+ Collection<T> entitiesOnTarget,
+ boolean supportOverwrites,
+ Function<T, Object> entityIdentifierSupplier
+ ) {
+ Set<Object> sourceEntityIdentifiers = entitiesOnSource.stream().map(entityIdentifierSupplier).collect(Collectors.toSet());
+ Set<Object> targetEntityIdentifiers = entitiesOnTarget.stream().map(entityIdentifierSupplier).collect(Collectors.toSet());
+
+ SynchronizationPlan<T> plan = new SynchronizationPlan<>();
+
+ for (T entityOnSource : entitiesOnSource) {
+ Object sourceEntityId = entityIdentifierSupplier.apply(entityOnSource);
+ if (targetEntityIdentifiers.contains(sourceEntityId)) {
+ if (supportOverwrites) {
+ // if the same entity identifier is on the source and the target,
+ // overwrite the one on the target with the one on the source
+ plan.overwriteEntity(entityOnSource);
+ }
+ } else {
+ // if the entity identifier only exists on the source, that means
+ // we need to create it for the first time on the target
+ plan.createEntity(entityOnSource);
+ }
+ }
+
+ for (T entityOnTarget : entitiesOnTarget) {
+ Object targetEntityId = entityIdentifierSupplier.apply(entityOnTarget);
+ if (!sourceEntityIdentifiers.contains(targetEntityId)) {
+ // if the entity exists on the target but doesn't exist on the source,
+ // clean it up from the target
+
+ // this is especially important for access control entities, as, for example,
+ // we could have a scenario where a grant was revoked from a catalog role,
+ // or a catalog role was revoked from a principal role, in which case the target
+ // should reflect this change when the tool is run multiple times, because we don't
+ // want to take chances with over-extending privileges
+ plan.removeEntity(entityOnTarget);
+ }
+ }
+
+ return plan;
+ }
+
+ @Override
+ public SynchronizationPlan<Principal> planPrincipalSync(
+ List<Principal> principalsOnSource, List<Principal> principalsOnTarget) {
+ return sortOnIdentifier(principalsOnSource, principalsOnTarget, /* supportsOverwrites */ true, Principal::getName);
+ }
+
+ @Override
+ public SynchronizationPlan<PrincipalRole> planAssignPrincipalsToPrincipalRolesSync(
+ String principalName,
+ List<PrincipalRole> assignedPrincipalRolesOnSource,
+ List<PrincipalRole> assignedPrincipalRolesOnTarget
+ ) {
+ return sortOnIdentifier(
+ assignedPrincipalRolesOnSource,
+ assignedPrincipalRolesOnTarget,
+ /* supportsOverwrites */ false, // do not need to overwrite an assignment of a principal role to a principal
+ PrincipalRole::getName
+ );
+ }
+
+ @Override
+ public SynchronizationPlan<PrincipalRole> planPrincipalRoleSync(
+ List<PrincipalRole> principalRolesOnSource, List<PrincipalRole> principalRolesOnTarget) {
+
+ return sortOnIdentifier(
+ principalRolesOnSource,
+ principalRolesOnTarget,
+ /* supportsOverwrites */ true,
+ PrincipalRole::getName
+ );
+ }
+
+ @Override
+ public SynchronizationPlan<Catalog> planCatalogSync(
+ List<Catalog> catalogsOnSource, List<Catalog> catalogsOnTarget) {
+ return sortOnIdentifier(catalogsOnSource, catalogsOnTarget, /* supportsOverwrites */ true, Catalog::getName);
+ }
+
+ @Override
+ public SynchronizationPlan<CatalogRole> planCatalogRoleSync(
+ String catalogName,
+ List<CatalogRole> catalogRolesOnSource,
+ List<CatalogRole> catalogRolesOnTarget) {
+ return sortOnIdentifier(
+ catalogRolesOnSource, catalogRolesOnTarget, /* supportsOverwrites */ true, CatalogRole::getName);
+ }
+
+ @Override
+ public SynchronizationPlan<GrantResource> planGrantSync(
+ String catalogName,
+ String catalogRoleName,
+ List<GrantResource> grantsOnSource,
+ List<GrantResource> grantsOnTarget) {
+ return sortOnIdentifier(
+ grantsOnSource,
+ grantsOnTarget,
+ /* supportsOverwrites */ false,
+ grant -> grant // grants can just be compared by the entire generated object
+ );
+ }
+
+ @Override
+ public SynchronizationPlan<PrincipalRole> planAssignPrincipalRolesToCatalogRolesSync(
+ String catalogName,
+ String catalogRoleName,
+ List<PrincipalRole> assignedPrincipalRolesOnSource,
+ List<PrincipalRole> assignedPrincipalRolesOnTarget) {
+ return sortOnIdentifier(
+ assignedPrincipalRolesOnSource,
+ assignedPrincipalRolesOnTarget,
+ /* supportsOverwrites */ false,
+ PrincipalRole::getName
+ );
+ }
+
+ @Override
+ public SynchronizationPlan<Namespace> planNamespaceSync(
+ String catalogName,
+ Namespace namespace,
+ List<Namespace> namespacesOnSource,
+ List<Namespace> namespacesOnTarget) {
+ return sortOnIdentifier(namespacesOnSource, namespacesOnTarget, /* supportsOverwrites */ true, ns -> ns);
+ }
+
+ @Override
+ public SynchronizationPlan<TableIdentifier> planTableSync(
+ String catalogName,
+ Namespace namespace,
+ Set<TableIdentifier> tablesOnSource,
+ Set<TableIdentifier> tablesOnTarget) {
+ return sortOnIdentifier(
+ tablesOnSource, tablesOnTarget, /* supportsOverwrites */ true, tableId -> tableId);
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java
new file mode 100644
index 0000000..93cddab
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/SynchronizationPlanner.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.planning;
+
+import java.util.List;
+import java.util.Set;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.polaris.core.admin.model.Catalog;
+import org.apache.polaris.core.admin.model.CatalogRole;
+import org.apache.polaris.core.admin.model.GrantResource;
+import org.apache.polaris.core.admin.model.Principal;
+import org.apache.polaris.core.admin.model.PrincipalRole;
+import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan;
+
+/**
+ * Generic interface to generate synchronization plans for different types of entities based on what
+ * principal roles exist on the source and target.
+ */
+public interface SynchronizationPlanner {
+
+ SynchronizationPlan<Principal> planPrincipalSync(
+ List<Principal> principalsOnSource, List<Principal> principalsOnTarget);
+
+ SynchronizationPlan<PrincipalRole> planAssignPrincipalsToPrincipalRolesSync(
+ String principalName,
+ List<PrincipalRole> assignedPrincipalRolesOnSource,
+ List<PrincipalRole> assignedPrincipalRolesOnTarget);
+
+ SynchronizationPlan<PrincipalRole> planPrincipalRoleSync(
+ List<PrincipalRole> principalRolesOnSource, List<PrincipalRole> principalRolesOnTarget);
+
+ SynchronizationPlan<Catalog> planCatalogSync(
+ List<Catalog> catalogsOnSource, List<Catalog> catalogsOnTarget);
+
+ SynchronizationPlan<CatalogRole> planCatalogRoleSync(
+ String catalogName,
+ List<CatalogRole> catalogRolesOnSource,
+ List<CatalogRole> catalogRolesOnTarget);
+
+ SynchronizationPlan<GrantResource> planGrantSync(
+ String catalogName,
+ String catalogRoleName,
+ List<GrantResource> grantsOnSource,
+ List<GrantResource> grantsOnTarget);
+
+ SynchronizationPlan<PrincipalRole> planAssignPrincipalRolesToCatalogRolesSync(
+ String catalogName,
+ String catalogRoleName,
+ List<PrincipalRole> assignedPrincipalRolesOnSource,
+ List<PrincipalRole> assignedPrincipalRolesOnTarget);
+
+ SynchronizationPlan<Namespace> planNamespaceSync(
+ String catalogName,
+ Namespace namespace,
+ List<Namespace> namespacesOnSource,
+ List<Namespace> namespacesOnTarget);
+
+ SynchronizationPlan<TableIdentifier> planTableSync(
+ String catalogName,
+ Namespace namespace,
+ Set<TableIdentifier> tablesOnSource,
+ Set<TableIdentifier> tablesOnTarget);
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/plan/PlannedAction.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/plan/PlannedAction.java
new file mode 100644
index 0000000..c5b5e50
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/plan/PlannedAction.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.planning.plan;
+
+public enum PlannedAction {
+
+ /** For entities that are being freshly created on target. */
+ CREATE,
+
+ /** For entities that have to be dropped and recreated on target. */
+ OVERWRITE,
+
+ /** For entities that need to be dropped from the target. */
+ REMOVE,
+
+ /**
+ * For entities that should be skipped. Note that their child entities will still be synced. For
+ * example, we may skip a catalog role but its grants and assignments to principal roles will
+ * still be synced.
+ */
+ SKIP,
+
+ /**
+ * For entities that should be skipped due to no modification detected. Note that their child
+ * entities will still be synced. For example, we may skip a catalog role but its grants and
+ * assignments to principal roles will still be synced.
+ */
+ SKIP_NOT_MODIFIED,
+
+ /**
+ * For entities that should be skipped along with also skipping their child entities. Used in
+ * cases where we don't want to mess with an entire entity tree. For example we may not want to
+ * edit the catalog roles assigned to the service_admin.
+ */
+ SKIP_AND_SKIP_CHILDREN
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/plan/SynchronizationPlan.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/plan/SynchronizationPlan.java
new file mode 100644
index 0000000..2153fc9
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/planning/plan/SynchronizationPlan.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris.planning.plan;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Data structure that holds the state of all the planned modifications that should be made on the
+ * target.
+ *
+ * @param <T> the entity type that the plan is for
+ */
+public class SynchronizationPlan<T> {
+
+ private final Map<PlannedAction, List<T>> entitiesForAction;
+
+ public SynchronizationPlan() {
+ this.entitiesForAction = new HashMap<>();
+
+ for (PlannedAction action : PlannedAction.values()) {
+ this.entitiesForAction.put(action, new ArrayList<>());
+ }
+ }
+
+ public List<T> entitiesForAction(PlannedAction action) {
+ return entitiesForAction.get(action);
+ }
+
+ public List<T> entitiesToCreate() {
+ return entitiesForAction(PlannedAction.CREATE);
+ }
+
+ public List<T> entitiesToOverwrite() {
+ return entitiesForAction(PlannedAction.OVERWRITE);
+ }
+
+ public List<T> entitiesToRemove() {
+ return entitiesForAction(PlannedAction.REMOVE);
+ }
+
+ public List<T> entitiesToSkip() {
+ return entitiesForAction(PlannedAction.SKIP);
+ }
+
+ public List<T> entitiesNotModified() {
+ return entitiesForAction(PlannedAction.SKIP_NOT_MODIFIED);
+ }
+
+ public List<T> entitiesToSkipAndSkipChildren() {
+ return entitiesForAction(PlannedAction.SKIP_AND_SKIP_CHILDREN);
+ }
+
+ public List<T> entitiesToSyncChildren() {
+ List<T> entities = new ArrayList<>();
+
+ for (PlannedAction action : PlannedAction.values()) {
+ if (action != PlannedAction.SKIP_AND_SKIP_CHILDREN && action != PlannedAction.REMOVE) {
+ entities.addAll(entitiesForAction(action));
+ }
+ }
+
+ return entities;
+ }
+
+ public void actOnEntity(PlannedAction action, T entity) {
+ this.entitiesForAction.get(action).add(entity);
+ }
+
+ public void createEntity(T entity) {
+ this.actOnEntity(PlannedAction.CREATE, entity);
+ }
+
+ public void overwriteEntity(T entity) {
+ this.actOnEntity(PlannedAction.OVERWRITE, entity);
+ }
+
+ public void removeEntity(T entity) {
+ this.actOnEntity(PlannedAction.REMOVE, entity);
+ }
+
+ public void skipEntity(T entity) {
+ this.actOnEntity(PlannedAction.SKIP, entity);
+ }
+
+ public void skipEntityNotModified(T entity) {
+ this.actOnEntity(PlannedAction.SKIP_NOT_MODIFIED, entity);
+ }
+
+ public void skipEntityAndSkipChildren(T entity) {
+ this.actOnEntity(PlannedAction.SKIP_AND_SKIP_CHILDREN, entity);
+ }
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/IcebergCatalogService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/IcebergCatalogService.java
new file mode 100644
index 0000000..fe7020b
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/IcebergCatalogService.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.tools.sync.polaris.service;
+
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Wrapper around {@link org.apache.iceberg.catalog.Catalog} that exposes functionality
+ * that uses multiple Iceberg operations. For example, cascading drops of namespaces.
+ */
+public interface IcebergCatalogService {
+
+ // NAMESPACES
+ List<Namespace> listNamespaces(Namespace parentNamespace);
+ Map<String, String> loadNamespaceMetadata(Namespace namespace);
+ void createNamespace(Namespace namespace, Map<String, String> namespaceMetadata);
+ void setNamespaceProperties(Namespace namespace, Map<String, String> namespaceProperties);
+
+ /**
+ * Drop a namespace by first dropping all nested namespaces and tables underneath the namespace
+ * hierarchy. The root namespace, {@link Namespace#empty()}, will not be dropped.
+ * @param namespace the namespace to drop
+ */
+ void dropNamespaceCascade(Namespace namespace);
+
+ // TABLES
+ List<TableIdentifier> listTables(Namespace namespace);
+ Table loadTable(TableIdentifier tableIdentifier);
+ void registerTable(TableIdentifier tableIdentifier, String metadataFileLocation);
+ void dropTableWithoutPurge(TableIdentifier tableIdentifier);
+
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/PolarisService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/PolarisService.java
new file mode 100644
index 0000000..f011880
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/PolarisService.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.tools.sync.polaris.service;
+
+import org.apache.polaris.core.admin.model.Catalog;
+import org.apache.polaris.core.admin.model.CatalogRole;
+import org.apache.polaris.core.admin.model.GrantResource;
+import org.apache.polaris.core.admin.model.Principal;
+import org.apache.polaris.core.admin.model.PrincipalRole;
+import org.apache.polaris.core.admin.model.PrincipalWithCredentials;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Generic wrapper for a Polaris entity store.
+ */
+public interface PolarisService {
+
+ /**
+ * Called to perform initializing tasks for a Polaris entity store.
+ * @param properties the properties used to initialize the service
+ * @throws Exception
+ */
+ void initialize(Map<String, String> properties) throws Exception;
+
+ // PRINCIPALS
+ List<Principal> listPrincipals();
+ Principal getPrincipal(String principalName);
+ PrincipalWithCredentials createPrincipal(Principal principal);
+ void dropPrincipal(String principalName);
+
+ // PRINCIPAL ROLES
+ List<PrincipalRole> listPrincipalRoles();
+ PrincipalRole getPrincipalRole(String principalRoleName);
+ void createPrincipalRole(PrincipalRole principalRole);
+ void dropPrincipalRole(String principalRoleName);
+
+ // ASSIGNMENT OF PRINCIPAL ROLES TO PRINCIPALS
+ List<PrincipalRole> listPrincipalRolesAssigned(String principalName);
+ void assignPrincipalRole(String principalName, String principalRoleName);
+ void revokePrincipalRole(String principalName, String principalRoleName);
+
+ // CATALOGS
+ List<Catalog> listCatalogs();
+ Catalog getCatalog(String catalogName);
+ void createCatalog(Catalog catalog);
+ void dropCatalogCascade(String catalogName);
+
+ // CATALOG ROLES
+ List<CatalogRole> listCatalogRoles(String catalogName);
+ CatalogRole getCatalogRole(String catalogName, String catalogRoleName);
+ void createCatalogRole(String catalogName, CatalogRole catalogRole);
+ void dropCatalogRole(String catalogName, String catalogRoleName);
+
+ // ASSIGNMENT OF CATALOG ROLES TO CATALOGS
+ List<PrincipalRole> listAssigneePrincipalRolesForCatalogRole(String catalogName, String catalogRoleName);
+ void assignCatalogRole(String principalRoleName, String catalogName, String catalogRoleName);
+ void revokeCatalogRole(String principalRoleName, String catalogName, String catalogRoleName);
+
+ // GRANTS
+ List<GrantResource> listGrants(String catalogName, String catalogRoleName);
+ void addGrant(String catalogName, String catalogRoleName, GrantResource grant);
+ void revokeGrant(String catalogName, String catalogRoleName, GrantResource grant);
+
+ // ICEBERG
+ IcebergCatalogService initializeIcebergCatalogService(String catalogName);
+
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java
new file mode 100644
index 0000000..f454775
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisApiService.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.tools.sync.polaris.service.impl;
+
+import org.apache.http.HttpHeaders;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.polaris.core.admin.model.AddGrantRequest;
+import org.apache.polaris.core.admin.model.Catalog;
+import org.apache.polaris.core.admin.model.CatalogRole;
+import org.apache.polaris.core.admin.model.CreateCatalogRequest;
+import org.apache.polaris.core.admin.model.CreateCatalogRoleRequest;
+import org.apache.polaris.core.admin.model.CreatePrincipalRequest;
+import org.apache.polaris.core.admin.model.CreatePrincipalRoleRequest;
+import org.apache.polaris.core.admin.model.GrantCatalogRoleRequest;
+import org.apache.polaris.core.admin.model.GrantPrincipalRoleRequest;
+import org.apache.polaris.core.admin.model.GrantResource;
+import org.apache.polaris.core.admin.model.Principal;
+import org.apache.polaris.core.admin.model.PrincipalRole;
+import org.apache.polaris.core.admin.model.PrincipalWithCredentials;
+import org.apache.polaris.core.admin.model.PrincipalWithCredentialsCredentials;
+import org.apache.polaris.core.admin.model.RevokeGrantRequest;
+import org.apache.polaris.management.ApiClient;
+import org.apache.polaris.management.client.PolarisManagementDefaultApi;
+import org.apache.polaris.tools.sync.polaris.access.AccessControlService;
+import org.apache.polaris.tools.sync.polaris.http.OAuth2Util;
+import org.apache.polaris.tools.sync.polaris.service.IcebergCatalogService;
+import org.apache.polaris.tools.sync.polaris.service.PolarisService;
+
+import java.util.List;
+import java.util.Map;
+
+public class PolarisApiService implements PolarisService {
+
+ /**
+ * Property that denotes whether this service should create omnipotent principals
+ * with write access when initializing {@link org.apache.iceberg.catalog.Catalog}.
+ */
+ public static final String ICEBERG_WRITE_ACCESS_PROPERTY = "iceberg-write-access";
+
+ private Map<String, String> properties = null;
+
+ private String baseUrl = null;
+
+ private PolarisManagementDefaultApi api = null;
+
+ private AccessControlService accessControlService = null;
+
+ private PrincipalWithCredentials omnipotentPrincipal = null;
+
+ private PrincipalRole omnipotentPrincipalRole = null;
+
+ private boolean icebergWriteAccess = false;
+
+ public PolarisApiService() {}
+
+ @Override
+ public void initialize(Map<String, String> properties) throws Exception {
+ this.properties = properties;
+
+ String baseUrl = properties.get("base-url");
+ String token = properties.get("bearer-token");
+
+ if (token == null) {
+ String oauth2ServerUri = properties.get("oauth2-server-uri");
+ String clientId = properties.get("client-id");
+ String clientSecret = properties.get("client-secret");
+ String scope = properties.get("scope");
+
+ token = OAuth2Util.fetchToken(oauth2ServerUri, clientId, clientSecret, scope);
+ }
+
+ String bearerToken = token; // to make it effectively final to use it in a lambda
+
+ ApiClient client = new ApiClient();
+ client.updateBaseUri(baseUrl + "/api/management/v1");
+
+ // TODO: Add token refresh
+ client.setRequestInterceptor(requestBuilder ->
+ requestBuilder.header(HttpHeaders.AUTHORIZATION, "Bearer " + bearerToken));
+
+ this.baseUrl = baseUrl;
+ this.api = new PolarisManagementDefaultApi(client);
+
+ this.omnipotentPrincipal = new PrincipalWithCredentials()
+ .principal(new Principal().name(properties.get("omnipotent-principal-name")))
+ .credentials(new PrincipalWithCredentialsCredentials()
+ .clientId(properties.get("omnipotent-principal-client-id"))
+ .clientSecret(properties.get("omnipotent-principal-client-secret")));
+
+ this.accessControlService = new AccessControlService(this);
+ this.icebergWriteAccess = Boolean.parseBoolean(
+ properties.getOrDefault(ICEBERG_WRITE_ACCESS_PROPERTY, Boolean.toString(false))); // by default false
+ }
+
+ @Override
+ public List<Principal> listPrincipals() {
+ return this.api.listPrincipals().getPrincipals();
+ }
+
+ @Override
+ public Principal getPrincipal(String principalName) {
+ return this.api.getPrincipal(principalName);
+ }
+
+ @Override
+ public PrincipalWithCredentials createPrincipal(Principal principal) {
+ CreatePrincipalRequest request = new CreatePrincipalRequest().principal(principal);
+ return this.api.createPrincipal(request);
+ }
+
+ @Override
+ public void dropPrincipal(String principalName) {
+ this.api.deletePrincipal(principalName);
+ }
+
+ @Override
+ public List<PrincipalRole> listPrincipalRoles() {
+ return this.api.listPrincipalRoles().getRoles();
+ }
+
+ @Override
+ public PrincipalRole getPrincipalRole(String principalRoleName) {
+ return this.api.getPrincipalRole(principalRoleName);
+ }
+
+ @Override
+ public void createPrincipalRole(PrincipalRole principalRole) {
+ CreatePrincipalRoleRequest request = new CreatePrincipalRoleRequest().principalRole(principalRole);
+ this.api.createPrincipalRole(request);
+ }
+
+ @Override
+ public void dropPrincipalRole(String principalRoleName) {
+ this.api.deletePrincipalRole(principalRoleName);
+ }
+
+ @Override
+ public List<PrincipalRole> listPrincipalRolesAssigned(String principalName) {
+ return this.api.listPrincipalRolesAssigned(principalName).getRoles();
+ }
+
+ @Override
+ public void assignPrincipalRole(String principalName, String principalRoleName) {
+ GrantPrincipalRoleRequest request = new GrantPrincipalRoleRequest()
+ .principalRole(new PrincipalRole().name(principalRoleName));
+ this.api.assignPrincipalRole(principalName, request);
+ }
+
+ @Override
+ public void revokePrincipalRole(String principalName, String principalRoleName) {
+ this.api.revokePrincipalRole(principalName, principalRoleName);
+ }
+
+ @Override
+ public List<Catalog> listCatalogs() {
+ return this.api.listCatalogs().getCatalogs();
+ }
+
+ @Override
+ public Catalog getCatalog(String catalogName) {
+ return this.api.getCatalog(catalogName);
+ }
+
+ @Override
+ public void createCatalog(Catalog catalog) {
+ CreateCatalogRequest request = new CreateCatalogRequest().catalog(catalog);
+ this.api.createCatalog(request);
+ setupOmnipotentCatalogRoleIfNotExists(catalog.getName());
+ }
+
+ @Override
+ public void dropCatalogCascade(String catalogName) {
+ setupOmnipotentCatalogRoleIfNotExists(catalogName);
+ IcebergCatalogService icebergCatalogService = this.initializeIcebergCatalogService(catalogName);
+
+ // drop all namespaces within catalog
+ icebergCatalogService.dropNamespaceCascade(Namespace.empty());
+
+ List<CatalogRole> catalogRoles = this.listCatalogRoles(catalogName);
+
+ // drop all catalog roles within catalog
+ for (CatalogRole catalogRole : catalogRoles) {
+ if (!catalogRole.getName().equals("catalog_admin")) {
+ this.dropCatalogRole(catalogName, catalogRole.getName());
+ }
+ }
+
+ this.api.deleteCatalog(catalogName);
+ }
+
+ @Override
+ public List<CatalogRole> listCatalogRoles(String catalogName) {
+ return this.api.listCatalogRoles(catalogName).getRoles();
+ }
+
+ @Override
+ public CatalogRole getCatalogRole(String catalogName, String catalogRoleName) {
+ return this.api.getCatalogRole(catalogName, catalogRoleName);
+ }
+
+ @Override
+ public void createCatalogRole(String catalogName, CatalogRole catalogRole) {
+ CreateCatalogRoleRequest request = new CreateCatalogRoleRequest().catalogRole(catalogRole);
+ this.api.createCatalogRole(catalogName, request);
+ }
+
+ @Override
+ public void dropCatalogRole(String catalogName, String catalogRoleName) {
+ this.api.deleteCatalogRole(catalogName, catalogRoleName);
+ }
+
+ @Override
+ public List<PrincipalRole> listAssigneePrincipalRolesForCatalogRole(String catalogName, String catalogRoleName) {
+ return this.api.listAssigneePrincipalRolesForCatalogRole(catalogName, catalogRoleName).getRoles();
+ }
+
+ @Override
+ public void assignCatalogRole(String principalRoleName, String catalogName, String catalogRoleName) {
+ GrantCatalogRoleRequest request = new GrantCatalogRoleRequest()
+ .catalogRole(new CatalogRole().name(catalogRoleName));
+ this.api.assignCatalogRoleToPrincipalRole(principalRoleName, catalogName, request);
+ }
+
+ @Override
+ public void revokeCatalogRole(String principalRoleName, String catalogName, String catalogRoleName) {
+ this.api.revokeCatalogRoleFromPrincipalRole(principalRoleName, catalogName, catalogRoleName);
+ }
+
+ @Override
+ public List<GrantResource> listGrants(String catalogName, String catalogRoleName) {
+ return this.api.listGrantsForCatalogRole(catalogName, catalogRoleName).getGrants();
+ }
+
+ @Override
+ public void addGrant(String catalogName, String catalogRoleName, GrantResource grant) {
+ AddGrantRequest request = new AddGrantRequest().grant(grant);
+ this.api.addGrantToCatalogRole(catalogName, catalogRoleName, request);
+ }
+
+ @Override
+ public void revokeGrant(String catalogName, String catalogRoleName, GrantResource grant) {
+ RevokeGrantRequest request = new RevokeGrantRequest().grant(grant);
+ this.api.revokeGrantFromCatalogRole(catalogName, catalogRoleName, false, request);
+ }
+
+ private void setupOmnipotentCatalogRoleIfNotExists(String catalogName) {
+ if (this.omnipotentPrincipalRole == null) {
+ this.omnipotentPrincipalRole =
+ this.accessControlService.getOmnipotentPrincipalRoleForPrincipal(
+ omnipotentPrincipal.getPrincipal().getName());
+ }
+
+ if (!this.accessControlService.omnipotentCatalogRoleExists(catalogName)) {
+ this.accessControlService.setupOmnipotentRoleForCatalog(
+ catalogName,
+ omnipotentPrincipalRole,
+ false /* replace */,
+ icebergWriteAccess /* withWriteAccess */
+ );
+ }
+ }
+
+ @Override
+ public IcebergCatalogService initializeIcebergCatalogService(String catalogName) {
+ setupOmnipotentCatalogRoleIfNotExists(catalogName);
+ return new PolarisIcebergCatalogService(
+ baseUrl, catalogName, omnipotentPrincipal, properties);
+ }
+
+}
diff --git a/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisIcebergCatalogService.java b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisIcebergCatalogService.java
new file mode 100644
index 0000000..3955c67
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/java/org/apache/polaris/tools/sync/polaris/service/impl/PolarisIcebergCatalogService.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.tools.sync.polaris.service.impl;
+
+import org.apache.iceberg.CatalogUtil;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.rest.ResourcePaths;
+import org.apache.polaris.core.admin.model.PrincipalWithCredentials;
+import org.apache.polaris.tools.sync.polaris.catalog.PolarisCatalog;
+import org.apache.polaris.tools.sync.polaris.service.IcebergCatalogService;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class PolarisIcebergCatalogService implements IcebergCatalogService {
+
+ private final PolarisCatalog catalog;
+
+ public PolarisIcebergCatalogService(
+ String baseUrl,
+ String catalogName,
+ PrincipalWithCredentials omnipotentPrincipal,
+ Map<String, String> properties
+ ) {
+ Map<String, String> catalogProperties = new HashMap<>();
+ String uri = baseUrl + "/api/catalog";
+ catalogProperties.put("uri", uri);
+ catalogProperties.put("warehouse", catalogName);
+
+ // Default to /v1/oauth/tokens endpoint unless an explicit property was provided
+ String oauth2ServerUri = properties.getOrDefault(
+ "omnipotent-principal-oauth2-server-uri", uri + "/" + ResourcePaths.tokens());
+
+ catalogProperties.put("oauth2-server-uri", oauth2ServerUri);
+
+ String clientId = omnipotentPrincipal.getCredentials().getClientId();
+ String clientSecret = omnipotentPrincipal.getCredentials().getClientSecret();
+ catalogProperties.putIfAbsent(
+ "credential", String.format("%s:%s", clientId, clientSecret));
+ catalogProperties.putIfAbsent("scope", "PRINCIPAL_ROLE:ALL");
+
+ this.catalog = (PolarisCatalog) CatalogUtil.loadCatalog(
+ PolarisCatalog.class.getName(),
+ "SOURCE_CATALOG_REST_" + catalogName,
+ catalogProperties,
+ null
+ );
+ }
+
+ @Override
+ public List<Namespace> listNamespaces(Namespace parentNamespace) {
+ return this.catalog.listNamespaces(parentNamespace);
+ }
+
+ /**
+ * List all namespaces in hierarchy underneath a particular namespace in addition to all
+ * immediate children.
+ * @param parentNamespace the namespace to search for child namespaces under
+ * @return all child namespaces in hierarchy
+ */
+ private List<Namespace> listAllChildNamespaces(Namespace parentNamespace) {
+ List<Namespace> immediateChildren = this.listNamespaces(parentNamespace);
+
+ List<Namespace> allChildNamespaces = new ArrayList<>(immediateChildren);
+
+ for (Namespace childNamespace : immediateChildren) {
+ allChildNamespaces.addAll(this.listAllChildNamespaces(childNamespace));
+ }
+
+ return allChildNamespaces;
+ }
+
+ @Override
+ public Map<String, String> loadNamespaceMetadata(Namespace namespace) {
+ return this.catalog.loadNamespaceMetadata(namespace);
+ }
+
+ @Override
+ public void createNamespace(Namespace namespace, Map<String, String> namespaceMetadata) {
+ this.catalog.createNamespace(namespace, namespaceMetadata);
+ }
+
+ @Override
+ public void setNamespaceProperties(Namespace namespace, Map<String, String> namespaceProperties) {
+ this.catalog.setProperties(namespace, namespaceProperties);
+ }
+
+ @Override
+ public void dropNamespaceCascade(Namespace namespace) {
+ List<Namespace> allChildNamespaces = this.listAllChildNamespaces(namespace);
+
+ List<TableIdentifier> tables = new ArrayList<>();
+
+ for (Namespace childNamespace : allChildNamespaces) {
+ tables.addAll(this.catalog.listTables(childNamespace));
+ }
+
+ if (!namespace.isEmpty()) {
+ tables.addAll(this.catalog.listTables(namespace));
+ }
+
+ for (TableIdentifier tableIdentifier : tables) {
+ this.catalog.dropTable(tableIdentifier);
+ }
+
+ // go over in reverse order of namespaces since we discover namespaces
+ // in the parent -> child order, so we need to drop all children
+ // before we can drop the parent
+ for (Namespace childNamespace : allChildNamespaces.reversed()) {
+ this.catalog.dropNamespace(childNamespace);
+ }
+
+ if (!namespace.isEmpty()) {
+ this.catalog.dropNamespace(namespace);
+ }
+ }
+
+ @Override
+ public List<TableIdentifier> listTables(Namespace namespace) {
+ return this.catalog.listTables(namespace);
+ }
+
+ @Override
+ public Table loadTable(TableIdentifier tableIdentifier) {
+ return this.catalog.loadTable(tableIdentifier);
+ }
+
+ public Table loadTable(TableIdentifier tableIdentifier, String etag) {
+ return this.catalog.loadTable(tableIdentifier, etag);
+ }
+
+ @Override
+ public void registerTable(TableIdentifier tableIdentifier, String metadataFileLocation) {
+ this.catalog.registerTable(tableIdentifier, metadataFileLocation);
+ }
+
+ @Override
+ public void dropTableWithoutPurge(TableIdentifier tableIdentifier) {
+ this.catalog.dropTable(tableIdentifier, false /* purge */);
+ }
+
+}
diff --git a/polaris-synchronizer/api/src/main/resources/polaris-management-service.yml b/polaris-synchronizer/api/src/main/resources/polaris-management-service.yml
new file mode 100644
index 0000000..ecf374c
--- /dev/null
+++ b/polaris-synchronizer/api/src/main/resources/polaris-management-service.yml
@@ -0,0 +1,1432 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+openapi: 3.0.3
+info:
+ title: Polaris Management Service
+ version: 0.0.1
+ description:
+ Defines the management APIs for using Polaris to create and manage Iceberg catalogs and their principals
+servers:
+ - url: "{scheme}://{host}/api/management/v1"
+ description: Server URL when the port can be inferred from the scheme
+ variables:
+ scheme:
+ description: The scheme of the URI, either http or https.
+ default: https
+ host:
+ description: The host address for the specified server
+ default: localhost
+# All routes are currently configured using an Authorization header.
+security:
+ - OAuth2: []
+
+paths:
+ /catalogs:
+ get:
+ operationId: listCatalogs
+ description: List all catalogs in this polaris service
+ responses:
+ 200:
+ description: List of catalogs in the polaris service
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/Catalogs"
+ 403:
+ description: "The caller does not have permission to list catalog details"
+ post:
+ operationId: createCatalog
+ description: Add a new Catalog
+ requestBody:
+ description: The Catalog to create
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/CreateCatalogRequest"
+ responses:
+ 201:
+ description: "Successful response"
+ 403:
+ description: "The caller does not have permission to create a catalog"
+ 404:
+ description: "The catalog does not exist"
+ 409:
+ description: "A catalog with the specified name already exists"
+
+ /catalogs/{catalogName}:
+ parameters:
+ - name: catalogName
+ in: path
+ description: The name of the catalog
+ required: true
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ get:
+ operationId: getCatalog
+ description: Get the details of a catalog
+ responses:
+ 200:
+ description: The catalog details
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/Catalog"
+ 403:
+ description: "The caller does not have permission to read catalog details"
+ 404:
+ description: "The catalog does not exist"
+
+ put:
+ operationId: updateCatalog
+ description: Update an existing catalog
+ requestBody:
+ description: The catalog details to use in the update
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/UpdateCatalogRequest"
+ responses:
+ 200:
+ description: The catalog details
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/Catalog"
+ 403:
+ description: "The caller does not have permission to update catalog details"
+ 404:
+ description: "The catalog does not exist"
+ 409:
+ description: "The entity version doesn't match the currentEntityVersion; retry after fetching latest version"
+
+ delete:
+ operationId: deleteCatalog
+ description: Delete an existing catalog. The catalog must be empty.
+ responses:
+ 204:
+ description: "Success, no content"
+ 403:
+ description: "The caller does not have permission to delete a catalog"
+ 404:
+ description: "The catalog does not exist"
+
+ /principals:
+ get:
+ operationId: listPrincipals
+ description: List the principals for the current catalog
+ responses:
+ 200:
+ description: List of principals for this catalog
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/Principals"
+ 403:
+ description: "The caller does not have permission to list catalog admins"
+ 404:
+ description: "The catalog does not exist"
+
+ post:
+ operationId: createPrincipal
+ description: Create a principal
+ requestBody:
+ description: The principal to create
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/CreatePrincipalRequest"
+ responses:
+ 201:
+ description: "Successful response"
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/PrincipalWithCredentials"
+ 403:
+ description: "The caller does not have permission to add a principal"
+
+ /principals/{principalName}:
+ parameters:
+ - name: principalName
+ in: path
+ description: The principal name
+ required: true
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ get:
+ operationId: getPrincipal
+ description: Get the principal details
+ responses:
+ 200:
+ description: The requested principal
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/Principal"
+ 403:
+ description: "The caller does not have permission to get principal details"
+ 404:
+ description: "The catalog or principal does not exist"
+
+ put:
+ operationId: updatePrincipal
+ description: Update an existing principal
+ requestBody:
+ description: The principal details to use in the update
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/UpdatePrincipalRequest"
+ responses:
+ 200:
+ description: The updated principal
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/Principal"
+ 403:
+ description: "The caller does not have permission to update principal details"
+ 404:
+ description: "The principal does not exist"
+ 409:
+ description: "The entity version doesn't match the currentEntityVersion; retry after fetching latest version"
+
+ delete:
+ operationId: deletePrincipal
+ description: Remove a principal from polaris
+ responses:
+ 204:
+ description: "Success, no content"
+ 403:
+ description: "The caller does not have permission to delete a principal"
+ 404:
+ description: "The principal does not exist"
+
+ /principals/{principalName}/rotate:
+ parameters:
+ - name: principalName
+ in: path
+ description: The user name
+ required: true
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ post:
+ operationId: rotateCredentials
+ description: Rotate a principal's credentials. The new credentials will be returned in the response. This is the only
+ API, aside from createPrincipal, that returns the user's credentials. This API is *not* idempotent.
+ responses:
+ 200:
+ description: The principal details along with the newly rotated credentials
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/PrincipalWithCredentials"
+ 403:
+ description: "The caller does not have permission to rotate credentials"
+ 404:
+ description: "The principal does not exist"
+
+ /principals/{principalName}/principal-roles:
+ parameters:
+ - name: principalName
+ in: path
+ description: The name of the target principal
+ required: true
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ get:
+ operationId: listPrincipalRolesAssigned
+ description: List the roles assigned to the principal
+ responses:
+ 200:
+ description: List of roles assigned to this principal
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/PrincipalRoles"
+ 403:
+ description: "The caller does not have permission to list roles"
+ 404:
+ description: "The principal or catalog does not exist"
+
+ put:
+ operationId: assignPrincipalRole
+ description: Add a role to the principal
+ requestBody:
+ description: The principal role to assign
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/GrantPrincipalRoleRequest"
+ responses:
+ 201:
+ description: "Successful response"
+ 403:
+ description: "The caller does not have permission to add assign a role to the principal"
+ 404:
+ description: "The catalog, the principal, or the role does not exist"
+
+ /principals/{principalName}/principal-roles/{principalRoleName}:
+ parameters:
+ - name: principalName
+ in: path
+ description: The name of the target principal
+ required: true
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ - name: principalRoleName
+ in: path
+ description: The name of the role
+ required: true
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ delete:
+ operationId: revokePrincipalRole
+ description: Remove a role from a catalog principal
+ responses:
+ 204:
+ description: "Success, no content"
+ 403:
+ description: "The caller does not have permission to remove a role from the principal"
+ 404:
+ description: "The catalog or principal does not exist"
+
+ /principal-roles:
+ get:
+ operationId: listPrincipalRoles
+ description: List the principal roles
+ responses:
+ 200:
+ description: List of principal roles
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/PrincipalRoles"
+ 403:
+ description: "The caller does not have permission to list principal roles"
+ 404:
+ description: "The catalog does not exist"
+
+ post:
+ operationId: createPrincipalRole
+ description: Create a principal role
+ requestBody:
+ description: The principal to create
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/CreatePrincipalRoleRequest"
+ responses:
+ 201:
+ description: "Successful response"
+ 403:
+ description: "The caller does not have permission to add a principal role"
+
+ /principal-roles/{principalRoleName}:
+ parameters:
+ - name: principalRoleName
+ in: path
+ description: The principal role name
+ required: true
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ get:
+ operationId: getPrincipalRole
+ description: Get the principal role details
+ responses:
+ 200:
+ description: The requested principal role
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/PrincipalRole"
+ 403:
+ description: "The caller does not have permission to get principal role details"
+ 404:
+ description: "The principal role does not exist"
+
+ put:
+ operationId: updatePrincipalRole
+ description: Update an existing principalRole
+ requestBody:
+ description: The principalRole details to use in the update
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/UpdatePrincipalRoleRequest"
+ responses:
+ 200:
+ description: The updated principal role
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/PrincipalRole"
+ 403:
+ description: "The caller does not have permission to update principal role details"
+ 404:
+ description: "The principal role does not exist"
+ 409:
+ description: "The entity version doesn't match the currentEntityVersion; retry after fetching latest version"
+
+ delete:
+ operationId: deletePrincipalRole
+ description: Remove a principal role from polaris
+ responses:
+ 204:
+ description: "Success, no content"
+ 403:
+ description: "The caller does not have permission to delete a principal role"
+ 404:
+ description: "The principal role does not exist"
+
+ /principal-roles/{principalRoleName}/principals:
+ parameters:
+ - name: principalRoleName
+ in: path
+ description: The principal role name
+ required: true
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ get:
+ operationId: listAssigneePrincipalsForPrincipalRole
+ description: List the Principals to whom the target principal role has been assigned
+ responses:
+ 200:
+ description: List the Principals to whom the target principal role has been assigned
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/Principals"
+ 403:
+ description: "The caller does not have permission to list principals"
+ 404:
+ description: "The principal role does not exist"
+
+ /principal-roles/{principalRoleName}/catalog-roles/{catalogName}:
+ parameters:
+ - name: principalRoleName
+ in: path
+ description: The principal role name
+ required: true
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ - name: catalogName
+ in: path
+ required: true
+ description: The name of the catalog where the catalogRoles reside
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ get:
+ operationId: listCatalogRolesForPrincipalRole
+ description: Get the catalog roles mapped to the principal role
+ responses:
+ 200:
+ description: The list of catalog roles mapped to the principal role
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/CatalogRoles"
+ 403:
+ description: "The caller does not have permission to list catalog roles"
+ 404:
+ description: "The principal role does not exist"
+
+ put:
+ operationId: assignCatalogRoleToPrincipalRole
+ description: Assign a catalog role to a principal role
+ requestBody:
+ description: The principal to create
+ required: true
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/GrantCatalogRoleRequest"
+ responses:
+ 201:
+ description: "Successful response"
+ 403:
+ description: "The caller does not have permission to assign a catalog role"
+
+ /principal-roles/{principalRoleName}/catalog-roles/{catalogName}/{catalogRoleName}:
+ parameters:
+ - name: principalRoleName
+ in: path
+ description: The principal role name
+ required: true
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ - name: catalogName
+ in: path
+ description: The name of the catalog that contains the role to revoke
+ required: true
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ - name: catalogRoleName
+ in: path
+ description: The name of the catalog role that should be revoked
+ required: true
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ delete:
+ operationId: revokeCatalogRoleFromPrincipalRole
+ description: Remove a catalog role from a principal role
+ responses:
+ 204:
+ description: "Success, no content"
+ 403:
+ description: "The caller does not have permission to revoke a catalog role"
+ 404:
+ description: "The principal role does not exist"
+
+ /catalogs/{catalogName}/catalog-roles:
+ parameters:
+ - name: catalogName
+ in: path
+ description: The catalog for which we are reading/updating roles
+ required: true
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ get:
+ operationId: listCatalogRoles
+ description: List existing roles in the catalog
+ responses:
+ 200:
+ description: The list of roles that exist in this catalog
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/CatalogRoles"
+ post:
+ operationId: createCatalogRole
+ description: Create a new role in the catalog
+ requestBody:
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/CreateCatalogRoleRequest"
+ responses:
+ 201:
+ description: "Successful response"
+ 403:
+ description: "The principal is not authorized to create roles"
+ 404:
+ description: "The catalog does not exist"
+
+ /catalogs/{catalogName}/catalog-roles/{catalogRoleName}:
+ parameters:
+ - name: catalogName
+ in: path
+ description: The catalog for which we are retrieving roles
+ required: true
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ - name: catalogRoleName
+ in: path
+ description: The name of the role
+ required: true
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ get:
+ operationId: getCatalogRole
+ description: Get the details of an existing role
+ responses:
+ 200:
+ description: The specified role details
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/CatalogRole"
+ 403:
+ description: "The principal is not authorized to read role data"
+ 404:
+ description: "The catalog or the role does not exist"
+
+ put:
+ operationId: updateCatalogRole
+ description: Update an existing role in the catalog
+ requestBody:
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/UpdateCatalogRoleRequest"
+ responses:
+ 200:
+ description: The specified role details
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/CatalogRole"
+ 403:
+ description: "The principal is not authorized to update roles"
+ 404:
+ description: "The catalog or the role does not exist"
+ 409:
+ description: "The entity version doesn't match the currentEntityVersion; retry after fetching latest version"
+
+ delete:
+ operationId: deleteCatalogRole
+ description: Delete an existing role from the catalog. All associated grants will also be deleted
+ responses:
+ 204:
+ description: "Success, no content"
+ 403:
+ description: "The principal is not authorized to delete roles"
+ 404:
+ description: "The catalog or the role does not exist"
+
+ /catalogs/{catalogName}/catalog-roles/{catalogRoleName}/principal-roles:
+ parameters:
+ - name: catalogName
+ in: path
+ required: true
+ description: The name of the catalog where the catalog role resides
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ - name: catalogRoleName
+ in: path
+ required: true
+ description: The name of the catalog role
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ get:
+ operationId: listAssigneePrincipalRolesForCatalogRole
+ description: List the PrincipalRoles to which the target catalog role has been assigned
+ responses:
+ 200:
+ description: List the PrincipalRoles to which the target catalog role has been assigned
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/PrincipalRoles"
+ 403:
+ description: "The caller does not have permission to list principal roles"
+ 404:
+ description: "The catalog or catalog role does not exist"
+
+ /catalogs/{catalogName}/catalog-roles/{catalogRoleName}/grants:
+ parameters:
+ - name: catalogName
+ in: path
+ required: true
+ description: The name of the catalog where the role will receive the grant
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ - name: catalogRoleName
+ in: path
+ required: true
+ description: The name of the role receiving the grant (must exist)
+ schema:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ get:
+ operationId: listGrantsForCatalogRole
+ description: List the grants the catalog role holds
+ responses:
+ 200:
+ description: List of all grants given to the role in this catalog
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/GrantResources"
+ put:
+ operationId: addGrantToCatalogRole
+ description: Add a new grant to the catalog role
+ requestBody:
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/AddGrantRequest"
+ responses:
+ 201:
+ description: "Successful response"
+ 403:
+ description: "The principal is not authorized to create grants"
+ 404:
+ description: "The catalog or the role does not exist"
+ post:
+ operationId: revokeGrantFromCatalogRole
+ description:
+ Delete a specific grant from the role. This may be a subset or a superset of the grants the role has. In case of
+ a subset, the role will retain the grants not specified. If the `cascade` parameter is true, grant revocation
+ will have a cascading effect - that is, if a principal has specific grants on a subresource, and grants are revoked
+ on a parent resource, the grants present on the subresource will be revoked as well. By default, this behavior
+ is disabled and grant revocation only affects the specified resource.
+ parameters:
+ - name: cascade
+ in: query
+ schema:
+ type: boolean
+ default: false
+ description: If true, the grant revocation cascades to all subresources.
+ requestBody:
+ content:
+ application/json:
+ schema:
+ $ref: "#/components/schemas/RevokeGrantRequest"
+ responses:
+ 201:
+ description: "Successful response"
+ 403:
+ description: "The principal is not authorized to create grants"
+ 404:
+ description: "The catalog or the role does not exist"
+
+components:
+ securitySchemes:
+ OAuth2:
+ type: oauth2
+ description: Uses OAuth 2 with client credentials flow
+ flows:
+ implicit:
+ authorizationUrl: "{scheme}://{host}/api/v1/oauth/tokens"
+ scopes: {}
+
+ schemas:
+ Catalogs:
+ type: object
+ description: A list of Catalog objects
+ properties:
+ catalogs:
+ type: array
+ items:
+ $ref: "#/components/schemas/Catalog"
+ required:
+ - catalogs
+
+ CreateCatalogRequest:
+ type: object
+ description: Request to create a new catalog
+ properties:
+ catalog:
+ $ref: "#/components/schemas/Catalog"
+ required:
+ - catalog
+
+ Catalog:
+ type: object
+ description: A catalog object. A catalog may be internal or external. External catalogs are managed entirely by
+ an external catalog interface. Third party catalogs may be other Iceberg REST implementations or other services
+ with their own proprietary APIs
+ properties:
+ type:
+ type: string
+ enum:
+ - INTERNAL
+ - EXTERNAL
+ description: the type of catalog - internal or external
+ default: INTERNAL
+ name:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ description: The name of the catalog
+ properties:
+ type: object
+ properties:
+ default-base-location:
+ type: string
+ additionalProperties:
+ type: string
+ required:
+ - default-base-location
+ createTimestamp:
+ type: integer
+ format: "int64"
+ description: The creation time represented as unix epoch timestamp in milliseconds
+ lastUpdateTimestamp:
+ type: integer
+ format: "int64"
+ description: The last update time represented as unix epoch timestamp in milliseconds
+ entityVersion:
+ type: integer
+ description: The version of the catalog object used to determine if the catalog metadata has changed
+ storageConfigInfo:
+ $ref: "#/components/schemas/StorageConfigInfo"
+ required:
+ - name
+ - type
+ - storageConfigInfo
+ - properties
+ discriminator:
+ propertyName: type
+ mapping:
+ INTERNAL: "#/components/schemas/PolarisCatalog"
+ EXTERNAL: "#/components/schemas/ExternalCatalog"
+
+
+ PolarisCatalog:
+ type: object
+ allOf:
+ - $ref: "#/components/schemas/Catalog"
+ description: The base catalog type - this contains all the fields necessary to construct an INTERNAL catalog
+
+ ExternalCatalog:
+ description: An externally managed catalog
+ type: object
+ allOf:
+ - $ref: "#/components/schemas/Catalog"
+ - type: object
+ properties:
+ connectionConfigInfo:
+ $ref: "#/components/schemas/ConnectionConfigInfo"
+
+ ConnectionConfigInfo:
+ type: object
+ description: A connection configuration representing a remote catalog service. IMPORTANT - Specifying a
+ ConnectionConfigInfo in an ExternalCatalog is currently an experimental API and is subject to change.
+ properties:
+ connectionType:
+ type: string
+ enum:
+ - ICEBERG_REST
+ description: The type of remote catalog service represented by this connection
+ uri:
+ type: string
+ description: URI to the remote catalog service
+ authenticationParameters:
+ $ref: "#/components/schemas/AuthenticationParameters"
+ required:
+ - connectionType
+ discriminator:
+ propertyName: connectionType
+ mapping:
+ ICEBERG_REST: "#/components/schemas/IcebergRestConnectionConfigInfo"
+
+ IcebergRestConnectionConfigInfo:
+ type: object
+ description: Configuration necessary for connecting to an Iceberg REST Catalog
+ allOf:
+ - $ref: '#/components/schemas/ConnectionConfigInfo'
+ properties:
+ remoteCatalogName:
+ type: string
+ description: The name of a remote catalog instance within the remote catalog service; in some older systems
+ this is specified as the 'warehouse' when multiple logical catalogs are served under the same base
+ uri, and often translates into a 'prefix' added to all REST resource paths
+
+ AuthenticationParameters:
+ type: object
+ description: Authentication-specific information for a REST connection
+ properties:
+ authenticationType:
+ type: string
+ enum:
+ - OAUTH
+ - BEARER
+ description: The type of authentication to use when connecting to the remote rest service
+ required:
+ - authenticationType
+ discriminator:
+ propertyName: authenticationType
+ mapping:
+ OAUTH: "#/components/schemas/OAuthClientCredentialsParameters"
+ BEARER: "#/components/schemas/BearerAuthenticationParameters"
+
+ OAuthClientCredentialsParameters:
+ type: object
+ description: OAuth authentication based on client_id/client_secret
+ allOf:
+ - $ref: '#/components/schemas/AuthenticationParameters'
+ properties:
+ tokenUri:
+ type: string
+ description: Token server URI
+ clientId:
+ type: string
+ description: oauth client id
+ clientSecret:
+ type: string
+ format: password
+ description: oauth client secret (input-only)
+ scopes:
+ type: array
+ items:
+ type: string
+ description: oauth scopes to specify when exchanging for a short-lived access token
+
+ BearerAuthenticationParameters:
+ type: object
+ description: Bearer authentication directly embedded in request auth headers
+ allOf:
+ - $ref: '#/components/schemas/AuthenticationParameters'
+ properties:
+ bearerToken:
+ type: string
+ format: password
+ description: Bearer token (input-only)
+
+ StorageConfigInfo:
+ type: object
+ description: A storage configuration used by catalogs
+ properties:
+ storageType:
+ type: string
+ enum:
+ - S3
+ - GCS
+ - AZURE
+ - FILE
+ description: The cloud provider type this storage is built on. FILE is supported for testing purposes only
+ allowedLocations:
+ type: array
+ items:
+ type: string
+ example: "For AWS [s3://bucketname/prefix/], for AZURE [abfss://container@storageaccount.blob.core.windows.net/prefix/], for GCP [gs://bucketname/prefix/]"
+ required:
+ - storageType
+ discriminator:
+ propertyName: storageType
+ mapping:
+ S3: "#/components/schemas/AwsStorageConfigInfo"
+ AZURE: "#/components/schemas/AzureStorageConfigInfo"
+ GCS: "#/components/schemas/GcpStorageConfigInfo"
+ FILE: "#/components/schemas/FileStorageConfigInfo"
+
+ AwsStorageConfigInfo:
+ type: object
+ description: aws storage configuration info
+ allOf:
+ - $ref: '#/components/schemas/StorageConfigInfo'
+ properties:
+ roleArn:
+ type: string
+ description: the aws role arn that grants privileges on the S3 buckets
+ example: "arn:aws:iam::123456789001:principal/abc1-b-self1234"
+ externalId:
+ type: string
+ description: an optional external id used to establish a trust relationship with AWS in the trust policy
+ userArn:
+ type: string
+ description: the aws user arn used to assume the aws role
+ example: "arn:aws:iam::123456789001:user/abc1-b-self1234"
+ region:
+ type: string
+ description: the aws region where data is stored
+ example: "us-east-2"
+ required:
+ - roleArn
+
+ AzureStorageConfigInfo:
+ type: object
+ description: azure storage configuration info
+ allOf:
+ - $ref: '#/components/schemas/StorageConfigInfo'
+ properties:
+ tenantId:
+ type: string
+ description: the tenant id that the storage accounts belong to
+ multiTenantAppName:
+ type: string
+ description: the name of the azure client application
+ consentUrl:
+ type: string
+ description: URL to the Azure permissions request page
+ required:
+ - tenantId
+
+ GcpStorageConfigInfo:
+ type: object
+ description: gcp storage configuration info
+ allOf:
+ - $ref: '#/components/schemas/StorageConfigInfo'
+ properties:
+ gcsServiceAccount:
+ type: string
+ description: a Google cloud storage service account
+
+ FileStorageConfigInfo:
+ type: object
+ description: file storage configuration info
+ allOf:
+ - $ref: '#/components/schemas/StorageConfigInfo'
+
+ UpdateCatalogRequest:
+ description: Updates to apply to a Catalog. Any fields which are required in the Catalog
+ will remain unaltered if omitted from the contents of this Update request.
+ type: object
+ properties:
+ currentEntityVersion:
+ type: integer
+ description: The version of the object onto which this update is applied; if the object changed, the update will fail and the caller should retry after fetching the latest version.
+ properties:
+ type: object
+ additionalProperties:
+ type: string
+ storageConfigInfo:
+ $ref: "#/components/schemas/StorageConfigInfo"
+
+ Principals:
+ description: A list of Principals
+ type: object
+ properties:
+ principals:
+ type: array
+ items:
+ $ref: "#/components/schemas/Principal"
+ required:
+ - principals
+
+ PrincipalWithCredentials:
+ description: A user with its client id and secret. This type is returned when a new principal is created or when its
+ credentials are rotated
+ type: object
+ properties:
+ principal:
+ $ref: "#/components/schemas/Principal"
+ credentials:
+ type: object
+ properties:
+ clientId:
+ type: string
+ clientSecret:
+ type: string
+ format: password
+ required:
+ - principal
+ - credentials
+
+ CreatePrincipalRequest:
+ type: object
+ properties:
+ principal:
+ $ref: '#/components/schemas/Principal'
+ credentialRotationRequired:
+ type: boolean
+ description: If true, the initial credentials can only be used to call rotateCredentials
+
+ Principal:
+ description: A Polaris principal.
+ type: object
+ properties:
+ name:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ clientId:
+ type: string
+ description: The output-only OAuth clientId associated with this principal if applicable
+ properties:
+ type: object
+ additionalProperties:
+ type: string
+ createTimestamp:
+ type: integer
+ format: "int64"
+ lastUpdateTimestamp:
+ type: integer
+ format: "int64"
+ entityVersion:
+ type: integer
+ description: The version of the principal object used to determine if the principal metadata has changed
+ required:
+ - name
+
+ UpdatePrincipalRequest:
+ description: Updates to apply to a Principal
+ type: object
+ properties:
+ currentEntityVersion:
+ type: integer
+ description: The version of the object onto which this update is applied; if the object changed, the update will fail and the caller should retry after fetching the latest version.
+ properties:
+ type: object
+ additionalProperties:
+ type: string
+ required:
+ - currentEntityVersion
+ - properties
+
+ PrincipalRoles:
+ type: object
+ properties:
+ roles:
+ type: array
+ items:
+ $ref: "#/components/schemas/PrincipalRole"
+ required:
+ - roles
+
+ GrantPrincipalRoleRequest:
+ type: object
+ properties:
+ principalRole:
+ $ref: '#/components/schemas/PrincipalRole'
+
+ CreatePrincipalRoleRequest:
+ type: object
+ properties:
+ principalRole:
+ $ref: '#/components/schemas/PrincipalRole'
+
+ PrincipalRole:
+ type: object
+ properties:
+ name:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ description: The name of the role
+ properties:
+ type: object
+ additionalProperties:
+ type: string
+ createTimestamp:
+ type: integer
+ format: "int64"
+ lastUpdateTimestamp:
+ type: integer
+ format: "int64"
+ entityVersion:
+ type: integer
+ description: The version of the principal role object used to determine if the principal role metadata has changed
+ required:
+ - name
+
+ UpdatePrincipalRoleRequest:
+ description: Updates to apply to a Principal Role
+ type: object
+ properties:
+ currentEntityVersion:
+ type: integer
+ description: The version of the object onto which this update is applied; if the object changed, the update will fail and the caller should retry after fetching the latest version.
+ properties:
+ type: object
+ additionalProperties:
+ type: string
+ required:
+ - currentEntityVersion
+ - properties
+
+ CatalogRoles:
+ type: object
+ properties:
+ roles:
+ type: array
+ items:
+ $ref: "#/components/schemas/CatalogRole"
+ description: The list of catalog roles
+ required:
+ - roles
+
+ GrantCatalogRoleRequest:
+ type: object
+ properties:
+ catalogRole:
+ $ref: '#/components/schemas/CatalogRole'
+
+ CreateCatalogRoleRequest:
+ type: object
+ properties:
+ catalogRole:
+ $ref: '#/components/schemas/CatalogRole'
+
+ CatalogRole:
+ type: object
+ properties:
+ name:
+ type: string
+ minLength: 1
+ maxLength: 256
+ pattern: '^(?!\s*[s|S][y|Y][s|S][t|T][e|E][m|M]\$).*$'
+ description: The name of the role
+ properties:
+ type: object
+ additionalProperties:
+ type: string
+ createTimestamp:
+ type: integer
+ format: "int64"
+ lastUpdateTimestamp:
+ type: integer
+ format: "int64"
+ entityVersion:
+ type: integer
+ description: The version of the catalog role object used to determine if the catalog role metadata has changed
+ required:
+ - name
+
+ UpdateCatalogRoleRequest:
+ description: Updates to apply to a Catalog Role
+ type: object
+ properties:
+ currentEntityVersion:
+ type: integer
+ description: The version of the object onto which this update is applied; if the object changed, the update will fail and the caller should retry after fetching the latest version.
+ properties:
+ type: object
+ additionalProperties:
+ type: string
+ required:
+ - currentEntityVersion
+ - properties
+
+ ViewPrivilege:
+ type: string
+ enum:
+ - CATALOG_MANAGE_ACCESS
+ - VIEW_DROP
+ - VIEW_LIST
+ - VIEW_READ_PROPERTIES
+ - VIEW_WRITE_PROPERTIES
+ - VIEW_FULL_METADATA
+
+ TablePrivilege:
+ type: string
+ enum:
+ - CATALOG_MANAGE_ACCESS
+ - TABLE_DROP
+ - TABLE_LIST
+ - TABLE_READ_PROPERTIES
+ - TABLE_WRITE_PROPERTIES
+ - TABLE_READ_DATA
+ - TABLE_WRITE_DATA
+ - TABLE_FULL_METADATA
+
+ NamespacePrivilege:
+ type: string
+ enum:
+ - CATALOG_MANAGE_ACCESS
+ - CATALOG_MANAGE_CONTENT
+ - CATALOG_MANAGE_METADATA
+ - NAMESPACE_CREATE
+ - TABLE_CREATE
+ - VIEW_CREATE
+ - NAMESPACE_DROP
+ - TABLE_DROP
+ - VIEW_DROP
+ - NAMESPACE_LIST
+ - TABLE_LIST
+ - VIEW_LIST
+ - NAMESPACE_READ_PROPERTIES
+ - TABLE_READ_PROPERTIES
+ - VIEW_READ_PROPERTIES
+ - NAMESPACE_WRITE_PROPERTIES
+ - TABLE_WRITE_PROPERTIES
+ - VIEW_WRITE_PROPERTIES
+ - TABLE_READ_DATA
+ - TABLE_WRITE_DATA
+ - NAMESPACE_FULL_METADATA
+ - TABLE_FULL_METADATA
+ - VIEW_FULL_METADATA
+
+ CatalogPrivilege:
+ type: string
+ enum:
+ - CATALOG_MANAGE_ACCESS
+ - CATALOG_MANAGE_CONTENT
+ - CATALOG_MANAGE_METADATA
+ - CATALOG_READ_PROPERTIES
+ - CATALOG_WRITE_PROPERTIES
+ - NAMESPACE_CREATE
+ - TABLE_CREATE
+ - VIEW_CREATE
+ - NAMESPACE_DROP
+ - TABLE_DROP
+ - VIEW_DROP
+ - NAMESPACE_LIST
+ - TABLE_LIST
+ - VIEW_LIST
+ - NAMESPACE_READ_PROPERTIES
+ - TABLE_READ_PROPERTIES
+ - VIEW_READ_PROPERTIES
+ - NAMESPACE_WRITE_PROPERTIES
+ - TABLE_WRITE_PROPERTIES
+ - VIEW_WRITE_PROPERTIES
+ - TABLE_READ_DATA
+ - TABLE_WRITE_DATA
+ - NAMESPACE_FULL_METADATA
+ - TABLE_FULL_METADATA
+ - VIEW_FULL_METADATA
+
+ AddGrantRequest:
+ type: object
+ properties:
+ grant:
+ $ref: '#/components/schemas/GrantResource'
+
+ RevokeGrantRequest:
+ type: object
+ properties:
+ grant:
+ $ref: '#/components/schemas/GrantResource'
+
+ ViewGrant:
+ allOf:
+ - $ref: '#/components/schemas/GrantResource'
+ - type: object
+ properties:
+ namespace:
+ type: array
+ items:
+ type: string
+ viewName:
+ type: string
+ minLength: 1
+ maxLength: 256
+ privilege:
+ $ref: '#/components/schemas/ViewPrivilege'
+ required:
+ - namespace
+ - viewName
+ - privilege
+
+ TableGrant:
+ allOf:
+ - $ref: '#/components/schemas/GrantResource'
+ - type: object
+ properties:
+ namespace:
+ type: array
+ items:
+ type: string
+ tableName:
+ type: string
+ minLength: 1
+ maxLength: 256
+ privilege:
+ $ref: '#/components/schemas/TablePrivilege'
+ required:
+ - namespace
+ - tableName
+ - privilege
+
+ NamespaceGrant:
+ allOf:
+ - $ref: '#/components/schemas/GrantResource'
+ - type: object
+ properties:
+ namespace:
+ type: array
+ items:
+ type: string
+ privilege:
+ $ref: '#/components/schemas/NamespacePrivilege'
+ required:
+ - namespace
+ - privilege
+
+
+ CatalogGrant:
+ allOf:
+ - $ref: '#/components/schemas/GrantResource'
+ - type: object
+ properties:
+ privilege:
+ $ref: '#/components/schemas/CatalogPrivilege'
+ required:
+ - privilege
+
+ GrantResource:
+ type: object
+ discriminator:
+ propertyName: type
+ mapping:
+ catalog: '#/components/schemas/CatalogGrant'
+ namespace: '#/components/schemas/NamespaceGrant'
+ table: '#/components/schemas/TableGrant'
+ view: '#/components/schemas/ViewGrant'
+ properties:
+ type:
+ type: string
+ enum:
+ - catalog
+ - namespace
+ - table
+ - view
+ required:
+ - type
+
+ GrantResources:
+ type: object
+ properties:
+ grants:
+ type: array
+ items:
+ $ref: "#/components/schemas/GrantResource"
+ required:
+ - grants
\ No newline at end of file
diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java
new file mode 100644
index 0000000..6f6cbc7
--- /dev/null
+++ b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/AccessControlAwarePlannerTest.java
@@ -0,0 +1,212 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris;
+
+import java.util.List;
+import org.apache.polaris.core.admin.model.CatalogRole;
+import org.apache.polaris.core.admin.model.Principal;
+import org.apache.polaris.core.admin.model.PrincipalRole;
+import org.apache.polaris.tools.sync.polaris.access.AccessControlConstants;
+import org.apache.polaris.tools.sync.polaris.planning.AccessControlAwarePlanner;
+import org.apache.polaris.tools.sync.polaris.planning.NoOpSyncPlanner;
+import org.apache.polaris.tools.sync.polaris.planning.SynchronizationPlanner;
+import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class AccessControlAwarePlannerTest {
+
+ private static final Principal omnipotentPrincipalSource =
+ new Principal()
+ .name("omnipotent-principal-XXXXX")
+ .putPropertiesItem(AccessControlConstants.OMNIPOTENCE_PROPERTY, "");
+
+ private static final Principal omnipotentPrincipalTarget =
+ new Principal()
+ .name("omnipotent-principal-YYYYY")
+ .putPropertiesItem(AccessControlConstants.OMNIPOTENCE_PROPERTY, "");
+
+ @Test
+ public void filtersOmnipotentPrincipal() {
+ SynchronizationPlanner accessControlAwarePlanner
+ = new AccessControlAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<Principal> plan = accessControlAwarePlanner
+ .planPrincipalSync(List.of(omnipotentPrincipalSource), List.of(omnipotentPrincipalTarget));
+
+ Assertions.assertTrue(plan.entitiesToSkipAndSkipChildren().contains(omnipotentPrincipalSource));
+ Assertions.assertTrue(plan.entitiesToSkipAndSkipChildren().contains(omnipotentPrincipalTarget));
+ }
+
+ private static final Principal rootPrincipalSource = new Principal().name("root");
+
+ private static final Principal rootPrincipalTarget = new Principal().name("root");
+
+ @Test
+ public void filtersRootPrincipal() {
+ SynchronizationPlanner accessControlAwarePlanner
+ = new AccessControlAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<Principal> plan = accessControlAwarePlanner
+ .planPrincipalSync(List.of(rootPrincipalSource), List.of(rootPrincipalTarget));
+
+ Assertions.assertTrue(plan.entitiesToSkipAndSkipChildren().contains(rootPrincipalSource));
+ Assertions.assertTrue(plan.entitiesToSkipAndSkipChildren().contains(rootPrincipalTarget));
+ }
+
+ @Test
+ public void filtersPrincipalAssignmentToOmnipotentPrincipalRole() {
+ SynchronizationPlanner accessControlAwarePlanner
+ = new AccessControlAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<PrincipalRole> plan = accessControlAwarePlanner.planAssignPrincipalsToPrincipalRolesSync(
+ "principal", List.of(omnipotentPrincipalRoleSource), List.of(omnipotentPrincipalRoleTarget));
+
+ Assertions.assertTrue(plan.entitiesToSkip().contains(omnipotentPrincipalRoleSource));
+ Assertions.assertTrue(plan.entitiesToSkip().contains(omnipotentPrincipalRoleTarget));
+ }
+
+ @Test
+ public void filtersAssignmentToServiceAdmin() {
+ SynchronizationPlanner accessControlAwarePlanner
+ = new AccessControlAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<PrincipalRole> plan = accessControlAwarePlanner.planAssignPrincipalsToPrincipalRolesSync(
+ "principal", List.of(serviceAdminSource), List.of(serviceAdminTarget));
+
+ Assertions.assertTrue(plan.entitiesToSkip().contains(serviceAdminSource));
+ Assertions.assertTrue(plan.entitiesToSkip().contains(serviceAdminTarget));
+ }
+
+ private static final PrincipalRole omnipotentPrincipalRoleSource =
+ new PrincipalRole()
+ .name("omnipotent-principal-XXXXX")
+ .putPropertiesItem(AccessControlConstants.OMNIPOTENCE_PROPERTY, "");
+
+ private static final PrincipalRole omnipotentPrincipalRoleTarget =
+ new PrincipalRole()
+ .name("omnipotent-principal-YYYYY")
+ .putPropertiesItem(AccessControlConstants.OMNIPOTENCE_PROPERTY, "");
+
+ @Test
+ public void filtersOmnipotentPrincipalRoles() {
+ SynchronizationPlanner accessControlAwarePlanner =
+ new AccessControlAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<PrincipalRole> plan =
+ accessControlAwarePlanner.planPrincipalRoleSync(
+ List.of(omnipotentPrincipalRoleSource), List.of(omnipotentPrincipalRoleTarget));
+
+ Assertions.assertTrue(plan.entitiesToSkip().contains(omnipotentPrincipalRoleSource));
+ Assertions.assertTrue(plan.entitiesToSkip().contains(omnipotentPrincipalRoleTarget));
+ }
+
+ private static final PrincipalRole serviceAdminSource = new PrincipalRole().name("service_admin");
+
+ private static final PrincipalRole serviceAdminTarget = new PrincipalRole().name("service_admin");
+
+ @Test
+ public void filtersServiceAdmin() {
+ SynchronizationPlanner accessControlAwarePlanner =
+ new AccessControlAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<PrincipalRole> plan =
+ accessControlAwarePlanner.planPrincipalRoleSync(
+ List.of(serviceAdminSource), List.of(serviceAdminTarget));
+
+ Assertions.assertTrue(plan.entitiesToSkip().contains(serviceAdminSource));
+ Assertions.assertTrue(plan.entitiesToSkip().contains(serviceAdminTarget));
+ }
+
+ private static final CatalogRole omnipotentCatalogRoleSource =
+ new CatalogRole()
+ .name("omnipotent-principal-XXXXX")
+ .putPropertiesItem(AccessControlConstants.OMNIPOTENCE_PROPERTY, "");
+
+ private static final CatalogRole omnipotentCatalogRoleTarget =
+ new CatalogRole()
+ .name("omnipotent-principal-YYYYY")
+ .putPropertiesItem(AccessControlConstants.OMNIPOTENCE_PROPERTY, "");
+
+ @Test
+ public void filtersOmnipotentCatalogRolesAndChildren() {
+ SynchronizationPlanner accessControlAwarePlanner =
+ new AccessControlAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<CatalogRole> plan =
+ accessControlAwarePlanner.planCatalogRoleSync(
+ "catalogName",
+ List.of(omnipotentCatalogRoleSource),
+ List.of(omnipotentCatalogRoleTarget));
+
+ Assertions.assertTrue(
+ plan.entitiesToSkipAndSkipChildren().contains(omnipotentCatalogRoleSource));
+ Assertions.assertTrue(
+ plan.entitiesToSkipAndSkipChildren().contains(omnipotentCatalogRoleTarget));
+ }
+
+ private static final CatalogRole catalogAdminSource = new CatalogRole().name("catalog_admin");
+
+ private static final CatalogRole catalogAdminTarget = new CatalogRole().name("catalog_admin");
+
+ @Test
+ public void filtersCatalogAdminAndChildren() {
+ SynchronizationPlanner accessControlAwarePlanner =
+ new AccessControlAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<CatalogRole> plan =
+ accessControlAwarePlanner.planCatalogRoleSync(
+ "catalogName", List.of(catalogAdminSource), List.of(catalogAdminTarget));
+
+ Assertions.assertTrue(plan.entitiesToSkipAndSkipChildren().contains(catalogAdminSource));
+ Assertions.assertTrue(plan.entitiesToSkipAndSkipChildren().contains(catalogAdminTarget));
+ }
+
+ @Test
+ public void filtersOutAssignmentOfOmnipotentPrincipalRoles() {
+ SynchronizationPlanner accessControlAwarePlanner =
+ new AccessControlAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<PrincipalRole> plan =
+ accessControlAwarePlanner.planAssignPrincipalRolesToCatalogRolesSync(
+ "catalogName",
+ "catalogRoleName",
+ List.of(omnipotentPrincipalRoleSource),
+ List.of(omnipotentPrincipalRoleTarget));
+
+ Assertions.assertTrue(plan.entitiesToSkip().contains(omnipotentPrincipalRoleSource));
+ Assertions.assertTrue(plan.entitiesToSkip().contains(omnipotentPrincipalRoleTarget));
+ }
+
+ @Test
+ public void filtersOutAssignmentOfServiceAdmin() {
+ SynchronizationPlanner accessControlAwarePlanner =
+ new AccessControlAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<PrincipalRole> plan =
+ accessControlAwarePlanner.planAssignPrincipalRolesToCatalogRolesSync(
+ "catalogName",
+ "catalogRoleName",
+ List.of(serviceAdminSource),
+ List.of(serviceAdminTarget));
+
+ Assertions.assertTrue(plan.entitiesToSkip().contains(serviceAdminSource));
+ Assertions.assertTrue(plan.entitiesToSkip().contains(serviceAdminTarget));
+ }
+}
diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/ModificationAwarePlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/ModificationAwarePlannerTest.java
new file mode 100644
index 0000000..5be1788
--- /dev/null
+++ b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/ModificationAwarePlannerTest.java
@@ -0,0 +1,401 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris;
+
+import java.util.List;
+import org.apache.polaris.core.admin.model.AwsStorageConfigInfo;
+import org.apache.polaris.core.admin.model.AzureStorageConfigInfo;
+import org.apache.polaris.core.admin.model.Catalog;
+import org.apache.polaris.core.admin.model.CatalogGrant;
+import org.apache.polaris.core.admin.model.CatalogPrivilege;
+import org.apache.polaris.core.admin.model.CatalogProperties;
+import org.apache.polaris.core.admin.model.CatalogRole;
+import org.apache.polaris.core.admin.model.ExternalCatalog;
+import org.apache.polaris.core.admin.model.GcpStorageConfigInfo;
+import org.apache.polaris.core.admin.model.GrantResource;
+import org.apache.polaris.core.admin.model.PolarisCatalog;
+import org.apache.polaris.core.admin.model.Principal;
+import org.apache.polaris.core.admin.model.PrincipalRole;
+import org.apache.polaris.core.admin.model.StorageConfigInfo;
+import org.apache.polaris.tools.sync.polaris.planning.ModificationAwarePlanner;
+import org.apache.polaris.tools.sync.polaris.planning.NoOpSyncPlanner;
+import org.apache.polaris.tools.sync.polaris.planning.SynchronizationPlanner;
+import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class ModificationAwarePlannerTest {
+
+ private static final Principal principal = new Principal().name("principal");
+
+ private static final Principal modifiedPrincipal =
+ new Principal().name("principal").putPropertiesItem("newproperty", "newvalue");
+
+ private static final Principal principalWithClientId =
+ new Principal()
+ .name("principal")
+ .clientId("clientId");
+
+ private static final Principal principalWithResetClientId =
+ new Principal()
+ .name("principal")
+ .clientId("clientIdNew");
+
+ @Test
+ public void testPrincipalNotModified() {
+ SynchronizationPlanner modificationPlanner =
+ new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<Principal> plan =
+ modificationPlanner.planPrincipalSync(List.of(principal), List.of(principal));
+
+ Assertions.assertTrue(plan.entitiesNotModified().contains(principal));
+ }
+
+ @Test
+ public void testPrincipalModified() {
+ SynchronizationPlanner modificationPlanner =
+ new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<Principal> plan =
+ modificationPlanner.planPrincipalSync(List.of(principal), List.of(modifiedPrincipal));
+
+ Assertions.assertFalse(plan.entitiesNotModified().contains(principal));
+ }
+
+ @Test
+ public void testPrincipalNotModifiedWithResetClientId() {
+ SynchronizationPlanner modificationPlanner =
+ new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<Principal> plan =
+ modificationPlanner.planPrincipalSync(List.of(principalWithClientId), List.of(principalWithResetClientId));
+
+ Assertions.assertTrue(plan.entitiesNotModified().contains(principalWithClientId));
+ }
+
+ private static final PrincipalRole assignedToPrincipal = new PrincipalRole().name("assigned");
+
+ @Test
+ public void principalRoleAlreadyAssignedToPrincipal() {
+ SynchronizationPlanner modificationPlanner = new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<PrincipalRole> plan = modificationPlanner.planAssignPrincipalsToPrincipalRolesSync(
+ "principal", List.of(assignedToPrincipal), List.of(assignedToPrincipal));
+
+ Assertions.assertTrue(plan.entitiesNotModified().contains(assignedToPrincipal));
+ }
+
+ @Test
+ public void principalRoleNotAssignedToPrincipal() {
+ SynchronizationPlanner modificationPlanner = new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<PrincipalRole> plan = modificationPlanner.planAssignPrincipalsToPrincipalRolesSync(
+ "principal", List.of(assignedToPrincipal), List.of());
+
+ Assertions.assertFalse(plan.entitiesNotModified().contains(assignedToPrincipal));
+ }
+
+ private static final PrincipalRole principalRole = new PrincipalRole().name("principal-role");
+
+ private static final PrincipalRole modifiedPrincipalRole =
+ new PrincipalRole().name("principal-role").putPropertiesItem("newproperty", "newvalue");
+
+ @Test
+ public void testPrincipalRoleNotModified() {
+ SynchronizationPlanner modificationPlanner =
+ new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<PrincipalRole> plan =
+ modificationPlanner.planPrincipalRoleSync(List.of(principalRole), List.of(principalRole));
+
+ Assertions.assertTrue(plan.entitiesNotModified().contains(principalRole));
+ }
+
+ @Test
+ public void testPrincipalRoleModified() {
+ SynchronizationPlanner modificationPlanner =
+ new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<PrincipalRole> plan =
+ modificationPlanner.planPrincipalRoleSync(
+ List.of(principalRole), List.of(modifiedPrincipalRole));
+
+ Assertions.assertFalse(plan.entitiesNotModified().contains(principalRole));
+ }
+
+ private static final CatalogRole catalogRole = new CatalogRole().name("catalog-role");
+
+ private static final CatalogRole modifiedCatalogRole =
+ new CatalogRole().name("catalog-role").putPropertiesItem("newproperty", "newvalue");
+
+ @Test
+ public void testCatalogRoleNotModified() {
+ SynchronizationPlanner modificationPlanner =
+ new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<CatalogRole> plan =
+ modificationPlanner.planCatalogRoleSync(
+ "catalog", List.of(catalogRole), List.of(catalogRole));
+
+ Assertions.assertTrue(plan.entitiesNotModified().contains(catalogRole));
+ }
+
+ @Test
+ public void testCatalogRoleModified() {
+ SynchronizationPlanner modificationPlanner =
+ new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<CatalogRole> plan =
+ modificationPlanner.planCatalogRoleSync(
+ "catalog", List.of(catalogRole), List.of(modifiedCatalogRole));
+
+ Assertions.assertFalse(plan.entitiesNotModified().contains(catalogRole));
+ }
+
+ private static final GrantResource grant =
+ new GrantResource().type(GrantResource.TypeEnum.CATALOG);
+
+ @Test
+ public void testGrantNotRevoked() {
+ SynchronizationPlanner modificationPlanner =
+ new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<GrantResource> plan =
+ modificationPlanner.planGrantSync("catalog", "catalogRole", List.of(grant), List.of(grant));
+
+ Assertions.assertTrue(plan.entitiesNotModified().contains(grant));
+ }
+
+ private static final Catalog catalog =
+ new PolarisCatalog()
+ .name("catalog")
+ .type(Catalog.TypeEnum.INTERNAL)
+ .properties(new CatalogProperties())
+ .storageConfigInfo(
+ new AwsStorageConfigInfo()
+ .storageType(StorageConfigInfo.StorageTypeEnum.S3)
+ .roleArn("roleArn")
+ .userArn("userArn")
+ .externalId("externalId")
+ .region("region"));
+
+ private static final Catalog catalogWithTypeChange =
+ new ExternalCatalog()
+ .name("catalog")
+ .type(Catalog.TypeEnum.EXTERNAL) // changed type
+ .properties(new CatalogProperties())
+ .storageConfigInfo(
+ new AwsStorageConfigInfo()
+ .storageType(StorageConfigInfo.StorageTypeEnum.S3)
+ .roleArn("roleArn")
+ .userArn("userArn")
+ .externalId("externalId")
+ .region("region"));
+
+ private static final Catalog catalogWithStorageConfigInfoChange =
+ new PolarisCatalog()
+ .name("catalog")
+ .type(Catalog.TypeEnum.EXTERNAL) // changed type
+ .properties(new CatalogProperties())
+ .storageConfigInfo(
+ new AzureStorageConfigInfo()
+ .storageType(StorageConfigInfo.StorageTypeEnum.AZURE)
+ .consentUrl("consentUrl")
+ .tenantId("tenantId")
+ .multiTenantAppName("multiTenantAppName"));
+
+ private static final Catalog catalogWithOnlyUserArnChange =
+ new PolarisCatalog()
+ .name("catalog")
+ .type(Catalog.TypeEnum.INTERNAL)
+ .properties(new CatalogProperties())
+ .storageConfigInfo(
+ new AwsStorageConfigInfo()
+ .storageType(StorageConfigInfo.StorageTypeEnum.S3)
+ .roleArn("roleArn")
+ .userArn("userArnChanged") // only user arn changed
+ .externalId("externalId")
+ .region("region"));
+
+ private static final Catalog catalogWithPropertyChange =
+ new PolarisCatalog()
+ .name("catalog")
+ .type(Catalog.TypeEnum.INTERNAL)
+ .properties(new CatalogProperties().putAdditionalProperty("newproperty", "newvalue"))
+ .storageConfigInfo(
+ new AwsStorageConfigInfo()
+ .storageType(StorageConfigInfo.StorageTypeEnum.S3)
+ .roleArn("roleArn")
+ .userArn("userArn")
+ .externalId("externalId")
+ .region("region"));
+
+ private static final Catalog azureCatalog =
+ new PolarisCatalog()
+ .name("catalog")
+ .type(Catalog.TypeEnum.INTERNAL)
+ .properties(new CatalogProperties())
+ .storageConfigInfo(
+ new AzureStorageConfigInfo()
+ .storageType(StorageConfigInfo.StorageTypeEnum.AZURE)
+ .consentUrl("consentUrl")
+ .multiTenantAppName("multiTenantAppName")
+ .tenantId("tenantId"));
+
+ private static final Catalog azureCatalogConsentUrlAndTenantAppNameChange =
+ new PolarisCatalog()
+ .name("catalog")
+ .type(Catalog.TypeEnum.INTERNAL)
+ .properties(new CatalogProperties())
+ .storageConfigInfo(
+ new AzureStorageConfigInfo()
+ .storageType(StorageConfigInfo.StorageTypeEnum.AZURE)
+ .consentUrl("consentUrlChanged")
+ .multiTenantAppName("multiTenantAppNameChanged")
+ .tenantId("tenantId"));
+
+ private static final Catalog gcpCatalog =
+ new PolarisCatalog()
+ .name("catalog")
+ .type(Catalog.TypeEnum.INTERNAL)
+ .properties(new CatalogProperties())
+ .storageConfigInfo(
+ new GcpStorageConfigInfo()
+ .storageType(StorageConfigInfo.StorageTypeEnum.AZURE)
+ .gcsServiceAccount("gcsServiceAccount"));
+
+ private static final Catalog gcpCatalogGcsServiceAccountChange =
+ new PolarisCatalog()
+ .name("catalog")
+ .type(Catalog.TypeEnum.INTERNAL)
+ .properties(new CatalogProperties())
+ .storageConfigInfo(
+ new GcpStorageConfigInfo()
+ .storageType(StorageConfigInfo.StorageTypeEnum.AZURE)
+ .gcsServiceAccount("gcsServiceAccountChanged"));
+
+ @Test
+ public void testCatalogNotModified() {
+ SynchronizationPlanner modificationPlanner =
+ new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<Catalog> plan =
+ modificationPlanner.planCatalogSync(List.of(catalog), List.of(catalog));
+
+ Assertions.assertTrue(plan.entitiesNotModified().contains(catalog));
+ }
+
+ @Test
+ public void testCatalogTypeModified() {
+ SynchronizationPlanner modificationPlanner =
+ new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<Catalog> plan =
+ modificationPlanner.planCatalogSync(List.of(catalogWithTypeChange), List.of(catalog));
+
+ Assertions.assertFalse(plan.entitiesNotModified().contains(catalogWithTypeChange));
+ }
+
+ @Test
+ public void testCatalogStorageConfigInfoModified() {
+ SynchronizationPlanner modificationPlanner =
+ new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<Catalog> plan =
+ modificationPlanner.planCatalogSync(
+ List.of(catalogWithStorageConfigInfoChange), List.of(catalog));
+
+ Assertions.assertFalse(plan.entitiesNotModified().contains(catalogWithStorageConfigInfoChange));
+ }
+
+ @Test
+ public void testCatalogPropertiesModified() {
+ SynchronizationPlanner modificationPlanner =
+ new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<Catalog> plan =
+ modificationPlanner.planCatalogSync(List.of(catalogWithPropertyChange), List.of(catalog));
+
+ Assertions.assertFalse(plan.entitiesNotModified().contains(catalogWithPropertyChange));
+ }
+
+ @Test
+ public void testOnlyUserArnModifiedForAws() {
+ SynchronizationPlanner modificationPlanner =
+ new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<Catalog> plan =
+ modificationPlanner.planCatalogSync(
+ List.of(catalogWithOnlyUserArnChange), List.of(catalog));
+
+ Assertions.assertTrue(plan.entitiesNotModified().contains(catalogWithOnlyUserArnChange));
+ }
+
+ @Test
+ public void testOnlyConsentUrlAndTenantAppNameChangeAzure() {
+ SynchronizationPlanner modificationPlanner =
+ new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<Catalog> plan =
+ modificationPlanner.planCatalogSync(
+ List.of(azureCatalogConsentUrlAndTenantAppNameChange), List.of(azureCatalog));
+
+ Assertions.assertTrue(
+ plan.entitiesNotModified().contains(azureCatalogConsentUrlAndTenantAppNameChange));
+ }
+
+ @Test
+ public void testOnlyGcsServiceAccountChangeGCP() {
+ SynchronizationPlanner modificationPlanner =
+ new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<Catalog> plan =
+ modificationPlanner.planCatalogSync(
+ List.of(gcpCatalogGcsServiceAccountChange), List.of(gcpCatalog));
+
+ Assertions.assertTrue(plan.entitiesNotModified().contains(gcpCatalogGcsServiceAccountChange));
+ }
+
+ private final static PrincipalRole assignedToCatalogRole = new PrincipalRole().name("assigned");
+
+ @Test
+ public void principalRoleAlreadyAssignedToCatalogRole() {
+ SynchronizationPlanner modificationPlanner = new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<PrincipalRole> plan = modificationPlanner.planAssignPrincipalRolesToCatalogRolesSync(
+ "catalog",
+ "catalog-role",
+ List.of(assignedToCatalogRole), List.of(assignedToCatalogRole));
+
+ Assertions.assertTrue(plan.entitiesNotModified().contains(assignedToPrincipal));
+ }
+
+ @Test
+ public void principalRoleNotAssignedToCatalogRole() {
+ SynchronizationPlanner modificationPlanner = new ModificationAwarePlanner(new NoOpSyncPlanner());
+
+ SynchronizationPlan<PrincipalRole> plan = modificationPlanner.planAssignPrincipalRolesToCatalogRolesSync(
+ "catalog",
+ "catalog-role",
+ List.of(assignedToCatalogRole), List.of());
+
+ Assertions.assertFalse(plan.entitiesNotModified().contains(assignedToPrincipal));
+ }
+}
diff --git a/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java
new file mode 100644
index 0000000..18a961e
--- /dev/null
+++ b/polaris-synchronizer/api/src/test/java/org/apache/polaris/tools/sync/polaris/SourceParitySynchronizationPlannerTest.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris;
+
+import java.util.List;
+import java.util.Set;
+import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.polaris.core.admin.model.Catalog;
+import org.apache.polaris.core.admin.model.CatalogRole;
+import org.apache.polaris.core.admin.model.GrantResource;
+import org.apache.polaris.core.admin.model.Principal;
+import org.apache.polaris.core.admin.model.PrincipalRole;
+import org.apache.polaris.tools.sync.polaris.planning.SourceParitySynchronizationPlanner;
+import org.apache.polaris.tools.sync.polaris.planning.plan.SynchronizationPlan;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class SourceParitySynchronizationPlannerTest {
+
+ private static final Catalog CATALOG_1 = new Catalog().name("catalog-1");
+
+ private static final Catalog CATALOG_2 = new Catalog().name("catalog-2");
+
+ private static final Catalog CATALOG_3 = new Catalog().name("catalog-3");
+
+ @Test
+ public void testCreatesNewCatalogOverwritesOldCatalogRemovesDroppedCatalog() {
+ SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner();
+
+ SynchronizationPlan<Catalog> plan =
+ planner.planCatalogSync(List.of(CATALOG_1, CATALOG_2), List.of(CATALOG_2, CATALOG_3));
+
+ Assertions.assertTrue(plan.entitiesToCreate().contains(CATALOG_1));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(CATALOG_1));
+ Assertions.assertFalse(plan.entitiesToRemove().contains(CATALOG_1));
+
+ Assertions.assertFalse(plan.entitiesToCreate().contains(CATALOG_2));
+ Assertions.assertTrue(plan.entitiesToOverwrite().contains(CATALOG_2));
+ Assertions.assertFalse(plan.entitiesToRemove().contains(CATALOG_2));
+
+ Assertions.assertFalse(plan.entitiesToCreate().contains(CATALOG_3));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(CATALOG_3));
+ Assertions.assertTrue(plan.entitiesToRemove().contains(CATALOG_3));
+ }
+
+ private static final Principal PRINCIPAL_1 =
+ new Principal().name("principal-1");
+
+ private static final Principal PRINCIPAL_2 =
+ new Principal().name("principal-2");
+
+ private static final Principal PRINCIPAL_3 =
+ new Principal().name("principal-3");
+
+ @Test
+ public void testCreatesNewPrincipalOverwritesOldPrincipalRemovesDroppedPrincipal() {
+ SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner();
+
+ SynchronizationPlan<Principal> plan =
+ planner.planPrincipalSync(List.of(PRINCIPAL_1, PRINCIPAL_2), List.of(PRINCIPAL_2, PRINCIPAL_3));
+
+ Assertions.assertTrue(plan.entitiesToCreate().contains(PRINCIPAL_1));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(PRINCIPAL_1));
+ Assertions.assertFalse(plan.entitiesToRemove().contains(PRINCIPAL_1));
+
+ Assertions.assertFalse(plan.entitiesToCreate().contains(PRINCIPAL_2));
+ Assertions.assertTrue(plan.entitiesToOverwrite().contains(PRINCIPAL_2));
+ Assertions.assertFalse(plan.entitiesToRemove().contains(PRINCIPAL_2));
+
+ Assertions.assertFalse(plan.entitiesToCreate().contains(PRINCIPAL_3));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(PRINCIPAL_3));
+ Assertions.assertTrue(plan.entitiesToRemove().contains(PRINCIPAL_3));
+ }
+
+ private static final PrincipalRole ASSIGNED_TO_PRINCIPAL_1 =
+ new PrincipalRole().name("principal-role-1");
+
+ private static final PrincipalRole ASSIGNED_TO_PRINCIPAL_2 =
+ new PrincipalRole().name("principal-role-2");
+
+ private static final PrincipalRole ASSIGNED_TO_PRINCIPAL_3 =
+ new PrincipalRole().name("principal-role-3");
+
+ @Test
+ public void testAssignsNewPrincipalRoleRevokesDroppedPrincipalRoleForPrincipal() {
+ SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner();
+
+ SynchronizationPlan<PrincipalRole> plan =
+ planner.planAssignPrincipalsToPrincipalRolesSync(
+ "principal",
+ List.of(ASSIGNED_TO_PRINCIPAL_1, ASSIGNED_TO_PRINCIPAL_2),
+ List.of(ASSIGNED_TO_PRINCIPAL_2, ASSIGNED_TO_PRINCIPAL_3));
+
+ Assertions.assertTrue(plan.entitiesToCreate().contains(ASSIGNED_TO_PRINCIPAL_1));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_TO_PRINCIPAL_1));
+ Assertions.assertFalse(plan.entitiesToRemove().contains(ASSIGNED_TO_PRINCIPAL_1));
+
+ // special case: no concept of overwriting the assignment of a principal role
+ Assertions.assertFalse(plan.entitiesToCreate().contains(ASSIGNED_TO_PRINCIPAL_2));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_TO_PRINCIPAL_2));
+ Assertions.assertFalse(plan.entitiesToRemove().contains(ASSIGNED_TO_PRINCIPAL_2));
+
+ Assertions.assertFalse(plan.entitiesToCreate().contains(ASSIGNED_TO_PRINCIPAL_3));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_TO_PRINCIPAL_3));
+ Assertions.assertTrue(plan.entitiesToRemove().contains(ASSIGNED_TO_PRINCIPAL_3));
+ }
+
+ private static final PrincipalRole PRINCIPAL_ROLE_1 =
+ new PrincipalRole().name("principal-role-1");
+
+ private static final PrincipalRole PRINCIPAL_ROLE_2 =
+ new PrincipalRole().name("principal-role-2");
+
+ private static final PrincipalRole PRINCIPAL_ROLE_3 =
+ new PrincipalRole().name("principal-role-3");
+
+ @Test
+ public void testCreatesNewPrincipalRoleOverwritesOldPrincipalRoleRemovesDroppedPrincipalRole() {
+ SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner();
+
+ SynchronizationPlan<PrincipalRole> plan =
+ planner.planPrincipalRoleSync(
+ List.of(PRINCIPAL_ROLE_1, PRINCIPAL_ROLE_2),
+ List.of(PRINCIPAL_ROLE_2, PRINCIPAL_ROLE_3));
+
+ Assertions.assertTrue(plan.entitiesToCreate().contains(PRINCIPAL_ROLE_1));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(PRINCIPAL_ROLE_1));
+ Assertions.assertFalse(plan.entitiesToRemove().contains(PRINCIPAL_ROLE_1));
+
+ Assertions.assertFalse(plan.entitiesToCreate().contains(PRINCIPAL_ROLE_2));
+ Assertions.assertTrue(plan.entitiesToOverwrite().contains(PRINCIPAL_ROLE_2));
+ Assertions.assertFalse(plan.entitiesToRemove().contains(PRINCIPAL_ROLE_2));
+
+ Assertions.assertFalse(plan.entitiesToCreate().contains(PRINCIPAL_ROLE_3));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(PRINCIPAL_ROLE_3));
+ Assertions.assertTrue(plan.entitiesToRemove().contains(PRINCIPAL_ROLE_3));
+ }
+
+ private static final CatalogRole CATALOG_ROLE_1 = new CatalogRole().name("catalog-role-1");
+
+ private static final CatalogRole CATALOG_ROLE_2 = new CatalogRole().name("catalog-role-2");
+
+ private static final CatalogRole CATALOG_ROLE_3 = new CatalogRole().name("catalog-role-3");
+
+ @Test
+ public void testCreatesNewCatalogRoleOverwritesOldCatalogRoleRemovesDroppedCatalogRole() {
+ SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner();
+
+ SynchronizationPlan<CatalogRole> plan =
+ planner.planCatalogRoleSync(
+ "catalog",
+ List.of(CATALOG_ROLE_1, CATALOG_ROLE_2),
+ List.of(CATALOG_ROLE_2, CATALOG_ROLE_3));
+
+ Assertions.assertTrue(plan.entitiesToCreate().contains(CATALOG_ROLE_1));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(CATALOG_ROLE_1));
+ Assertions.assertFalse(plan.entitiesToRemove().contains(CATALOG_ROLE_1));
+
+ Assertions.assertFalse(plan.entitiesToCreate().contains(CATALOG_ROLE_2));
+ Assertions.assertTrue(plan.entitiesToOverwrite().contains(CATALOG_ROLE_2));
+ Assertions.assertFalse(plan.entitiesToRemove().contains(CATALOG_ROLE_2));
+
+ Assertions.assertFalse(plan.entitiesToCreate().contains(CATALOG_ROLE_3));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(CATALOG_ROLE_3));
+ Assertions.assertTrue(plan.entitiesToRemove().contains(CATALOG_ROLE_3));
+ }
+
+ private static final GrantResource GRANT_1 =
+ new GrantResource().type(GrantResource.TypeEnum.CATALOG);
+
+ private static final GrantResource GRANT_2 =
+ new GrantResource().type(GrantResource.TypeEnum.NAMESPACE);
+
+ private static final GrantResource GRANT_3 =
+ new GrantResource().type(GrantResource.TypeEnum.TABLE);
+
+ @Test
+ public void testCreatesNewGrantResourceRemovesDroppedGrantResource() {
+ SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner();
+
+ SynchronizationPlan<GrantResource> plan =
+ planner.planGrantSync(
+ "catalog", "catalogRole", List.of(GRANT_1, GRANT_2), List.of(GRANT_2, GRANT_3));
+
+ Assertions.assertTrue(plan.entitiesToCreate().contains(GRANT_1));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(GRANT_1));
+ Assertions.assertFalse(plan.entitiesToRemove().contains(GRANT_1));
+
+ // special case: no concept of overwriting a grant
+ Assertions.assertFalse(plan.entitiesToCreate().contains(GRANT_2));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(GRANT_2));
+ Assertions.assertFalse(plan.entitiesToRemove().contains(GRANT_2));
+
+ Assertions.assertFalse(plan.entitiesToCreate().contains(GRANT_3));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(GRANT_3));
+ Assertions.assertTrue(plan.entitiesToRemove().contains(GRANT_3));
+ }
+
+ private static final PrincipalRole ASSIGNED_TO_CATALOG_ROLE_1 =
+ new PrincipalRole().name("principal-role-1");
+
+ private static final PrincipalRole ASSIGNED_TO_CATALOG_ROLE_2 =
+ new PrincipalRole().name("principal-role-2");
+
+ private static final PrincipalRole ASSIGNED_TO_CATALOG_ROLE_3 =
+ new PrincipalRole().name("principal-role-3");
+
+ @Test
+ public void testAssignsNewPrincipalRoleRevokesDroppedPrincipalRole() {
+ SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner();
+
+ SynchronizationPlan<PrincipalRole> plan =
+ planner.planAssignPrincipalRolesToCatalogRolesSync(
+ "catalog",
+ "catalogRole",
+ List.of(ASSIGNED_TO_PRINCIPAL_1, ASSIGNED_TO_PRINCIPAL_2),
+ List.of(ASSIGNED_TO_PRINCIPAL_2, ASSIGNED_TO_PRINCIPAL_3));
+
+ Assertions.assertTrue(plan.entitiesToCreate().contains(ASSIGNED_TO_PRINCIPAL_1));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_TO_PRINCIPAL_1));
+ Assertions.assertFalse(plan.entitiesToRemove().contains(ASSIGNED_TO_PRINCIPAL_1));
+
+ // special case: no concept of overwriting the assignment of a principal role
+ Assertions.assertFalse(plan.entitiesToCreate().contains(ASSIGNED_TO_PRINCIPAL_2));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_TO_PRINCIPAL_2));
+ Assertions.assertFalse(plan.entitiesToRemove().contains(ASSIGNED_TO_PRINCIPAL_2));
+
+ Assertions.assertFalse(plan.entitiesToCreate().contains(ASSIGNED_TO_PRINCIPAL_3));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(ASSIGNED_TO_PRINCIPAL_3));
+ Assertions.assertTrue(plan.entitiesToRemove().contains(ASSIGNED_TO_PRINCIPAL_3));
+ }
+
+ private static final Namespace NS_1 = Namespace.of("ns1");
+
+ private static final Namespace NS_2 = Namespace.of("ns2");
+
+ private static final Namespace NS_3 = Namespace.of("ns3");
+
+ @Test
+ public void testCreatesNewNamespaceOverwritesOldNamespaceDropsDroppedNamespace() {
+ SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner();
+ SynchronizationPlan<Namespace> plan =
+ planner.planNamespaceSync(
+ "catalog", Namespace.empty(), List.of(NS_1, NS_2), List.of(NS_2, NS_3));
+
+ Assertions.assertTrue(plan.entitiesToCreate().contains(NS_1));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(NS_1));
+ Assertions.assertFalse(plan.entitiesToRemove().contains(NS_1));
+
+ Assertions.assertFalse(plan.entitiesToCreate().contains(NS_2));
+ Assertions.assertTrue(plan.entitiesToOverwrite().contains(NS_2));
+ Assertions.assertFalse(plan.entitiesToRemove().contains(NS_2));
+
+ Assertions.assertFalse(plan.entitiesToCreate().contains(NS_3));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(NS_3));
+ Assertions.assertTrue(plan.entitiesToRemove().contains(NS_3));
+ }
+
+ private static final TableIdentifier TABLE_1 = TableIdentifier.of("ns", "table1");
+
+ private static final TableIdentifier TABLE_2 = TableIdentifier.of("ns", "table2");
+
+ private static final TableIdentifier TABLE_3 = TableIdentifier.of("ns", "table3");
+
+ @Test
+ public void
+ testCreatesNewTableIdentifierOverwritesOldTableIdentifierRevokesDroppedTableIdentifier() {
+ SourceParitySynchronizationPlanner planner = new SourceParitySynchronizationPlanner();
+
+ SynchronizationPlan<TableIdentifier> plan =
+ planner.planTableSync(
+ "catalog", Namespace.empty(), Set.of(TABLE_1, TABLE_2), Set.of(TABLE_2, TABLE_3));
+
+ Assertions.assertTrue(plan.entitiesToCreate().contains(TABLE_1));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(TABLE_1));
+ Assertions.assertFalse(plan.entitiesToRemove().contains(TABLE_1));
+
+ Assertions.assertFalse(plan.entitiesToCreate().contains(TABLE_2));
+ Assertions.assertTrue(plan.entitiesToOverwrite().contains(TABLE_2));
+ Assertions.assertFalse(plan.entitiesToRemove().contains(TABLE_2));
+
+ Assertions.assertFalse(plan.entitiesToCreate().contains(TABLE_3));
+ Assertions.assertFalse(plan.entitiesToOverwrite().contains(TABLE_3));
+ Assertions.assertTrue(plan.entitiesToRemove().contains(TABLE_3));
+ }
+}
diff --git a/polaris-synchronizer/build.gradle.kts b/polaris-synchronizer/build.gradle.kts
new file mode 100644
index 0000000..a0f2bee
--- /dev/null
+++ b/polaris-synchronizer/build.gradle.kts
@@ -0,0 +1,24 @@
+plugins {
+ id("java")
+}
+
+group = "org.apache.polaris.tools"
+version = "0.1-SNAPSHOT"
+
+repositories {
+ mavenCentral()
+}
+
+dependencies {
+ testImplementation(platform("org.junit:junit-bom:5.10.0"))
+ testImplementation("org.junit.jupiter:junit-jupiter")
+}
+
+// don't build jar in the root project
+tasks.named<Jar>("jar").configure {
+ enabled = false
+}
+
+tasks.test {
+ useJUnitPlatform()
+}
\ No newline at end of file
diff --git a/polaris-synchronizer/cli/build.gradle.kts b/polaris-synchronizer/cli/build.gradle.kts
new file mode 100644
index 0000000..34cb4c6
--- /dev/null
+++ b/polaris-synchronizer/cli/build.gradle.kts
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar
+
+plugins {
+ `java-library`
+ `maven-publish`
+ signing
+ id("com.github.johnrengelman.shadow") version "8.1.1"
+}
+
+repositories {
+ mavenCentral()
+}
+
+dependencies {
+ implementation(project(":polaris-synchronizer-api"))
+
+ implementation("info.picocli:picocli:4.7.6")
+ implementation("org.slf4j:log4j-over-slf4j:2.0.17")
+ implementation("org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:1.7.1")
+ implementation("org.apache.commons:commons-csv:1.13.0")
+ runtimeOnly("ch.qos.logback:logback-classic:1.5.17")
+
+ testImplementation("org.junit.jupiter:junit-jupiter-params:5.10.0")
+ testImplementation("org.junit.jupiter:junit-jupiter-api:5.10.0")
+ testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.10.0")
+ testRuntimeOnly("org.junit.platform:junit-platform-launcher:1.10.0")
+}
+
+tasks.withType<Test>().configureEach { useJUnitPlatform() }
+
+val mainClassName = "org.apache.polaris.tools.sync.polaris.PolarisSynchronizerCLI"
+
+tasks.named<ShadowJar>("shadowJar") {
+ manifest {
+ attributes["Main-Class"] = mainClassName
+ }
+ outputs.cacheIf { false }
+ archiveClassifier.set("") // no `-all`
+ archiveVersion.set("") // optional
+ isZip64 = true
+}
+
+tasks.named("build") {
+ dependsOn("shadowJar")
+}
+
+tasks.named("assemble") {
+ dependsOn("shadowJar")
+}
+
+// Optionally disable raw JAR
+tasks.named<Jar>("jar") {
+ enabled = false
+}
diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java
new file mode 100644
index 0000000..df65f81
--- /dev/null
+++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CreateOmnipotentPrincipalCommand.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.polaris.core.admin.model.Catalog;
+import org.apache.polaris.core.admin.model.PrincipalRole;
+import org.apache.polaris.core.admin.model.PrincipalWithCredentials;
+import org.apache.polaris.tools.sync.polaris.access.AccessControlService;
+import org.apache.polaris.tools.sync.polaris.service.PolarisService;
+import org.apache.polaris.tools.sync.polaris.service.impl.PolarisApiService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+/**
+ * Command that creates the omnipotent principal to access entities internal to a catalog with the appropriate
+ * grants.
+ */
+@CommandLine.Command(
+ name = "create-omnipotent-principal",
+ mixinStandardHelpOptions = true,
+ sortOptions = false,
+ description =
+ "Creates a principal, associated principal role, and associated catalog role for each catalog "
+ + "with appropriate access permissions.")
+public class CreateOmnipotentPrincipalCommand implements Callable<Integer> {
+
+ private final Logger consoleLog = LoggerFactory.getLogger("console-log");
+
+ @CommandLine.Option(
+ names = {"--polaris-api-connection-properties"},
+ required = true,
+ description = "The connection properties to connect to the Polaris API." +
+ "\nProperties:" +
+ "\n\t- base-url: the base url of the Polaris instance (eg. http://localhost:8181)" +
+ "\n\t- bearer-token: the bearer token to authenticate against the Polaris instance with. Must " +
+ "be provided if any of oauth2-server-uri, client-id, client-secret, or scope are not provided." +
+ "\n\t- oauth2-server-uri: the uri of the OAuth2 server to authenticate to. (eg. http://localhost:8181/api/catalog/v1/oauth/tokens)" +
+ "\n\t- client-id: the client id belonging to a service admin to authenticate with" +
+ "\n\t- client-secret: the client secret belong to a service admin to authenticate with" +
+ "\n\t- scope: the scope to authenticate with for the service_admin (eg. PRINCIPAL_ROLE:ALL)"
+ )
+ private Map<String, String> polarisApiConnectionProperties;
+
+ @CommandLine.Option(
+ names = {"--replace"},
+ description = {
+ "Optional flag to enable overwriting the existing omnipotent principal and associated entity if it exists."
+ })
+ private boolean replace;
+
+ @CommandLine.Option(
+ names = {"--write-access"},
+ description = {
+ "Optional flag to create the principal with write access to every catalog. This is required if "
+ + "the Polaris instance is the target of a sync."
+ })
+ private boolean withWriteAccess;
+
+ @CommandLine.Option(
+ names = {"--concurrency"},
+ defaultValue = "1",
+ description = {
+ "Optional flag to specify the number of concurrent threads to use to setup catalog roles."
+ })
+ private int concurrency;
+
+ @Override
+ public Integer call() throws Exception {
+ polarisApiConnectionProperties.putIfAbsent(PolarisApiService.ICEBERG_WRITE_ACCESS_PROPERTY,
+ String.valueOf(withWriteAccess));
+
+ PolarisService polaris = PolarisServiceFactory.createPolarisService(
+ PolarisServiceFactory.ServiceType.API, polarisApiConnectionProperties);
+
+ AccessControlService accessControlService = new AccessControlService((PolarisApiService) polaris);
+
+ PrincipalWithCredentials principalWithCredentials;
+
+ try {
+ principalWithCredentials = accessControlService.createOmnipotentPrincipal(replace);
+ } catch (Exception e) {
+ consoleLog.error("Failed to create omnipotent principal.", e);
+ return 1;
+ }
+
+ consoleLog.info(
+ "Created omnipotent principal {}.", principalWithCredentials.getPrincipal().getName());
+
+ PrincipalRole principalRole;
+
+ try {
+ principalRole =
+ accessControlService.createAndAssignPrincipalRole(principalWithCredentials, replace);
+ } catch (Exception e) {
+ consoleLog.error("Failed to create omnipotent principal role and assign it to principal.", e);
+ return 1;
+ }
+
+ consoleLog.info(
+ "Created omnipotent principal role {} and assigned it to omnipotent principal {}.",
+ principalWithCredentials.getPrincipal().getName(),
+ principalRole.getName());
+
+ List<Catalog> catalogs = polaris.listCatalogs();
+
+ consoleLog.info("Identified {} catalogs to create catalog roles for.", catalogs.size());
+
+ final String permissionLevel = withWriteAccess ? "write" : "readonly";
+
+ AtomicInteger completedCatalogSetups = new AtomicInteger(0);
+
+ Queue<Catalog> failedCatalogs = new ConcurrentLinkedQueue<>();
+
+ ExecutorService executor = Executors.newFixedThreadPool(concurrency);
+
+ List<CompletableFuture<Void>> futures = new ArrayList<>();
+
+ for (Catalog catalog : catalogs) {
+ CompletableFuture<Void> future =
+ CompletableFuture.runAsync(
+ () -> {
+ try {
+ accessControlService.setupOmnipotentRoleForCatalog(
+ catalog.getName(), principalRole, replace, withWriteAccess);
+ } catch (Exception e) {
+ failedCatalogs.add(catalog);
+ consoleLog.error(
+ "Failed to setup omnipotent catalog role for catalog {} with {} access. - {}/{}",
+ catalog.getName(),
+ permissionLevel,
+ completedCatalogSetups.incrementAndGet(),
+ catalogs.size(),
+ e);
+ return;
+ }
+
+ consoleLog.info(
+ "Finished omnipotent principal setup for catalog {} with {} access. - {}/{}",
+ catalog.getName(),
+ permissionLevel,
+ completedCatalogSetups.incrementAndGet(),
+ catalogs.size());
+ },
+ executor);
+
+ futures.add(future);
+ }
+
+ futures.forEach(CompletableFuture::join);
+
+ consoleLog.info(
+ "Encountered issues creating catalog roles for the following catalogs: {}",
+ failedCatalogs.stream().map(Catalog::getName).toList());
+
+ consoleLog.info(
+ "\n======================================================\n"
+ + "Omnipotent Principal Credentials:\n"
+ + "\tname = {}\n"
+ + "\tclientId = {}\n"
+ + "\tclientSecret = {}\n"
+ + "======================================================",
+ principalWithCredentials.getPrincipal().getName(),
+ principalWithCredentials.getCredentials().getClientId(),
+ principalWithCredentials.getCredentials().getClientSecret());
+
+ return 0;
+ }
+}
diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvETagManager.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvETagManager.java
new file mode 100644
index 0000000..4bcaab5
--- /dev/null
+++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/CsvETagManager.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+import java.io.BufferedWriter;
+import java.io.Closeable;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVPrinter;
+import org.apache.commons.csv.CSVRecord;
+import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.polaris.tools.sync.polaris.catalog.ETagManager;
+
+/** Implementation that stores/loads ETags to/from a CSV file. */
+public class CsvETagManager implements ETagManager, Closeable {
+
+ public static final String CSV_FILE_PROPERTY = "csv-file";
+
+ private static final String CATALOG_HEADER = "Catalog";
+
+ private static final String TABLE_ID_HEADER = "TableIdentifier";
+
+ private static final String ETAG_HEADER = "ETag";
+
+ private static final String[] HEADERS = {CATALOG_HEADER, TABLE_ID_HEADER, ETAG_HEADER};
+
+ private File file;
+
+ private final Map<String, Map<TableIdentifier, String>> tablesByCatalogName;
+
+ public CsvETagManager() {
+ this.tablesByCatalogName = new HashMap<>();
+ }
+
+ @Override
+ public void initialize(Map<String, String> properties) {
+ if (!properties.containsKey(CSV_FILE_PROPERTY)) {
+ throw new IllegalArgumentException("Missing required property " + CSV_FILE_PROPERTY);
+ }
+
+ this.file = new File(properties.get(CSV_FILE_PROPERTY));
+
+ if (file.exists()) {
+ CSVFormat readerCSVFormat =
+ CSVFormat.DEFAULT.builder().setHeader(HEADERS).setSkipHeaderRecord(true).get();
+
+ CSVParser parser;
+
+ try {
+ parser = CSVParser.parse(Files.newBufferedReader(file.toPath(), UTF_8), readerCSVFormat);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ for (CSVRecord record : parser.getRecords()) {
+ this.tablesByCatalogName.putIfAbsent(record.get(CATALOG_HEADER), new HashMap<>());
+
+ TableIdentifier tableId = TableIdentifier.parse(record.get(TABLE_ID_HEADER));
+
+ this.tablesByCatalogName
+ .get(record.get(CATALOG_HEADER))
+ .put(tableId, record.get(ETAG_HEADER));
+ }
+
+ try {
+ parser.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public String getETag(String catalogName, TableIdentifier tableIdentifier) {
+ if (tablesByCatalogName.get(catalogName) != null) {
+ return tablesByCatalogName
+ .get(catalogName)
+ .get(tableIdentifier); // will return null anyway if table id not available
+ }
+ return null;
+ }
+
+ @Override
+ public void storeETag(String catalogName, TableIdentifier tableIdentifier, String etag) {
+ this.tablesByCatalogName.putIfAbsent(catalogName, new HashMap<>());
+ this.tablesByCatalogName.get(catalogName).put(tableIdentifier, etag);
+ }
+
+ @Override
+ public void close() throws IOException {
+ BufferedWriter writer = Files.newBufferedWriter(file.toPath(), UTF_8);
+
+ writer.write(""); // clear file
+
+ CSVFormat csvFormat = CSVFormat.DEFAULT.builder().setHeader(HEADERS).get();
+
+ CSVPrinter printer = new CSVPrinter(writer, csvFormat);
+
+ // write etags to file
+ tablesByCatalogName.forEach(
+ (catalogName, etagsByTable) -> {
+ etagsByTable.forEach(
+ (tableIdentifier, etag) -> {
+ try {
+ printer.printRecord(catalogName, tableIdentifier.toString(), etag);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ });
+
+ printer.flush();
+ printer.close();
+ }
+}
diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/ETagManagerFactory.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/ETagManagerFactory.java
new file mode 100644
index 0000000..9f1dc66
--- /dev/null
+++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/ETagManagerFactory.java
@@ -0,0 +1,67 @@
+package org.apache.polaris.tools.sync.polaris;
+
+import org.apache.polaris.tools.sync.polaris.catalog.ETagManager;
+import org.apache.polaris.tools.sync.polaris.catalog.NoOpETagManager;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Factory class to construct configurable {@link ETagManager} implementations.
+ */
+public class ETagManagerFactory {
+
+ /**
+ * Property that will hold class name for custom {@link ETagManager} implementation.
+ */
+ public static final String CUSTOM_CLASS_NAME_PROPERTY = "custom-impl";
+
+ private ETagManagerFactory() {}
+
+ /**
+ * Recognized types of {@link ETagManager} implementations
+ */
+ public enum Type {
+ NONE,
+ FILE,
+ CUSTOM
+ }
+
+ /**
+ * Construct a new {@link ETagManager} instance.
+ * @param type the recognized type of the {@link ETagManager} to construct
+ * @param properties properties to use when initializing the {@link ETagManager}
+ * @return the constructed and initialized {@link ETagManager}
+ */
+ public static ETagManager createETagManager(Type type, Map<String, String> properties) {
+ try {
+ properties = properties == null ? new HashMap<>() : properties;
+
+ ETagManager manager = switch (type) {
+ case NONE -> new NoOpETagManager();
+ case FILE -> new CsvETagManager();
+ case CUSTOM -> {
+ String customManagerClassname = properties.get(CUSTOM_CLASS_NAME_PROPERTY);
+
+ if (customManagerClassname == null) {
+ throw new IllegalArgumentException("Missing required property " + CUSTOM_CLASS_NAME_PROPERTY);
+ }
+
+ Object custom = Class.forName(customManagerClassname).getDeclaredConstructor().newInstance();
+
+ if (custom instanceof ETagManager customManager) {
+ yield customManager;
+ }
+
+ throw new InstantiationException("Custom ETagManager '" + customManagerClassname + "' does not implement ETagManager");
+ }
+ };
+
+ manager.initialize(properties);
+ return manager;
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to construct ETagManager", e);
+ }
+ }
+
+}
diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java
new file mode 100644
index 0000000..8d7ea28
--- /dev/null
+++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisServiceFactory.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.polaris.tools.sync.polaris;
+
+import org.apache.polaris.tools.sync.polaris.service.PolarisService;
+import org.apache.polaris.tools.sync.polaris.service.impl.PolarisApiService;
+
+import java.util.Map;
+
+public class PolarisServiceFactory {
+
+ public enum ServiceType {
+ API
+ }
+
+ /**
+ * Constructs and initializes a {@link PolarisService} implementation
+ * @param serviceType a recognized service type // TODO: support a custom implementation
+ * @param properties the properties to use to initialize the service
+ * @return the constructed and initialized service
+ */
+ public static PolarisService createPolarisService(
+ ServiceType serviceType,
+ Map<String, String> properties
+ ) {
+ PolarisService service = switch (serviceType) {
+ case API -> new PolarisApiService();
+ };
+
+ try {
+ service.initialize(properties);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ return service;
+ }
+
+}
diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizerCLI.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizerCLI.java
new file mode 100644
index 0000000..9cb3a57
--- /dev/null
+++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/PolarisSynchronizerCLI.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris;
+
+import picocli.CommandLine;
+
+/**
+ * Main entrypoint into the tool CLI. Sets up some base level configuration for the commands to share.
+ */
+@CommandLine.Command(
+ name = "polaris-synchronizer",
+ mixinStandardHelpOptions = true,
+ subcommands = {SyncPolarisCommand.class, CreateOmnipotentPrincipalCommand.class})
+public class PolarisSynchronizerCLI {
+
+ public PolarisSynchronizerCLI() {}
+
+ public static void main(String[] args) {
+ CommandLine commandLine =
+ new CommandLine(new PolarisSynchronizerCLI())
+ .setExecutionExceptionHandler(
+ (ex, cmd, parseResult) -> {
+ cmd.getErr().println(cmd.getColorScheme().richStackTraceString(ex)); // ensure stacktrace is printed
+ return 1;
+ });
+ commandLine.setUsageHelpWidth(150);
+ int exitCode = commandLine.execute(args);
+ System.exit(exitCode);
+ }
+}
diff --git a/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java
new file mode 100644
index 0000000..bfd64e2
--- /dev/null
+++ b/polaris-synchronizer/cli/src/main/java/org/apache/polaris/tools/sync/polaris/SyncPolarisCommand.java
@@ -0,0 +1,171 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.tools.sync.polaris;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import org.apache.polaris.tools.sync.polaris.catalog.ETagManager;
+import org.apache.polaris.tools.sync.polaris.planning.AccessControlAwarePlanner;
+import org.apache.polaris.tools.sync.polaris.planning.ModificationAwarePlanner;
+import org.apache.polaris.tools.sync.polaris.planning.SourceParitySynchronizationPlanner;
+import org.apache.polaris.tools.sync.polaris.planning.SynchronizationPlanner;
+import org.apache.polaris.tools.sync.polaris.service.PolarisService;
+import org.apache.polaris.tools.sync.polaris.service.impl.PolarisApiService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import picocli.CommandLine;
+
+/**
+ * Command to run the synchronization between a source and target Polaris instance.
+ */
+@CommandLine.Command(
+ name = "sync-polaris",
+ mixinStandardHelpOptions = true,
+ sortOptions = false,
+ description =
+ "Idempotent synchronization of one Polaris instance to another. Entities will not be removed from the source Polaris instance.")
+public class SyncPolarisCommand implements Callable<Integer> {
+
+ private final Logger consoleLog = LoggerFactory.getLogger("console-log");
+
+ @CommandLine.Option(
+ names = {"--source-properties"},
+ required = true,
+ description = "Properties to initialize Polaris entity source." +
+ "\nProperties:" +
+ "\n\t- base-url: the base url of the Polaris instance (eg. http://localhost:8181)" +
+ "\n\t- bearer-token: the bearer token to authenticate against the Polaris instance with. Must " +
+ "be provided if any of oauth2-server-uri, client-id, client-secret, or scope are not provided." +
+ "\n\t- oauth2-server-uri: the uri of the OAuth2 server to authenticate to. (eg. http://localhost:8181/api/catalog/v1/oauth/tokens)" +
+ "\n\t- client-id: the client id belonging to a service admin to authenticate with" +
+ "\n\t- client-secret: the client secret belong to a service admin to authenticate with" +
+ "\n\t- scope: the scope to authenticate with for the service_admin (eg. PRINCIPAL_ROLE:ALL)" +
+ "\nOmnipotent Principal Properties:" +
+ "\n\t- omnipotent-principal-name: the name of the omnipotent principal created using create-omnipotent-principal on the source Polaris" +
+ "\n\t- omnipotent-principal-client-id: the client id of the omnipotent principal created using create-omnipotent-principal on the source Polaris" +
+ "\n\t- omnipotent-principal-client-secret: the client secret of the omnipotent principal created using create-omnipotent-principal on the source Polaris" +
+ "\n\t- omnipotent-principal-oauth2-server-uri: (default: /v1/oauth/tokens endpoint for provided Polaris base-url) "
+ + "the OAuth2 server to use to authenticate the omnipotent-principal for Iceberg catalog access"
+ )
+ private Map<String, String> sourceProperties;
+
+ @CommandLine.Option(
+ names = {"--target-properties"},
+ required = true,
+ description = "Properties to initialize Polaris entity target." +
+ "\nProperties:" +
+ "\n\t- base-url: the base url of the Polaris instance (eg. http://localhost:8181)" +
+ "\n\t- bearer-token: the bearer token to authenticate against the Polaris instance with. Must " +
+ "be provided if any of oauth2-server-uri, client-id, client-secret, or scope are not provided." +
+ "\n\t- oauth2-server-uri: the uri of the OAuth2 server to authenticate to. (eg. http://localhost:8181/api/catalog/v1/oauth/tokens)" +
+ "\n\t- client-id: the client id belonging to a service admin to authenticate with" +
+ "\n\t- client-secret: the client secret belong to a service admin to authenticate with" +
+ "\n\t- scope: the scope to authenticate with for the service_admin (eg. PRINCIPAL_ROLE:ALL)" +
+ "\nOmnipotent Principal Properties:" +
+ "\n\t- omnipotent-principal-name: the name of the omnipotent principal created using create-omnipotent-principal on the target Polaris" +
+ "\n\t- omnipotent-principal-client-id: the client id of the omnipotent principal created using create-omnipotent-principal on the target Polaris" +
+ "\n\t- omnipotent-principal-client-secret: the client secret of the omnipotent principal created using create-omnipotent-principal on the target Polaris" +
+ "\n\t- omnipotent-principal-oauth2-server-uri: (default: /v1/oauth/tokens endpoint for provided Polaris base-url) "
+ + "the OAuth2 server to use to retrieve a bearer token for the omnipotent-principal"
+ )
+ private Map<String, String> targetProperties;
+
+ @CommandLine.Option(
+ names = {"--etag-storage-type"},
+ defaultValue = "NONE",
+ description = "One of { NONE, FILE, CUSTOM }. Default: NONE. The storage manager to use for storing ETags."
+ )
+ private ETagManagerFactory.Type etagManagerType;
+
+ @CommandLine.Option(
+ names = {"--etag-storage-properties"},
+ description = "Properties to initialize ETag storage." +
+ "\nFor type FILE:" +
+ "\n\t- " + CsvETagManager.CSV_FILE_PROPERTY + ": The CSV file to read ETags from and write ETags to." +
+ "\nFor type CUSTOM:" +
+ "\n\t- " + ETagManagerFactory.CUSTOM_CLASS_NAME_PROPERTY+ ": The classname for the custom ETagManager implementation."
+ )
+ private Map<String, String> etagManagerProperties;
+
+ @CommandLine.Option(
+ names = {"--sync-principals"},
+ description = "Enable synchronization of principals across the source and target, and assign them to " +
+ "the appropriate principal roles. WARNING: Principal client-id and client-secret will be reset on " +
+ "the target Polaris instance, and the new credentials for the principals created on the target will " +
+ "be logged to stdout."
+ )
+ private boolean shouldSyncPrincipals;
+
+ @CommandLine.Option(
+ names = {"--halt-on-failure"},
+ description = "Hard fail and stop the synchronization when an error occurs."
+ )
+ private boolean haltOnFailure;
+
+ @Override
+ public Integer call() throws Exception {
+ SynchronizationPlanner sourceParityPlanner = new SourceParitySynchronizationPlanner();
+ SynchronizationPlanner modificationAwareSourceParityPlanner = new ModificationAwarePlanner(sourceParityPlanner);
+ SynchronizationPlanner accessControlAwarePlanner = new AccessControlAwarePlanner(modificationAwareSourceParityPlanner);
+
+ // auto generate omnipotent principals with write access on the target, read only access on source
+ sourceProperties.put(PolarisApiService.ICEBERG_WRITE_ACCESS_PROPERTY, Boolean.toString(false));
+ targetProperties.put(PolarisApiService.ICEBERG_WRITE_ACCESS_PROPERTY, Boolean.toString(true));
+
+ PolarisService source =
+ PolarisServiceFactory.createPolarisService(PolarisServiceFactory.ServiceType.API, sourceProperties);
+ PolarisService target =
+ PolarisServiceFactory.createPolarisService(PolarisServiceFactory.ServiceType.API, targetProperties);
+
+ ETagManager etagService = ETagManagerFactory.createETagManager(etagManagerType, etagManagerProperties);
+
+ Runtime.getRuntime()
+ .addShutdownHook(
+ new Thread(
+ () -> {
+ if (etagService instanceof Closeable closableETagService) {
+ try {
+ closableETagService.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }));
+
+ PolarisSynchronizer synchronizer =
+ new PolarisSynchronizer(
+ consoleLog,
+ haltOnFailure,
+ accessControlAwarePlanner,
+ source,
+ target,
+ etagService);
+ synchronizer.syncPrincipalRoles();
+ if (shouldSyncPrincipals) {
+ consoleLog.warn("Principal migration will reset credentials on the target Polaris instance. " +
+ "Principal migration will log the new target Principal credentials to stdout.");
+ synchronizer.syncPrincipals();
+ }
+ synchronizer.syncCatalogs();
+
+ return 0;
+ }
+}
diff --git a/polaris-synchronizer/cli/src/main/resources/logback.xml b/polaris-synchronizer/cli/src/main/resources/logback.xml
new file mode 100644
index 0000000..663e03f
--- /dev/null
+++ b/polaris-synchronizer/cli/src/main/resources/logback.xml
@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<configuration>
+ <statusListener class="ch.qos.logback.core.status.NopStatusListener"/>
+
+ <appender name="console" class="ch.qos.logback.core.ConsoleAppender">
+ <withJansi>true</withJansi>
+ <encoder>
+ <pattern>%highlight(%-5level) - %msg%n</pattern>
+ </encoder>
+ </appender>
+
+ <logger name="console-log" level="INFO" additivity="false">
+ <appender-ref ref="console"/>
+ </logger>
+ <root>
+ <level value="INFO"/>
+ </root>
+</configuration>
diff --git a/polaris-synchronizer/cli/src/test/java/org/apache/polaris/tools/sync/polaris/ETagManagerFactoryTest.java b/polaris-synchronizer/cli/src/test/java/org/apache/polaris/tools/sync/polaris/ETagManagerFactoryTest.java
new file mode 100644
index 0000000..741d813
--- /dev/null
+++ b/polaris-synchronizer/cli/src/test/java/org/apache/polaris/tools/sync/polaris/ETagManagerFactoryTest.java
@@ -0,0 +1,41 @@
+package org.apache.polaris.tools.sync.polaris;
+
+import org.apache.polaris.tools.sync.polaris.catalog.NoOpETagManager;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+public class ETagManagerFactoryTest {
+
+ @Test
+ public void constructCsvManagerSuccessfully() {
+ // Pass correct properties for default file etag manager
+ ETagManagerFactory.createETagManager(
+ ETagManagerFactory.Type.FILE, Map.of(CsvETagManager.CSV_FILE_PROPERTY, "test.csv"));
+ }
+
+ @Test
+ public void failToConstructCsvManager() {
+ // omit properties for default file etag manager
+ Assertions.assertThrows(Exception.class, () ->
+ ETagManagerFactory.createETagManager(ETagManagerFactory.Type.FILE, Map.of()));
+ }
+
+ @Test
+ public void constructCustomETagManagerSuccessfully() {
+ // correctly construct custom manager by passing classname
+ ETagManagerFactory.createETagManager(
+ ETagManagerFactory.Type.CUSTOM,
+ Map.of(ETagManagerFactory.CUSTOM_CLASS_NAME_PROPERTY, NoOpETagManager.class.getName()) // use a recognized one for now
+ );
+ }
+
+ @Test
+ public void failToConstructCustomETagManager() {
+ // fail to construct custom etag manager by omitting custom classname from config
+ Assertions.assertThrows(Exception.class, () ->
+ ETagManagerFactory.createETagManager(ETagManagerFactory.Type.CUSTOM, Map.of()));
+ }
+
+}
diff --git a/polaris-synchronizer/gradle/wrapper/gradle-wrapper.properties b/polaris-synchronizer/gradle/wrapper/gradle-wrapper.properties
new file mode 100644
index 0000000..9355b41
--- /dev/null
+++ b/polaris-synchronizer/gradle/wrapper/gradle-wrapper.properties
@@ -0,0 +1,7 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip
+networkTimeout=10000
+validateDistributionUrl=true
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
diff --git a/polaris-synchronizer/gradlew b/polaris-synchronizer/gradlew
new file mode 100755
index 0000000..f5feea6
--- /dev/null
+++ b/polaris-synchronizer/gradlew
@@ -0,0 +1,252 @@
+#!/bin/sh
+
+#
+# Copyright © 2015-2021 the original authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+##############################################################################
+#
+# Gradle start up script for POSIX generated by Gradle.
+#
+# Important for running:
+#
+# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
+# noncompliant, but you have some other compliant shell such as ksh or
+# bash, then to run this script, type that shell name before the whole
+# command line, like:
+#
+# ksh Gradle
+#
+# Busybox and similar reduced shells will NOT work, because this script
+# requires all of these POSIX shell features:
+# * functions;
+# * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
+# «${var#prefix}», «${var%suffix}», and «$( cmd )»;
+# * compound commands having a testable exit status, especially «case»;
+# * various built-in commands including «command», «set», and «ulimit».
+#
+# Important for patching:
+#
+# (2) This script targets any POSIX shell, so it avoids extensions provided
+# by Bash, Ksh, etc; in particular arrays are avoided.
+#
+# The "traditional" practice of packing multiple parameters into a
+# space-separated string is a well documented source of bugs and security
+# problems, so this is (mostly) avoided, by progressively accumulating
+# options in "$@", and eventually passing that to Java.
+#
+# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
+# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
+# see the in-line comments for details.
+#
+# There are tweaks for specific operating systems such as AIX, CygWin,
+# Darwin, MinGW, and NonStop.
+#
+# (3) This script is generated from the Groovy template
+# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+# within the Gradle project.
+#
+# You can find Gradle at https://github.com/gradle/gradle/.
+#
+##############################################################################
+
+# Attempt to set APP_HOME
+
+# Resolve links: $0 may be a link
+app_path=$0
+
+# Need this for daisy-chained symlinks.
+while
+ APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path
+ [ -h "$app_path" ]
+do
+ ls=$( ls -ld "$app_path" )
+ link=${ls#*' -> '}
+ case $link in #(
+ /*) app_path=$link ;; #(
+ *) app_path=$APP_HOME$link ;;
+ esac
+done
+
+# This is normally unused
+# shellcheck disable=SC2034
+APP_BASE_NAME=${0##*/}
+# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
+' "$PWD" ) || exit
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD=maximum
+
+warn () {
+ echo "$*"
+} >&2
+
+die () {
+ echo
+ echo "$*"
+ echo
+ exit 1
+} >&2
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "$( uname )" in #(
+ CYGWIN* ) cygwin=true ;; #(
+ Darwin* ) darwin=true ;; #(
+ MSYS* | MINGW* ) msys=true ;; #(
+ NONSTOP* ) nonstop=true ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+ if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+ # IBM's JDK on AIX uses strange locations for the executables
+ JAVACMD=$JAVA_HOME/jre/sh/java
+ else
+ JAVACMD=$JAVA_HOME/bin/java
+ fi
+ if [ ! -x "$JAVACMD" ] ; then
+ die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+ fi
+else
+ JAVACMD=java
+ if ! command -v java >/dev/null 2>&1
+ then
+ die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+ fi
+fi
+
+# Increase the maximum file descriptors if we can.
+if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
+ case $MAX_FD in #(
+ max*)
+ # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
+ # shellcheck disable=SC2039,SC3045
+ MAX_FD=$( ulimit -H -n ) ||
+ warn "Could not query maximum file descriptor limit"
+ esac
+ case $MAX_FD in #(
+ '' | soft) :;; #(
+ *)
+ # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
+ # shellcheck disable=SC2039,SC3045
+ ulimit -n "$MAX_FD" ||
+ warn "Could not set maximum file descriptor limit to $MAX_FD"
+ esac
+fi
+
+# Collect all arguments for the java command, stacking in reverse order:
+# * args from the command line
+# * the main class name
+# * -classpath
+# * -D...appname settings
+# * --module-path (only if needed)
+# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
+
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if "$cygwin" || "$msys" ; then
+ APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
+ CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
+
+ JAVACMD=$( cygpath --unix "$JAVACMD" )
+
+ # Now convert the arguments - kludge to limit ourselves to /bin/sh
+ for arg do
+ if
+ case $arg in #(
+ -*) false ;; # don't mess with options #(
+ /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath
+ [ -e "$t" ] ;; #(
+ *) false ;;
+ esac
+ then
+ arg=$( cygpath --path --ignore --mixed "$arg" )
+ fi
+ # Roll the args list around exactly as many times as the number of
+ # args, so each arg winds up back in the position where it started, but
+ # possibly modified.
+ #
+ # NB: a `for` loop captures its iteration list before it begins, so
+ # changing the positional parameters here affects neither the number of
+ # iterations, nor the values presented in `arg`.
+ shift # remove old arg
+ set -- "$@" "$arg" # push replacement arg
+ done
+fi
+
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Collect all arguments for the java command:
+# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+# and any embedded shellness will be escaped.
+# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
+# treated as '${Hostname}' itself on the command line.
+
+set -- \
+ "-Dorg.gradle.appname=$APP_BASE_NAME" \
+ -classpath "$CLASSPATH" \
+ org.gradle.wrapper.GradleWrapperMain \
+ "$@"
+
+# Stop when "xargs" is not available.
+if ! command -v xargs >/dev/null 2>&1
+then
+ die "xargs is not available"
+fi
+
+# Use "xargs" to parse quoted args.
+#
+# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
+#
+# In Bash we could simply go:
+#
+# readarray ARGS < <( xargs -n1 <<<"$var" ) &&
+# set -- "${ARGS[@]}" "$@"
+#
+# but POSIX shell has neither arrays nor command substitution, so instead we
+# post-process each arg (as a line of input to sed) to backslash-escape any
+# character that might be a shell metacharacter, then use eval to reverse
+# that process (while maintaining the separation between arguments), and wrap
+# the whole thing up as a single "set" statement.
+#
+# This will of course break if any of these variables contains a newline or
+# an unmatched quote.
+#
+
+eval "set -- $(
+ printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
+ xargs -n1 |
+ sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
+ tr '\n' ' '
+ )" '"$@"'
+
+exec "$JAVACMD" "$@"
diff --git a/polaris-synchronizer/gradlew.bat b/polaris-synchronizer/gradlew.bat
new file mode 100644
index 0000000..9b42019
--- /dev/null
+++ b/polaris-synchronizer/gradlew.bat
@@ -0,0 +1,94 @@
+@rem
+@rem Copyright 2015 the original author or authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem https://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+@rem
+@rem SPDX-License-Identifier: Apache-2.0
+@rem
+
+@if "%DEBUG%"=="" @echo off
+@rem ##########################################################################
+@rem
+@rem Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%"=="" set DIRNAME=.
+@rem This is normally unused
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Resolve any "." and ".." in APP_HOME to make it shorter.
+for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if %ERRORLEVEL% equ 0 goto execute
+
+echo. 1>&2
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
+echo. 1>&2
+echo Please set the JAVA_HOME variable in your environment to match the 1>&2
+echo location of your Java installation. 1>&2
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto execute
+
+echo. 1>&2
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
+echo. 1>&2
+echo Please set the JAVA_HOME variable in your environment to match the 1>&2
+echo location of your Java installation. 1>&2
+
+goto fail
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
+
+:end
+@rem End local scope for the variables with windows NT shell
+if %ERRORLEVEL% equ 0 goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+set EXIT_CODE=%ERRORLEVEL%
+if %EXIT_CODE% equ 0 set EXIT_CODE=1
+if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%
+exit /b %EXIT_CODE%
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega
diff --git a/polaris-synchronizer/settings.gradle.kts b/polaris-synchronizer/settings.gradle.kts
new file mode 100644
index 0000000..2521c97
--- /dev/null
+++ b/polaris-synchronizer/settings.gradle.kts
@@ -0,0 +1,10 @@
+rootProject.name = "polaris-synchronizer"
+
+fun polarisSynchronizerProject(name: String) {
+ include("polaris-synchronizer-$name")
+ project(":polaris-synchronizer-$name").projectDir = file(name)
+}
+
+polarisSynchronizerProject("api")
+
+polarisSynchronizerProject("cli")
\ No newline at end of file