You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by yz...@apache.org on 2026/08/14 06:06:28 UTC
(polaris) branch main updated: Fix setup export policy name collisions (#5216)
This is an automated email from the ASF dual-hosted git repository.
MonkeyCanCode pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new ffa69775ec Fix setup export policy name collisions (#5216)
ffa69775ec is described below
commit ffa69775ec832a2e8308e3f75ffbc9b2d556c472
Author: Matt Faltyn <76...@users.noreply.github.com>
AuthorDate: Fri Aug 14 08:06:24 2026 +0200
Fix setup export policy name collisions (#5216)
* Fix setup export policy name collisions
Fixes #5215
* Simplify setup policy round-trip test
* Clarify setup export format compatibility
---
CHANGELOG.md | 4 ++
client/python/apache_polaris/cli/command/setup.py | 23 ++++---
client/python/tests/test_setup_command.py | 72 +++++++++++++++++++++-
.../assets/polaris/reference-setup-config.yaml | 6 +-
.../in-dev/unreleased/command-line-interface.md | 2 +
5 files changed, 94 insertions(+), 13 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index d6e7e32846..37bfbd85df 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -46,6 +46,10 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti
### Fixes
+- Python CLI `setup export` now writes each catalog's `policies` as a list of
+ `{name, namespace, ...}` entries instead of the previous name-keyed mapping, preserving policies
+ with the same name in different namespaces. The new export format cannot be applied by older CLI
+ versions; use the exporting CLI version or newer for `setup apply`.
- Python CLI `setup export` now preserves user-defined properties on principals and catalog roles,
so exported configurations restore that metadata during `setup apply`.
- Python CLI `setup apply` no longer double-encodes policy content emitted by `setup export`, so
diff --git a/client/python/apache_polaris/cli/command/setup.py b/client/python/apache_polaris/cli/command/setup.py
index 03a8528eab..251b8d64a4 100644
--- a/client/python/apache_polaris/cli/command/setup.py
+++ b/client/python/apache_polaris/cli/command/setup.py
@@ -21,7 +21,7 @@ import logging
import yaml
import json
from dataclasses import dataclass, field
-from typing import Dict, Optional, List, Any, Set
+from typing import Dict, Optional, List, Any, Set, Union
from apache_polaris.cli.command import Command
from apache_polaris.cli.exceptions import CliError, CLI_ERROR_EXIT_CODE
@@ -485,9 +485,9 @@ class SetupCommand(Command):
api: PolarisDefaultApi,
catalog_name: str,
namespaces: List[List[str]],
- ) -> Dict[str, Any]:
+ ) -> List[Dict[str, Any]]:
"""Export all policies for a given catalog."""
- policies_map: Dict[str, Any] = {}
+ policies_list: List[Dict[str, Any]] = []
catalog_api_client = self._get_catalog_api(api)
try:
policy_api = PolicyAPI(catalog_api_client)
@@ -514,18 +514,19 @@ class SetupCommand(Command):
except json.JSONDecodeError:
compact_content = policy_obj.content
policy_info = {
+ "name": p.name,
"namespace": ns_name,
"type": policy_obj.policy_type,
"content": compact_content,
}
if policy_obj.description:
policy_info["description"] = policy_obj.description
- policies_map[p.name] = policy_info
+ policies_list.append(policy_info)
except Exception:
self._record_failure(
f"Failed to export policies for catalog '{catalog_name}'"
)
- return policies_map
+ return policies_list
def _load_setup_config(self) -> Dict[str, Any]:
"""Load and cache the setup configuration from a YAML file."""
@@ -1287,14 +1288,22 @@ class SetupCommand(Command):
self,
api: PolarisDefaultApi,
catalog_name: str,
- policies_config: Dict[str, Any],
+ policies_config: Union[Dict[str, Any], List[Dict[str, Any]]],
dry_run: bool = False,
) -> None:
"""Create policies and attach them."""
logger.info(f"--- Processing policies for catalog: {catalog_name} ---")
catalog_api_client = self._get_catalog_api(api)
policy_api = PolicyAPI(catalog_api_client)
- for policy_name, policy_data in policies_config.items():
+ policy_entries = (
+ policies_config.items()
+ if isinstance(policies_config, dict)
+ else ((policy.get("name"), policy) for policy in policies_config)
+ )
+ for policy_name, policy_data in policy_entries:
+ if not policy_name:
+ logger.warning("Skipping policy due to missing name.")
+ continue
ns_name = policy_data.get("namespace")
if not ns_name:
logger.warning(
diff --git a/client/python/tests/test_setup_command.py b/client/python/tests/test_setup_command.py
index adb75c6623..eb53e41618 100644
--- a/client/python/tests/test_setup_command.py
+++ b/client/python/tests/test_setup_command.py
@@ -481,13 +481,14 @@ class TestSetupCommand(CLITestBase):
)
self.assertEqual(
catalog["policies"],
- {
- "child-policy": {
+ [
+ {
+ "name": "child-policy",
"namespace": "parent.child",
"type": "data-compaction",
"content": '{"max-age":7}',
}
- },
+ ],
)
self.assertEqual(
catalog_api.list_namespaces.call_args_list,
@@ -552,6 +553,71 @@ class TestSetupCommand(CLITestBase):
request = policy_api.create_policy.call_args.kwargs["create_policy_request"]
self.assertEqual(request.content, policy_content)
+ @patch("apache_polaris.cli.command.setup.PolicyAPI")
+ def test_setup_export_preserves_same_named_policies_across_namespaces(
+ self, mock_policy_api_class: MagicMock
+ ) -> None:
+ policy_api = mock_policy_api_class.return_value
+ policy_api.list_policies.side_effect = lambda prefix, namespace: SimpleNamespace(
+ identifiers=(
+ [SimpleNamespace(name="retention")]
+ if namespace in {"finance", "science"}
+ else []
+ )
+ )
+ policy_api.load_policy.side_effect = [
+ SimpleNamespace(
+ policy=SimpleNamespace(
+ content='{"max-age":7}',
+ policy_type="system.snapshot-expiry",
+ description=None,
+ )
+ ),
+ SimpleNamespace(
+ policy=SimpleNamespace(
+ content='{"max-age":30}',
+ policy_type="system.snapshot-expiry",
+ description=None,
+ )
+ ),
+ NotFoundException(),
+ NotFoundException(),
+ ]
+
+ export_command = SetupCommand(
+ setup_subcommand=Subcommands.EXPORT,
+ _catalog_api=MagicMock(),
+ )
+ exported_policies = export_command._export_policies_for_catalog(
+ MagicMock(), "catalog", [["finance"], ["science"]]
+ )
+ loaded_config = yaml.safe_load(yaml.safe_dump({"policies": exported_policies}))
+
+ apply_command = SetupCommand(
+ setup_subcommand=Subcommands.APPLY,
+ _catalog_api=MagicMock(),
+ )
+ apply_command._create_policies_and_attachments(
+ MagicMock(),
+ "catalog",
+ loaded_config["policies"],
+ )
+
+ self.assertEqual(
+ [
+ (
+ create_call.kwargs["namespace"],
+ create_call.kwargs["create_policy_request"].name,
+ create_call.kwargs["create_policy_request"].content,
+ )
+ for create_call in policy_api.create_policy.call_args_list
+ ],
+ [
+ ("finance", "retention", '{"max-age":7}'),
+ ("science", "retention", '{"max-age":30}'),
+ ],
+ )
+
def test_setup_export_reports_top_level_read_failures(self) -> None:
for method_name in (
"list_principals",
diff --git a/site/content/guides/assets/polaris/reference-setup-config.yaml b/site/content/guides/assets/polaris/reference-setup-config.yaml
index fe5c0c6243..e5fe8cc505 100644
--- a/site/content/guides/assets/polaris/reference-setup-config.yaml
+++ b/site/content/guides/assets/polaris/reference-setup-config.yaml
@@ -117,7 +117,7 @@ catalogs:
# --- Policies ---
# Defines policies that can be attached to entities in this catalog.
policies:
- compaction-policy:
+ - name: compaction-policy
namespace: dev_namespace
type: "system.data-compaction"
description: "Policy to enable data compaction for tables"
@@ -134,7 +134,7 @@ catalogs:
## commented out as table/view doesn't exist out of box
# attach:
# - { type: table-like, path: dev_namespace.sub_namespace.dev_table }
- snapshot-expiry-policy:
+ - name: snapshot-expiry-policy
namespace: dev_namespace
type: "system.snapshot-expiry"
description: "Policy to expire old snapshots"
@@ -150,7 +150,7 @@ catalogs:
}
attach:
- { type: namespace, path: dev_namespace }
- orphan-file-policy:
+ - name: orphan-file-policy
namespace: dev_namespace
type: "system.orphan-file-removal"
description: "Policy to remove orphan files"
diff --git a/site/content/in-dev/unreleased/command-line-interface.md b/site/content/in-dev/unreleased/command-line-interface.md
index c3f20476d2..2879afc4c0 100644
--- a/site/content/in-dev/unreleased/command-line-interface.md
+++ b/site/content/in-dev/unreleased/command-line-interface.md
@@ -1688,6 +1688,8 @@ polaris setup apply setup-config.yaml
The `export` subcommand retrieves the current Polaris configuration and outputs it in a YAML format. This output is compatible with the apply subcommand, allowing you to easily back up, migrate, or recreate your Polaris environment.
+Exported policies are represented as a list with explicit `name` and `namespace` fields so policies with the same name in different namespaces are preserved. The `apply` subcommand also accepts the legacy policy mapping keyed by name, but that format cannot represent duplicate policy names because YAML mapping keys must be unique.
+
```
usage: polaris setup export [-h] [options]