You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@polaris.apache.org by yz...@apache.org on 2026/04/22 13:29:38 UTC
(polaris) branch main updated: CLI: Optimize generate_clients.py with dirs pruning (#4243)
This is an automated email from the ASF dual-hosted git repository.
MonkeyCanCode pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/polaris.git
The following commit(s) were added to refs/heads/main by this push:
new 0a6d7e677d CLI: Optimize generate_clients.py with dirs pruning (#4243)
0a6d7e677d is described below
commit 0a6d7e677d4684985049e45957af3f161196551e
Author: Yong Zheng <yo...@gmail.com>
AuthorDate: Wed Apr 22 08:29:32 2026 -0500
CLI: Optimize generate_clients.py with dirs pruning (#4243)
---
Makefile | 2 +-
client/python/.openapi-generator-ignore | 46 ++----
client/python/generate_clients.py | 183 +++++++--------------
client/python/{test => tests}/cli_test_utils.py | 0
.../{test => tests}/test_catalog_roles_command.py | 0
.../{test => tests}/test_catalogs_command.py | 0
client/python/{test => tests}/test_find_command.py | 0
.../{test => tests}/test_namespaces_command.py | 0
client/python/{test => tests}/test_parser_basic.py | 0
.../{test => tests}/test_policies_command.py | 0
.../test_principal_roles_command.py | 0
.../{test => tests}/test_principals_command.py | 0
.../{test => tests}/test_privileges_command.py | 0
.../{test => tests}/test_profiles_command.py | 0
.../python/{test => tests}/test_setup_command.py | 0
.../python/{test => tests}/test_tables_command.py | 0
regtests/run.sh | 4 +-
17 files changed, 81 insertions(+), 154 deletions(-)
diff --git a/Makefile b/Makefile
index c5902b40bf..b9b4d71620 100644
--- a/Makefile
+++ b/Makefile
@@ -210,7 +210,7 @@ client-regenerate: client-setup-env ## Regenerate the client code
.PHONY: client-unit-test
client-unit-test: client-setup-env ## Run client unit tests
@echo "--- Running client unit tests ---"
- @$(ACTIVATE_AND_CD) && uv run --active pytest test/
+ @$(ACTIVATE_AND_CD) && uv run --active pytest tests/
@echo "--- Client unit tests complete ---"
##@ Helm
diff --git a/client/python/.openapi-generator-ignore b/client/python/.openapi-generator-ignore
index bed868e87e..36d9b89a6e 100644
--- a/client/python/.openapi-generator-ignore
+++ b/client/python/.openapi-generator-ignore
@@ -23,36 +23,26 @@
# Use this file to prevent files from being overwritten by the generator.
# The patterns follow closely to .gitignore or .dockerignore.
-# As an example, the C# client generator defines ApiClient.cs.
-# You can make changes and tell OpenAPI Generator to ignore just this file by uncommenting the following line:
-#ApiClient.cs
+# Standard project files that should not be overwritten
+pyproject.toml
+README.md
+LICENSE
+NOTICE
-# You can match any string of characters against a directory, file or extension with a single asterisk (*):
-#foo/*/qux
-# The above matches foo/bar/qux and foo/baz/qux, but not foo/bar/baz/qux
+# Source code
+apache_polaris/cli/
-# You can recursively match patterns against a directory, file or extension with a double asterisk (**):
-#foo/**/qux
-# This matches foo/bar/qux, foo/baz/qux, and foo/bar/baz/qux
+# Tests
+tests
+integration_tests
-# You can also negate patterns with an exclamation (!).
-# For example, you can ignore all files in a docs folder with the file extension .md:
-#docs/*.md
-# Then explicitly reverse the ignore rule for a single file:
-#!docs/README.md
+# Build and configuration files
+hatch_build.py
+generate_clients.py
+docker-compose.yml
+.pre-commit-config.yaml
+.gitignore
-pyproject.toml
-requirements.txt
-test-requirements.txt
-setup.py
-.gitlab-ci.yml
-.travis.yml
-.github/workflows/python.yml
-git_push.sh
-setup.cfg
-tox.ini
-README.md
-pyproject.toml
-docs/
+# Generated files to ignore
**/catalog_README.md
-**/management_README.md
\ No newline at end of file
+**/management_README.md
diff --git a/client/python/generate_clients.py b/client/python/generate_clients.py
index e265dddc42..f05dd1ef1c 100644
--- a/client/python/generate_clients.py
+++ b/client/python/generate_clients.py
@@ -44,59 +44,18 @@ PACKAGE_NAME_POLARIS_CATALOG = (
PYTHON_VERSION = "--additional-properties=pythonVersion=3.10"
# Cleanup
-KEEP_TEST_FILES = [
- Path("test/cli_test_utils.py"),
- Path("test/test_catalog_roles_command.py"),
- Path("test/test_catalogs_command.py"),
- Path("test/test_find_command.py"),
- Path("test/test_namespaces_command.py"),
- Path("test/test_parser_basic.py"),
- Path("test/test_policies_command.py"),
- Path("test/test_principal_roles_command.py"),
- Path("test/test_principals_command.py"),
- Path("test/test_privileges_command.py"),
- Path("test/test_profiles_command.py"),
- Path("test/test_setup_command.py"),
- Path("test/test_tables_command.py"),
-]
EXCLUDE_PATHS = [
- Path(".gitignore"),
- Path(".openapi-generator"),
- Path(".openapi-generator-ignore"),
- Path(".pytest_cache"),
- Path("test/test_cli_parsing.py"),
- Path("__pycache__"),
- Path("apache_polaris/__pycache__"),
Path("apache_polaris/cli"),
- Path("apache_polaris/sdk/__pycache__"),
- Path("apache_polaris/sdk/catalog/__pycache__"),
- Path("apache_polaris/sdk/catalog/models/__pycache__"),
- Path("apache_polaris/sdk/catalog/api/__pycache__"),
- Path("apache_polaris/sdk/management/__pycache__"),
- Path("apache_polaris/sdk/management/models/__pycache__"),
- Path("apache_polaris/sdk/management/api/__pycache__"),
Path("integration_tests"),
- Path(".github/workflows/python.yml"),
- Path(".gitlab-ci.yml"),
+ Path("tests"),
Path("pyproject.toml"),
Path("requirements.txt"),
- Path("test-requirements.txt"),
- Path("setup.py"),
- Path(".DS_Store"),
- Path("Makefile"),
- Path("poetry.lock"),
- Path("uv.lock"),
Path("docker-compose.yml"),
- Path(".pre-commit-config.yaml"),
Path("README.md"),
Path("generate_clients.py"),
Path("hatch_build.py"),
- Path(".venv"),
- Path("venv"),
- Path("dist"),
Path("templates"),
Path("spec"),
- Path("PKG-INFO"),
Path("LICENSE"),
Path("NOTICE"),
]
@@ -106,43 +65,12 @@ EXCLUDE_EXTENSIONS = [
"keep",
"gitignore",
"pyc",
+ "lock",
]
logger = logging.getLogger(__name__)
-def clean_old_tests() -> None:
- logger.info("Deleting old tests...")
- test_dir = CLIENT_DIR / "test"
- if not test_dir.exists():
- logger.info(f"Test directory {test_dir} does not exist, skipping test cleanup.")
- return
-
- for item in test_dir.rglob("*"):
- if item.is_file():
- # Check if the file should be kept relative to CLIENT_DIR
- relative_path = item.relative_to(CLIENT_DIR)
- if relative_path not in KEEP_TEST_FILES:
- try:
- os.remove(item)
- logger.debug(f"{relative_path}: removed")
- except OSError as e:
- logger.error(f"Error removing {relative_path}: {e}")
- else:
- logger.debug(f"{relative_path}: skipped")
-
- init_py_to_delete = CLIENT_DIR / "test" / "__init__.py"
- if init_py_to_delete.exists():
- try:
- os.remove(init_py_to_delete)
- logger.debug(f"{init_py_to_delete.relative_to(CLIENT_DIR)}: removed")
- except OSError as e:
- logger.error(
- f"Error removing {init_py_to_delete.relative_to(CLIENT_DIR)}: {e}"
- )
- logger.info("Old test deletion complete.")
-
-
def generate_polaris_management_client() -> None:
subprocess.check_call(
[
@@ -219,66 +147,76 @@ def generate_iceberg_catalog_client() -> None:
)
-def _prepend_header_to_file(file_path: Path, header_file_path: Path) -> None:
+def _prepend_header_to_file(file_path: Path, header_file_path: Path) -> bool:
try:
with open(header_file_path, "r") as hf:
header_content = hf.read()
- with open(file_path, "r+") as f:
+ with open(file_path, "r") as f:
original_content = f.read()
- f.seek(0)
+ if original_content.startswith(header_content):
+ return False
+ with open(file_path, "w") as f:
f.write(header_content + original_content)
+ return True
except IOError as e:
logger.error(f"Error prepending header to {file_path}: {e}")
+ return False
def prepend_licenses() -> None:
logger.info("Re-applying license headers...")
- # Combine all paths to exclude into one set.
- all_excluded_paths = set(EXCLUDE_PATHS) | set(KEEP_TEST_FILES)
-
- for file_path in CLIENT_DIR.rglob("*"):
- if not file_path.is_file():
- continue
-
- relative_file_path = file_path.relative_to(CLIENT_DIR)
-
- # Determine file extension, handling dotfiles.
- file_extension = ""
- if (
- relative_file_path.name.startswith(".")
- and "." not in relative_file_path.name[1:]
- ):
- file_extension = relative_file_path.name.lstrip(".")
- else:
- file_extension = relative_file_path.suffix.lstrip(".")
-
- if file_extension in EXCLUDE_EXTENSIONS:
- logger.debug(f"{relative_file_path}: skipped (extension excluded)")
- continue
-
- # Check if the path should be excluded.
- is_excluded = False
- for excluded_path in all_excluded_paths:
- if (
- relative_file_path == excluded_path
- or excluded_path in relative_file_path.parents
- ):
- is_excluded = True
- break
-
- if is_excluded:
- logger.debug(f"{relative_file_path}: skipped (path excluded)")
- continue
-
- header_file_path = HEADER_DIR / f"header-{file_extension}.txt"
-
- if header_file_path.is_file():
- _prepend_header_to_file(file_path, header_file_path)
- logger.debug(f"{relative_file_path}: updated")
- else:
- logger.error(f"No header compatible with file {relative_file_path}")
- sys.exit(2)
+ all_excluded_paths = set(EXCLUDE_PATHS)
+ tmp_dirs = {
+ "__pycache__",
+ "venv",
+ ".venv",
+ "build",
+ "dist",
+ ".pytest_cache",
+ ".ruff_cache",
+ ".mypy_cache",
+ }
+
+ for root, dirs, files in os.walk(CLIENT_DIR):
+ relative_root = Path(root).relative_to(CLIENT_DIR)
+ # Prune dirs in-place to prevent os.walk from entering them to reduce I/O
+ ## Prune hidden dirs
+ ## Prune temp (cache/builds/venv) dirs
+ ## Prune manual exclusion dirs
+ dirs[:] = [
+ d
+ for d in dirs
+ if not (
+ d.startswith(".")
+ or d in tmp_dirs
+ or (relative_root / d) in all_excluded_paths
+ )
+ ]
+ for file in files:
+ file_path = Path(root) / file
+ relative_file_path = file_path.relative_to(CLIENT_DIR)
+ if file.startswith(".") and "." not in file[1:]:
+ file_extension = file.lstrip(".")
+ else:
+ file_extension = file_path.suffix.lstrip(".")
+ if file_extension in EXCLUDE_EXTENSIONS:
+ logger.debug(f"{relative_file_path}: skipped (extension excluded)")
+ continue
+ if file.startswith(".") or relative_file_path in all_excluded_paths:
+ logger.debug(f"{relative_file_path}: skipped (file excluded)")
+ continue
+ header_file_path = HEADER_DIR / f"header-{file_extension}.txt"
+ if header_file_path.is_file():
+ if _prepend_header_to_file(file_path, header_file_path):
+ logger.debug(f"{relative_file_path}: updated")
+ else:
+ logger.debug(
+ f"{relative_file_path}: skipped (header already present)"
+ )
+ else:
+ logger.error(f"No header file found for {relative_file_path}")
+ sys.exit(2)
logger.info("License fix complete.")
@@ -308,7 +246,6 @@ def prepare_spec_dir() -> None:
def build() -> None:
prepare_spec_dir()
- clean_old_tests()
generate_polaris_management_client()
generate_polaris_catalog_client()
generate_iceberg_catalog_client()
diff --git a/client/python/test/cli_test_utils.py b/client/python/tests/cli_test_utils.py
similarity index 100%
rename from client/python/test/cli_test_utils.py
rename to client/python/tests/cli_test_utils.py
diff --git a/client/python/test/test_catalog_roles_command.py b/client/python/tests/test_catalog_roles_command.py
similarity index 100%
rename from client/python/test/test_catalog_roles_command.py
rename to client/python/tests/test_catalog_roles_command.py
diff --git a/client/python/test/test_catalogs_command.py b/client/python/tests/test_catalogs_command.py
similarity index 100%
rename from client/python/test/test_catalogs_command.py
rename to client/python/tests/test_catalogs_command.py
diff --git a/client/python/test/test_find_command.py b/client/python/tests/test_find_command.py
similarity index 100%
rename from client/python/test/test_find_command.py
rename to client/python/tests/test_find_command.py
diff --git a/client/python/test/test_namespaces_command.py b/client/python/tests/test_namespaces_command.py
similarity index 100%
rename from client/python/test/test_namespaces_command.py
rename to client/python/tests/test_namespaces_command.py
diff --git a/client/python/test/test_parser_basic.py b/client/python/tests/test_parser_basic.py
similarity index 100%
rename from client/python/test/test_parser_basic.py
rename to client/python/tests/test_parser_basic.py
diff --git a/client/python/test/test_policies_command.py b/client/python/tests/test_policies_command.py
similarity index 100%
rename from client/python/test/test_policies_command.py
rename to client/python/tests/test_policies_command.py
diff --git a/client/python/test/test_principal_roles_command.py b/client/python/tests/test_principal_roles_command.py
similarity index 100%
rename from client/python/test/test_principal_roles_command.py
rename to client/python/tests/test_principal_roles_command.py
diff --git a/client/python/test/test_principals_command.py b/client/python/tests/test_principals_command.py
similarity index 100%
rename from client/python/test/test_principals_command.py
rename to client/python/tests/test_principals_command.py
diff --git a/client/python/test/test_privileges_command.py b/client/python/tests/test_privileges_command.py
similarity index 100%
rename from client/python/test/test_privileges_command.py
rename to client/python/tests/test_privileges_command.py
diff --git a/client/python/test/test_profiles_command.py b/client/python/tests/test_profiles_command.py
similarity index 100%
rename from client/python/test/test_profiles_command.py
rename to client/python/tests/test_profiles_command.py
diff --git a/client/python/test/test_setup_command.py b/client/python/tests/test_setup_command.py
similarity index 100%
rename from client/python/test/test_setup_command.py
rename to client/python/tests/test_setup_command.py
diff --git a/client/python/test/test_tables_command.py b/client/python/tests/test_tables_command.py
similarity index 100%
rename from client/python/test/test_tables_command.py
rename to client/python/tests/test_tables_command.py
diff --git a/regtests/run.sh b/regtests/run.sh
index 838401c844..83afe162e1 100755
--- a/regtests/run.sh
+++ b/regtests/run.sh
@@ -54,7 +54,7 @@ cd ${REGTEST_HOME}
if [ -z "${1}" ]; then
loginfo 'Running all tests'
- TEST_LIST="../client/python/test $(find t_* -wholename '*t_*/src/*')"
+ TEST_LIST="../client/python/tests $(find t_* -wholename '*t_*/src/*')"
else
loginfo "Running single test ${1}"
TEST_LIST=${1}
@@ -116,7 +116,7 @@ echo "Root bearer token: ${REGTEST_ROOT_BEARER_TOKEN}"
for TEST_FILE in ${TEST_LIST}; do
# Special-case running all client pytests
- if [ "${TEST_FILE}" == '../client/python/test' ]; then
+ if [ "${TEST_FILE}" == '../client/python/tests' ]; then
loginfo "Starting pytest for entire client suite"
SCRIPT_DIR="$SCRIPT_DIR" python3 -m pytest ${TEST_FILE}
CODE=$?