You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2023/03/07 20:05:20 UTC
[airflow] 01/02: Require explicit flag to remove MyPy cache when running breeze stop (#29493)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-5-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 5abaf5d265e6a658bbce98bb5ea27adbe643884e
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Feb 12 18:01:00 2023 +0100
Require explicit flag to remove MyPy cache when running breeze stop (#29493)
The change #29184 added cleanup of MyPy cache when `breeze stop`
was called, but this is a bit too agressive. It will remove
the cache always when breeze stop is run and it will prolong
(on MacOS by 10s of seconds) the MyPy static checks when it happens.
Since cleaning the cache is only really needed when MyPy cache
gets broken (for example when we upgrade MyPy), cleaning it
explicitly when you encounter such problem is a better idea.
This PR:
* adds flag to breeze stop to clean the mypy cache (disabled by
default)
* adds documentation about that feature in STATIC_CHECKS
* adds hint displayed to the user when mypy check fails
(cherry picked from commit 8841537e58b864bbfb162692b12cf84f7df29001)
---
STATIC_CODE_CHECKS.rst | 8 ++++++++
.../airflow_breeze/commands/developer_commands.py | 13 ++++++++++--
.../commands/developer_commands_config.py | 1 +
images/breeze/output-commands-hash.txt | 2 +-
images/breeze/output_stop.svg | 24 +++++++++++++---------
scripts/ci/pre_commit/pre_commit_mypy.py | 3 ++-
6 files changed, 37 insertions(+), 14 deletions(-)
diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst
index b27d170d1c..8433fa3395 100644
--- a/STATIC_CODE_CHECKS.rst
+++ b/STATIC_CODE_CHECKS.rst
@@ -127,6 +127,14 @@ require Breeze Docker image to be build locally.
the image by setting ``SKIP_IMAGE_PRE_COMMITS`` to "true". This will mark the tests as "green" automatically
when run locally (note that those checks will anyway run in CI).
+.. note:: Mypy volume cache
+
+ MyPy uses a separate docker-volume (called ``mypy-cache-volume``) that keeps the cache of last MyPy
+ execution in order to speed MyPy checks up (sometimes by order of magnitude). While in most cases MyPy
+ will handle refreshing the cache when and if needed, there are some cases when it won't (cache invalidation
+ is the hard problem in computer science). This might happen for example when we upgrade MyPY. In such
+ cases you might need to manually remove the cache volume by running ``breeze stop --cleanup-mypy-cache``.
+
.. BEGIN AUTO-GENERATED STATIC CHECK LIST
+-----------------------------------------------------------+------------------------------------------------------------------+---------+
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
index c9562ca12b..9db96edff7 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -472,12 +472,18 @@ def compile_www_assets(dev: bool):
@click.option(
"-p",
"--preserve-volumes",
- help="Skip removing volumes when stopping Breeze.",
+ help="Skip removing database volumes when stopping Breeze.",
+ is_flag=True,
+)
+@click.option(
+ "-c",
+ "--cleanup-mypy-cache",
+ help="Additionally cleanup MyPy cache.",
is_flag=True,
)
@option_verbose
@option_dry_run
-def stop(preserve_volumes: bool):
+def stop(preserve_volumes: bool, cleanup_mypy_cache: bool):
perform_environment_checks()
command_to_execute = [*DOCKER_COMPOSE_COMMAND, "down", "--remove-orphans"]
if not preserve_volumes:
@@ -485,6 +491,9 @@ def stop(preserve_volumes: bool):
shell_params = ShellParams(backend="all", include_mypy_volume=True)
env_variables = get_env_variables_for_docker_commands(shell_params)
run_command(command_to_execute, env=env_variables)
+ if cleanup_mypy_cache:
+ command_to_execute = ["docker", "volume", "rm", "--force", "mypy-cache-volume"]
+ run_command(command_to_execute, env=env_variables)
@main.command(name="exec", help="Joins the interactive shell of running airflow container.")
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py
index 073cc953c7..4265ab286c 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py
@@ -134,6 +134,7 @@ DEVELOPER_PARAMETERS: dict[str, list[dict[str, str | list[str]]]] = {
"name": "Stop flags",
"options": [
"--preserve-volumes",
+ "--cleanup-mypy-cache",
],
},
],
diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt
index 4078b82244..b721a519c5 100644
--- a/images/breeze/output-commands-hash.txt
+++ b/images/breeze/output-commands-hash.txt
@@ -56,7 +56,7 @@ setup:f383b9236f6141f95276136ccd9217f5
shell:affbf6f7f469408d0af47f75c6a38f6c
start-airflow:109728919a0dd5c5ff5640ae86ba9e90
static-checks:7a39e28c87fbca0a9fae0ebfe1591b71
-stop:8969537ccdd799f692ccb8600a7bbed6
+stop:e5aa686b4e53707ced4039d8414d5cd6
testing:docker-compose-tests:b86c044b24138af0659a05ed6331576c
testing:helm-tests:94a442e7f3f63b34c4831a84d165690a
testing:integration-tests:585da1e636f710be9c9de36a71586963
diff --git a/images/breeze/output_stop.svg b/images/breeze/output_stop.svg
index 3ed01097cf..81e2abb315 100644
--- a/images/breeze/output_stop.svg
+++ b/images/breeze/output_stop.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 367.2" xmlns="http://www.w3.org/2000/svg">
+<svg class="rich-terminal" viewBox="0 0 1482 391.59999999999997" xmlns="http://www.w3.org/2000/svg">
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -42,7 +42,7 @@
<defs>
<clipPath id="breeze-stop-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="316.2" />
+ <rect x="0" y="0" width="1463.0" height="340.59999999999997" />
</clipPath>
<clipPath id="breeze-stop-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
@@ -80,9 +80,12 @@
<clipPath id="breeze-stop-line-11">
<rect x="0" y="269.9" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="breeze-stop-line-12">
+ <rect x="0" y="294.3" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="365.2" rx="8"/><text class="breeze-stop-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: stop</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="389.6" rx="8"/><text class="breeze-stop-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: stop</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
@@ -98,13 +101,14 @@
</text><text class="breeze-stop-r2" x="12.2" y="93.2" textLength="390.4" clip-path="url(#breeze-stop-line-3)">Stop running breeze environment.</text><text class="breeze-stop-r2" x="1464" y="93.2" textLength="12.2" clip-path="url(#breeze-stop-line-3)">
</text><text class="breeze-stop-r2" x="1464" y="117.6" textLength="12.2" clip-path="url(#breeze-stop-line-4)">
</text><text class="breeze-stop-r4" x="0" y="142" textLength="24.4" clip-path="url(#breeze-stop-line-5)">╭─</text><text class="breeze-stop-r4" x="24.4" y="142" textLength="146.4" clip-path="url(#breeze-stop-line-5)"> Stop flags </text><text class="breeze-stop-r4" x="170.8" y="142" textLength="1268.8" clip-path="url(#breeze-stop-line-5)">────────────────────────────────────────────────────────────────────────────────────────────────────────</text><text class="breeze-stop-r4 [...]
-</text><text class="breeze-stop-r4" x="0" y="166.4" textLength="12.2" clip-path="url(#breeze-stop-line-6)">│</text><text class="breeze-stop-r5" x="24.4" y="166.4" textLength="12.2" clip-path="url(#breeze-stop-line-6)">-</text><text class="breeze-stop-r5" x="36.6" y="166.4" textLength="109.8" clip-path="url(#breeze-stop-line-6)">-preserve</text><text class="breeze-stop-r5" x="146.4" y="166.4" textLength="97.6" clip-path="url(#breeze-stop-line-6)">-volumes</text><text class="breeze-stop-r6 [...]
-</text><text class="breeze-stop-r4" x="0" y="190.8" textLength="1464" clip-path="url(#breeze-stop-line-7)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-stop-r2" x="1464" y="190.8" textLength="12.2" clip-path="url(#breeze-stop-line-7)">
-</text><text class="breeze-stop-r4" x="0" y="215.2" textLength="24.4" clip-path="url(#breeze-stop-line-8)">╭─</text><text class="breeze-stop-r4" x="24.4" y="215.2" textLength="195.2" clip-path="url(#breeze-stop-line-8)"> Common options </text><text class="breeze-stop-r4" x="219.6" y="215.2" textLength="1220" clip-path="url(#breeze-stop-line-8)">────────────────────────────────────────────────────────────────────────────────────────────────────</text><text class="breeze-sto [...]
-</text><text class="breeze-stop-r4" x="0" y="239.6" textLength="12.2" clip-path="url(#breeze-stop-line-9)">│</text><text class="breeze-stop-r5" x="24.4" y="239.6" textLength="12.2" clip-path="url(#breeze-stop-line-9)">-</text><text class="breeze-stop-r5" x="36.6" y="239.6" textLength="97.6" clip-path="url(#breeze-stop-line-9)">-verbose</text><text class="breeze-stop-r6" x="158.6" y="239.6" textLength="24.4" clip-path="url(#breeze-stop-line-9)">-v</text><text class="breeze-stop-r2" x="207 [...]
-</text><text class="breeze-stop-r4" x="0" y="264" textLength="12.2" clip-path="url(#breeze-stop-line-10)">│</text><text class="breeze-stop-r5" x="24.4" y="264" textLength="12.2" clip-path="url(#breeze-stop-line-10)">-</text><text class="breeze-stop-r5" x="36.6" y="264" textLength="48.8" clip-path="url(#breeze-stop-line-10)">-dry</text><text class="breeze-stop-r5" x="85.4" y="264" textLength="48.8" clip-path="url(#breeze-stop-line-10)">-run</text><text class="breeze-stop-r6" x="158.6" y=" [...]
-</text><text class="breeze-stop-r4" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-stop-line-11)">│</text><text class="breeze-stop-r5" x="24.4" y="288.4" textLength="12.2" clip-path="url(#breeze-stop-line-11)">-</text><text class="breeze-stop-r5" x="36.6" y="288.4" textLength="61" clip-path="url(#breeze-stop-line-11)">-help</text><text class="breeze-stop-r6" x="158.6" y="288.4" textLength="24.4" clip-path="url(#breeze-stop-line-11)">-h</text><text class="breeze-stop-r2" x="207. [...]
-</text><text class="breeze-stop-r4" x="0" y="312.8" textLength="1464" clip-path="url(#breeze-stop-line-12)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-stop-r2" x="1464" y="312.8" textLength="12.2" clip-path="url(#breeze-stop-line-12)">
+</text><text class="breeze-stop-r4" x="0" y="166.4" textLength="12.2" clip-path="url(#breeze-stop-line-6)">│</text><text class="breeze-stop-r5" x="24.4" y="166.4" textLength="12.2" clip-path="url(#breeze-stop-line-6)">-</text><text class="breeze-stop-r5" x="36.6" y="166.4" textLength="109.8" clip-path="url(#breeze-stop-line-6)">-preserve</text><text class="breeze-stop-r5" x="146.4" y="166.4" textLength="97.6" clip-path="url(#breeze-stop-line-6)">-volumes</text><text class="breeze-stop-r6 [...]
+</text><text class="breeze-stop-r4" x="0" y="190.8" textLength="12.2" clip-path="url(#breeze-stop-line-7)">│</text><text class="breeze-stop-r5" x="24.4" y="190.8" textLength="12.2" clip-path="url(#breeze-stop-line-7)">-</text><text class="breeze-stop-r5" x="36.6" y="190.8" textLength="97.6" clip-path="url(#breeze-stop-line-7)">-cleanup</text><text class="breeze-stop-r5" x="134.2" y="190.8" textLength="134.2" clip-path="url(#breeze-stop-line-7)">-mypy-cache</text><text class="breeze-stop- [...]
+</text><text class="breeze-stop-r4" x="0" y="215.2" textLength="1464" clip-path="url(#breeze-stop-line-8)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-stop-r2" x="1464" y="215.2" textLength="12.2" clip-path="url(#breeze-stop-line-8)">
+</text><text class="breeze-stop-r4" x="0" y="239.6" textLength="24.4" clip-path="url(#breeze-stop-line-9)">╭─</text><text class="breeze-stop-r4" x="24.4" y="239.6" textLength="195.2" clip-path="url(#breeze-stop-line-9)"> Common options </text><text class="breeze-stop-r4" x="219.6" y="239.6" textLength="1220" clip-path="url(#breeze-stop-line-9)">────────────────────────────────────────────────────────────────────────────────────────────────────</text><text class="breeze-sto [...]
+</text><text class="breeze-stop-r4" x="0" y="264" textLength="12.2" clip-path="url(#breeze-stop-line-10)">│</text><text class="breeze-stop-r5" x="24.4" y="264" textLength="12.2" clip-path="url(#breeze-stop-line-10)">-</text><text class="breeze-stop-r5" x="36.6" y="264" textLength="97.6" clip-path="url(#breeze-stop-line-10)">-verbose</text><text class="breeze-stop-r6" x="158.6" y="264" textLength="24.4" clip-path="url(#breeze-stop-line-10)">-v</text><text class="breeze-stop-r2" x="207.4" [...]
+</text><text class="breeze-stop-r4" x="0" y="288.4" textLength="12.2" clip-path="url(#breeze-stop-line-11)">│</text><text class="breeze-stop-r5" x="24.4" y="288.4" textLength="12.2" clip-path="url(#breeze-stop-line-11)">-</text><text class="breeze-stop-r5" x="36.6" y="288.4" textLength="48.8" clip-path="url(#breeze-stop-line-11)">-dry</text><text class="breeze-stop-r5" x="85.4" y="288.4" textLength="48.8" clip-path="url(#breeze-stop-line-11)">-run</text><text class="breeze-stop-r6" x="15 [...]
+</text><text class="breeze-stop-r4" x="0" y="312.8" textLength="12.2" clip-path="url(#breeze-stop-line-12)">│</text><text class="breeze-stop-r5" x="24.4" y="312.8" textLength="12.2" clip-path="url(#breeze-stop-line-12)">-</text><text class="breeze-stop-r5" x="36.6" y="312.8" textLength="61" clip-path="url(#breeze-stop-line-12)">-help</text><text class="breeze-stop-r6" x="158.6" y="312.8" textLength="24.4" clip-path="url(#breeze-stop-line-12)">-h</text><text class="breeze-stop-r2" x="207. [...]
+</text><text class="breeze-stop-r4" x="0" y="337.2" textLength="1464" clip-path="url(#breeze-stop-line-13)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="breeze-stop-r2" x="1464" y="337.2" textLength="12.2" clip-path="url(#breeze-stop-line-13)">
</text>
</g>
</g>
diff --git a/scripts/ci/pre_commit/pre_commit_mypy.py b/scripts/ci/pre_commit/pre_commit_mypy.py
index 2b99ab68dc..f6a08e8815 100755
--- a/scripts/ci/pre_commit/pre_commit_mypy.py
+++ b/scripts/ci/pre_commit/pre_commit_mypy.py
@@ -70,6 +70,7 @@ if __name__ == "__main__":
if cmd_result.returncode != 0:
get_console().print(
"[warning]If you see strange stacktraces above, "
- "run `breeze ci-image build --python 3.7` and try again."
+ "run `breeze ci-image build --python 3.7` and try again. "
+ "You can also run `breeze stop --cleanup-mypy-cache` to clean up the cache used."
)
sys.exit(cmd_result.returncode)