You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/04/30 23:54:09 UTC
[airflow] 04/09: resolving conflict (#23052)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-3-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 93b485f304ec0ac4465fd317da1f36ea4cbac5cc
Author: Bowrna <ma...@gmail.com>
AuthorDate: Fri Apr 29 01:01:49 2022 +0530
resolving conflict (#23052)
(cherry picked from commit 94c3203e86252ed120d624a70aed571b57083ea4)
---
BREEZE.rst | 23 +++-
.../airflow_breeze/commands/developer_commands.py | 35 +++++-
dev/breeze/src/airflow_breeze/shell/enter_shell.py | 42 ++++++-
images/breeze/output-commands.svg | 3 +-
images/breeze/output-exec.svg | 133 +++++++++++++++++++++
5 files changed, 227 insertions(+), 9 deletions(-)
diff --git a/BREEZE.rst b/BREEZE.rst
index 37b47f29cf..c9f44c90c5 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -472,7 +472,14 @@ Those are commands mostly used by contributors:
* Initialize local virtualenv with ``./scripts/tools/initialize_virtualenv.py`` command
* Run static checks with autocomplete support ``breeze static-checks`` command
* Run test specified with ``./breeze-legacy tests`` command
-* Join running interactive shell with ``./breeze-legacy exec`` command
+* Build CI docker image with ``breeze build-image`` command
+* Cleanup breeze with ``breeze cleanup`` command
+* Run static checks with autocomplete support ``breeze static-checks`` command
+* Run test specified with ``./breeze-legacy tests`` command
+
+Additional management tasks:
+
+* Join running interactive shell with ``breeze exec`` command
* Stop running interactive environment with ``breeze stop`` command
* Execute arbitrary docker-compose command with ``./breeze-legacy docker-compose`` command
@@ -580,8 +587,7 @@ capability of creating multiple virtual terminals and multiplex between them. Mo
found at `tmux GitHub wiki page <https://github.com/tmux/tmux/wiki>`_ . Tmux has several useful shortcuts
that allow you to split the terminals, open new tabs etc - it's pretty useful to learn it.
-Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy``
-command but it is very similar to current ``breeze`` command):
+Here is the part of Breeze video which is relevant:
.. raw:: html
@@ -595,12 +601,11 @@ command but it is very similar to current ``breeze`` command):
Another way is to exec into Breeze terminal from the host's terminal. Often you can
have multiple terminals in the host (Linux/MacOS/WSL2 on Windows) and you can simply use those terminals
-to enter the running container. It's as easy as launching ``./breeze-legacy exec`` while you already started the
+to enter the running container. It's as easy as launching ``breeze exec`` while you already started the
Breeze environment. You will be dropped into bash and environment variables will be read in the same
way as when you enter the environment. You can do it multiple times and open as many terminals as you need.
-Here is the part of Breeze video which is relevant (note that it refers to the old ``./breeze-legacy``
-command and it is not yet available in the current ``breeze`` command):
+Here is the part of Breeze video which is relevant:
.. raw:: html
@@ -612,6 +617,12 @@ command and it is not yet available in the current ``breeze`` command):
</div>
+Those are all available flags of ``exec`` command:
+
+.. image:: ./images/breeze/output-exec.svg
+ :width: 100%
+ :alt: Breeze exec
+
Additional tools
----------------
diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
index e44c360b6c..f2c8715f95 100644
--- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py
@@ -50,7 +50,7 @@ from airflow_breeze.global_constants import (
get_available_packages,
)
from airflow_breeze.pre_commit_ids import PRE_COMMIT_LIST
-from airflow_breeze.shell.enter_shell import enter_shell
+from airflow_breeze.shell.enter_shell import enter_shell, find_airflow_container
from airflow_breeze.shell.shell_params import ShellParams
from airflow_breeze.utils.confirm import set_forced_answer
from airflow_breeze.utils.console import get_console
@@ -66,6 +66,7 @@ DEVELOPER_COMMANDS = {
"commands": [
"shell",
"start-airflow",
+ "exec",
"stop",
"build-docs",
"static-checks",
@@ -144,6 +145,9 @@ DEVELOPER_PARAMETERS = {
],
},
],
+ "breeze exec": [
+ {"name": "Drops in the interactive shell of active airflow container"},
+ ],
"breeze stop": [
{
"name": "Stop flags",
@@ -478,3 +482,32 @@ class DocBuildParams:
for single_filter in self.package_filter:
doc_args.extend(["--package-filter", single_filter])
return doc_args
+
+
+@main.command(name='exec', help='Joins the interactive shell of running airflow container')
+@option_verbose
+@option_dry_run
+@click.argument('exec_args', nargs=-1, type=click.UNPROCESSED)
+def exec(verbose: bool, dry_run: bool, exec_args: Tuple):
+ container_running = find_airflow_container(verbose, dry_run)
+ if container_running:
+ cmd_to_run = [
+ "docker",
+ "exec",
+ "-it",
+ container_running,
+ "/opt/airflow/scripts/docker/entrypoint_exec.sh",
+ ]
+ if exec_args:
+ cmd_to_run.extend(exec_args)
+ process = run_command(
+ cmd_to_run,
+ verbose=verbose,
+ dry_run=dry_run,
+ check=False,
+ no_output_dump_on_exception=False,
+ text=True,
+ )
+ if not process:
+ sys.exit(1)
+ sys.exit(process.returncode)
diff --git a/dev/breeze/src/airflow_breeze/shell/enter_shell.py b/dev/breeze/src/airflow_breeze/shell/enter_shell.py
index c9d93abb3d..511d306516 100644
--- a/dev/breeze/src/airflow_breeze/shell/enter_shell.py
+++ b/dev/breeze/src/airflow_breeze/shell/enter_shell.py
@@ -18,7 +18,7 @@
import subprocess
import sys
from pathlib import Path
-from typing import Dict, Union
+from typing import Dict, Optional, Union
from airflow_breeze import global_constants
from airflow_breeze.build_image.ci.build_ci_image import build_ci_image
@@ -137,3 +137,43 @@ def run_shell_with_build_image_checks(
if cmd_added is not None:
cmd.extend(['-c', cmd_added])
return run_command(cmd, verbose=verbose, dry_run=dry_run, env=env_variables, text=True)
+
+
+def stop_exec_on_error(returncode: int):
+ get_console().print('\n[error]ERROR in finding the airflow docker-compose process id[/]\n')
+ sys.exit(returncode)
+
+
+def find_airflow_container(verbose, dry_run) -> Optional[str]:
+ exec_shell_params = ShellParams(verbose=verbose, dry_run=dry_run)
+ check_docker_resources(verbose, exec_shell_params.airflow_image_name, dry_run)
+ exec_shell_params.print_badge_info()
+ env_variables = construct_env_variables_docker_compose_command(exec_shell_params)
+ cmd = ['docker-compose', 'ps', '--all', '--filter', 'status=running', 'airflow']
+ docker_compose_ps_command = run_command(
+ cmd,
+ verbose=verbose,
+ dry_run=dry_run,
+ text=True,
+ capture_output=True,
+ env=env_variables,
+ # print output if run in verbose mode to better diagnose problems.
+ no_output_dump_on_exception=not verbose,
+ )
+ if dry_run:
+ return "CONTAINER_ID"
+ if docker_compose_ps_command.returncode != 0:
+ stop_exec_on_error(docker_compose_ps_command.returncode)
+ return None
+
+ output = docker_compose_ps_command.stdout
+ container_info = output.strip().split('\n')
+ if container_info:
+ container_running = container_info[-1].split(' ')[0]
+ if container_running.startswith('-'):
+ # On docker-compose v1 we get '--------' as output here
+ stop_exec_on_error(docker_compose_ps_command.returncode)
+ return container_running
+ else:
+ stop_exec_on_error(1)
+ return None
diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg
index e5cdec13fb..23ecba4348 100644
--- a/images/breeze/output-commands.svg
+++ b/images/breeze/output-commands.svg
@@ -1,4 +1,4 @@
-<svg width="1720.0" height="1814" viewBox="0 0 1720.0 1814"
+<svg width="1720.0" height="1836" viewBox="0 0 1720.0 1836"
xmlns="http://www.w3.org/2000/svg">
<style>
@font-face {
@@ -146,6 +146,7 @@
<div><span class="r4">╭─ Developer tools ────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> shell </span><span class="r1"> </span><span class="r1">Enter breeze.py environment. this is the default command use when no other is selected.</span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> start-airflow </span><span class="r1"> </span><span class="r1">Enter breeze.py environment and starts all Airflow components in the tmux session. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> exec </span><span class="r1"> </span><span class="r1">Joins the interactive shell of running airflow container </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> stop </span><span class="r1"> </span><span class="r1">Stop running breeze environment. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> build-docs </span><span class="r1"> </span><span class="r1">Build documentation in the container. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
<div><span class="r4">│</span><span class="r1"> </span><span class="r5"> static-checks </span><span class="r1"> </span><span class="r1">Run static checks. </span><span class="r1"> </span><span class="r1"> </span><span class="r1"> </span><span class="r4">│</span></div>
diff --git a/images/breeze/output-exec.svg b/images/breeze/output-exec.svg
new file mode 100644
index 0000000000..9e8900f0fb
--- /dev/null
+++ b/images/breeze/output-exec.svg
@@ -0,0 +1,133 @@
+<svg width="1720.0" height="582" viewBox="0 0 1720.0 582"
+ xmlns="http://www.w3.org/2000/svg">
+ <style>
+ @font-face {
+ font-family: "Fira Code";
+ src: local("FiraCode-Regular"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff2/FiraCode-Regular.woff2") format("woff2"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff/FiraCode-Regular.woff") format("woff");
+ font-style: normal;
+ font-weight: 400;
+ }
+ @font-face {
+ font-family: "Fira Code";
+ src: local("FiraCode-Bold"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff2/FiraCode-Bold.woff2") format("woff2"),
+ url("https://cdnjs.cloudflare.com/ajax/libs/firacode/6.2.0/woff/FiraCode-Bold.woff") format("woff");
+ font-style: bold;
+ font-weight: 700;
+ }
+ span {
+ display: inline-block;
+ white-space: pre;
+ vertical-align: top;
+ font-size: 18px;
+ font-family:'Fira Code','Cascadia Code',Monaco,Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace;
+ }
+ a {
+ text-decoration: none;
+ color: inherit;
+ }
+ .blink {
+ animation: blinker 1s infinite;
+ }
+ @keyframes blinker {
+ from { opacity: 1.0; }
+ 50% { opacity: 0.3; }
+ to { opacity: 1.0; }
+ }
+ #wrapper {
+ padding: 140px;
+ padding-top: 100px;
+ }
+ #terminal {
+ position: relative;
+ display: flex;
+ flex-direction: column;
+ align-items: center;
+ background-color: #0c0c0c;
+ border-radius: 14px;
+ outline: 1px solid #484848;
+ }
+ #terminal:after {
+ position: absolute;
+ width: 100%;
+ height: 100%;
+ content: '';
+ border-radius: 14px;
+ background: rgb(71,77,102);
+ background: linear-gradient(90deg, #804D69 0%, #4E4B89 100%);
+ transform: rotate(-4.5deg);
+ z-index: -1;
+ }
+ #terminal-header {
+ position: relative;
+ width: 100%;
+ background-color: #2e2e2e;
+ margin-bottom: 12px;
+ font-weight: bold;
+ border-radius: 14px 14px 0 0;
+ color: #f2f2f2;
+ font-size: 18px;
+ box-shadow: inset 0px -1px 0px 0px #4e4e4e,
+ inset 0px -4px 8px 0px #1a1a1a;
+ }
+ #terminal-title-tab {
+ display: inline-block;
+ margin-top: 14px;
+ margin-left: 124px;
+ font-family: sans-serif;
+ padding: 14px 28px;
+ border-radius: 6px 6px 0 0;
+ background-color: #0c0c0c;
+ box-shadow: inset 0px 1px 0px 0px #4e4e4e,
+ 0px -4px 4px 0px #1e1e1e,
+ inset 1px 0px 0px 0px #4e4e4e,
+ inset -1px 0px 0px 0px #4e4e4e;
+ }
+ #terminal-traffic-lights {
+ position: absolute;
+ top: 24px;
+ left: 20px;
+ }
+ #terminal-body {
+ line-height: 22px;
+ padding: 14px;
+ }
+ .r1 {color: #f2f2f2; text-decoration-color: #f2f2f2;background-color: #0c0c0c;}
+.r2 {font-weight: bold;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.r3 {color: #e5e510; text-decoration-color: #e5e510; font-weight: bold;background-color: #0c0c0c;}
+.r4 {color: #7f7f7f; text-decoration-color: #7f7f7f;color: #f2f2f2; text-decoration-color: #f2f2f2;;background-color: #0c0c0c;}
+.r5 {color: #11a8cd; text-decoration-color: #11a8cd; font-weight: bold;background-color: #0c0c0c;}
+.r6 {color: #0dbc79; text-decoration-color: #0dbc79; font-weight: bold;background-color: #0c0c0c;}
+ </style>
+ <foreignObject x="0" y="0" width="100%" height="100%">
+ <body xmlns="http://www.w3.org/1999/xhtml">
+ <div id="wrapper">
+ <div id="terminal">
+ <div id='terminal-header'>
+ <svg id="terminal-traffic-lights" width="90" height="21" viewBox="0 0 90 21" xmlns="http://www.w3.org/2000/svg">
+ <circle cx="14" cy="8" r="8" fill="#ff6159"/>
+ <circle cx="38" cy="8" r="8" fill="#ffbd2e"/>
+ <circle cx="62" cy="8" r="8" fill="#28c941"/>
+ </svg>
+ <div id="terminal-title-tab">Command: exec</div>
+ </div>
+ <div id='terminal-body'>
+ <div><span class="r2"> </span></div>
+<div><span class="r2"> </span><span class="r3">Usage: </span><span class="r2">breeze exec [OPTIONS] [EXEC_ARGS]... </span></div>
+<div><span class="r2"> </span></div>
+<div><span class="r1"> </span><span class="r1">Joins the interactive shell of running airflow container </span><span class="r1"> </span></div>
+<div><span class="r1"> </span></div>
+<div><span class="r4">╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--verbose</span><span class="r1"> </span><span class="r6">-v</span><span class="r1"> </span><span class="r1">Print verbose information about performed steps. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--dry-run</span><span class="r1"> </span><span class="r6">-D</span><span class="r1"> </span><span class="r1">If dry-run is set, commands are only printed, not executed. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">│</span><span class="r1"> </span><span class="r5">--help</span><span class="r1"> </span><span class="r6">-h</span><span class="r1"> </span><span class="r1">Show this message and exit. </span><span class="r1"> </span><span class="r4">│</span></div>
+<div><span class="r4">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</span></div>
+<div><span class="r1"></span><span class="r1"> </span></div>
+ </div>
+ </div>
+ </div>
+ </body>
+ </foreignObject>
+</svg>