You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/07/07 23:36:29 UTC
[airflow] branch main updated: Run "fix-ownership" with sudo rather than docker image if specified (#24871)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new bcf2c418d2 Run "fix-ownership" with sudo rather than docker image if specified (#24871)
bcf2c418d2 is described below
commit bcf2c418d261c6244e60e4c2d5de42b23b714bd1
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Fri Jul 8 01:36:20 2022 +0200
Run "fix-ownership" with sudo rather than docker image if specified (#24871)
The "fix-ownership" command uses our breeze docker image to fix
ownership of files generated by breeze. This is nice from the user
perspective because there is no need to authenticate with password
and - since we already have breeze image handy - it is fast and
painless. However in CI, there are often cases where "fix-ownership"
does not really have an image available and needs to pull one, which
takes 1 minute. However the user in CI has sudoer capability without
password, so we can use it to run fix-ownership with just plain
sudo/chown.
We switch all the CI jobs to "USE_SUDO" and also avoid running
fix-ownership on other systems that Linux, as the ownership of
files creaed from docker as root is only problem when the filesystem
is directly mounted and used in the container (which happens only on
Linux). MacOS and Windows have "user-space" filesystem that are
much slower, but then they perform user-remapping on their own
and files created in container have automatically ownership of
the user who runs the docker container command.
---
.github/workflows/build-images.yml | 1 +
.github/workflows/ci.yml | 1 +
.../src/airflow_breeze/commands/ci_commands.py | 72 ++++++++++++++++++-
images/breeze/output-commands-hash.txt | 2 +-
images/breeze/output-fix-ownership.svg | 84 ++++++++++++----------
5 files changed, 122 insertions(+), 38 deletions(-)
diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml
index 41cc29ea1d..11413ac95b 100644
--- a/.github/workflows/build-images.yml
+++ b/.github/workflows/build-images.yml
@@ -40,6 +40,7 @@ env:
# This token is WRITE one - pull_request_target type of events always have the WRITE token
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
IMAGE_TAG_FOR_THE_BUILD: "${{ github.event.pull_request.head.sha || github.sha }}"
+ USE_SUDO: "true"
concurrency:
group: build-${{ github.event.pull_request.number || github.ref }}
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 906cd5f1d9..83830c2154 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -47,6 +47,7 @@ env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ENABLE_TEST_COVERAGE: "${{ github.event_name == 'push' }}"
IMAGE_TAG_FOR_THE_BUILD: "${{ github.event.pull_request.head.sha || github.sha }}"
+ USE_SUDO: "true"
concurrency:
group: ci-${{ github.event.pull_request.number || github.ref }}
diff --git a/dev/breeze/src/airflow_breeze/commands/ci_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_commands.py
index c65753e1a6..fa399fbed9 100644
--- a/dev/breeze/src/airflow_breeze/commands/ci_commands.py
+++ b/dev/breeze/src/airflow_breeze/commands/ci_commands.py
@@ -14,7 +14,11 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+import os
+import platform
+import subprocess
import sys
+from pathlib import Path
from typing import Optional, Tuple
import click
@@ -49,6 +53,7 @@ from airflow_breeze.utils.docker_command_utils import (
)
from airflow_breeze.utils.find_newer_dependencies import find_newer_dependencies
from airflow_breeze.utils.image import find_available_ci_image
+from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT
from airflow_breeze.utils.run_utils import run_command
CI_COMMANDS = {
@@ -63,6 +68,14 @@ CI_COMMANDS = {
}
CI_PARAMETERS = {
+ "breeze fix-ownership": [
+ {
+ "name": "Fix ownership flags",
+ "options": [
+ "--use-sudo",
+ ],
+ }
+ ],
"breeze selective-check": [
{
"name": "Selective check flags",
@@ -114,11 +127,68 @@ def resource_check(verbose: bool, dry_run: bool):
check_docker_resources(shell_params.airflow_image_name, verbose=verbose, dry_run=dry_run)
+HOME_DIR = Path(os.path.expanduser('~')).resolve()
+
+DIRECTORIES_TO_FIX = [
+ AIRFLOW_SOURCES_ROOT,
+ HOME_DIR / ".aws",
+ HOME_DIR / ".azure",
+ HOME_DIR / ".config/gcloud",
+ HOME_DIR / ".docker",
+ AIRFLOW_SOURCES_ROOT,
+]
+
+
+def fix_ownership_for_file(file: Path, dry_run: bool, verbose: bool):
+ get_console().print(f"[info]Fixing ownership of {file}")
+ result = run_command(
+ ["sudo", "chown", f"{os.getuid}:{os.getgid()}", str(file.resolve())],
+ check=False,
+ stderr=subprocess.STDOUT,
+ dry_run=dry_run,
+ verbose=verbose,
+ )
+ if result.returncode != 0:
+ get_console().print(f"[warning]Could not fix ownership for {file}: {result.stdout}")
+
+
+def fix_ownership_for_path(path: Path, dry_run: bool, verbose: bool):
+ if path.is_dir():
+ for p in Path(path).rglob('*'):
+ if p.owner == 'root':
+ fix_ownership_for_file(p, dry_run=dry_run, verbose=verbose)
+ else:
+ if path.owner == 'root':
+ fix_ownership_for_file(path, dry_run=dry_run, verbose=verbose)
+
+
+def fix_ownership_without_docker(dry_run: bool, verbose: bool):
+ for directory_to_fix in DIRECTORIES_TO_FIX:
+ fix_ownership_for_path(directory_to_fix, dry_run=dry_run, verbose=verbose)
+
+
@main.command(name="fix-ownership", help="Fix ownership of source files to be same as host user.")
+@click.option(
+ '--use-sudo',
+ is_flag=True,
+ help="Use sudo instead of docker image to fix the ownership. You need to be a `sudoer` to run it",
+ envvar='USE_SUDO',
+)
@option_github_repository
@option_verbose
@option_dry_run
-def fix_ownership(github_repository: str, verbose: bool, dry_run: bool):
+def fix_ownership(github_repository: str, use_sudo: bool, verbose: bool, dry_run: bool):
+ system = platform.system().lower()
+ if system != 'linux':
+ get_console().print(
+ f"[warning]You should only need to run fix-ownership on Linux and your system is {system}"
+ )
+ sys.exit(0)
+ if use_sudo:
+ get_console().print("[info]Fixing ownership using sudo.")
+ fix_ownership_without_docker(dry_run=dry_run, verbose=verbose)
+ sys.exit(0)
+ get_console().print("[info]Fixing ownership using docker.")
perform_environment_checks(verbose=verbose)
shell_params = find_available_ci_image(github_repository, dry_run, verbose)
extra_docker_flags = get_extra_docker_flags(MOUNT_ALL)
diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt
index 2dd07945e5..4dac256558 100644
--- a/images/breeze/output-commands-hash.txt
+++ b/images/breeze/output-commands-hash.txt
@@ -12,7 +12,7 @@ config:92653afc11889e1b78e3a2e38f41107f
docker-compose-tests:8ae3b6211fd31db81a750d1c6b96ec3d
exec:e4329909b8b2a610fa4fad5116c4b896
find-newer-dependencies:00000f7afb289e36e8c573fcc654df44
-fix-ownership:596143cc74217f0a90850a554220ea45
+fix-ownership:84902165a54467564fbdd3598fa273e2
free-space:bb8e7ac63d12ab3ede272a898de2f527
generate-constraints:a5120e79439f30eb7fbee929dca23156
prepare-airflow-package:cff9d88ca313db10f3cc464c6798f6be
diff --git a/images/breeze/output-fix-ownership.svg b/images/breeze/output-fix-ownership.svg
index d8fe2ea84f..f7208613b1 100644
--- a/images/breeze/output-fix-ownership.svg
+++ b/images/breeze/output-fix-ownership.svg
@@ -1,4 +1,4 @@
-<svg class="rich-terminal" viewBox="0 0 1482 318.4" xmlns="http://www.w3.org/2000/svg">
+<svg class="rich-terminal" viewBox="0 0 1482 391.59999999999997" xmlns="http://www.w3.org/2000/svg">
<!-- Generated with Rich https://www.textualize.io -->
<style>
@@ -19,85 +19,97 @@
font-weight: 700;
}
- .terminal-2814491796-matrix {
+ .terminal-1020697717-matrix {
font-family: Fira Code, monospace;
font-size: 20px;
line-height: 24.4px;
font-variant-east-asian: full-width;
}
- .terminal-2814491796-title {
+ .terminal-1020697717-title {
font-size: 18px;
font-weight: bold;
font-family: arial;
}
- .terminal-2814491796-r1 { fill: #c5c8c6;font-weight: bold }
-.terminal-2814491796-r2 { fill: #c5c8c6 }
-.terminal-2814491796-r3 { fill: #d0b344;font-weight: bold }
-.terminal-2814491796-r4 { fill: #868887 }
-.terminal-2814491796-r5 { fill: #68a0b3;font-weight: bold }
-.terminal-2814491796-r6 { fill: #98a84b;font-weight: bold }
-.terminal-2814491796-r7 { fill: #8d7b39 }
+ .terminal-1020697717-r1 { fill: #c5c8c6;font-weight: bold }
+.terminal-1020697717-r2 { fill: #c5c8c6 }
+.terminal-1020697717-r3 { fill: #d0b344;font-weight: bold }
+.terminal-1020697717-r4 { fill: #868887 }
+.terminal-1020697717-r5 { fill: #68a0b3;font-weight: bold }
+.terminal-1020697717-r6 { fill: #98a84b;font-weight: bold }
+.terminal-1020697717-r7 { fill: #8d7b39 }
</style>
<defs>
- <clipPath id="terminal-2814491796-clip-terminal">
- <rect x="0" y="0" width="1463.0" height="267.4" />
+ <clipPath id="terminal-1020697717-clip-terminal">
+ <rect x="0" y="0" width="1463.0" height="340.59999999999997" />
</clipPath>
- <clipPath id="terminal-2814491796-line-0">
+ <clipPath id="terminal-1020697717-line-0">
<rect x="0" y="1.5" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2814491796-line-1">
+<clipPath id="terminal-1020697717-line-1">
<rect x="0" y="25.9" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2814491796-line-2">
+<clipPath id="terminal-1020697717-line-2">
<rect x="0" y="50.3" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2814491796-line-3">
+<clipPath id="terminal-1020697717-line-3">
<rect x="0" y="74.7" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2814491796-line-4">
+<clipPath id="terminal-1020697717-line-4">
<rect x="0" y="99.1" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2814491796-line-5">
+<clipPath id="terminal-1020697717-line-5">
<rect x="0" y="123.5" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2814491796-line-6">
+<clipPath id="terminal-1020697717-line-6">
<rect x="0" y="147.9" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2814491796-line-7">
+<clipPath id="terminal-1020697717-line-7">
<rect x="0" y="172.3" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2814491796-line-8">
+<clipPath id="terminal-1020697717-line-8">
<rect x="0" y="196.7" width="1464" height="24.65"/>
</clipPath>
-<clipPath id="terminal-2814491796-line-9">
+<clipPath id="terminal-1020697717-line-9">
<rect x="0" y="221.1" width="1464" height="24.65"/>
</clipPath>
+<clipPath id="terminal-1020697717-line-10">
+ <rect x="0" y="245.5" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="terminal-1020697717-line-11">
+ <rect x="0" y="269.9" width="1464" height="24.65"/>
+ </clipPath>
+<clipPath id="terminal-1020697717-line-12">
+ <rect x="0" y="294.3" width="1464" height="24.65"/>
+ </clipPath>
</defs>
- <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="316.4" rx="8"/><text class="terminal-2814491796-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: fix-ownership</text>
+ <rect fill="#292929" stroke="rgba(255,255,255,0.35)" stroke-width="1" x="1" y="1" width="1480" height="389.6" rx="8"/><text class="terminal-1020697717-title" fill="#c5c8c6" text-anchor="middle" x="740" y="27">Command: fix-ownership</text>
<g transform="translate(26,22)">
<circle cx="0" cy="0" r="7" fill="#ff5f57"/>
<circle cx="22" cy="0" r="7" fill="#febc2e"/>
<circle cx="44" cy="0" r="7" fill="#28c840"/>
</g>
- <g transform="translate(9, 41)" clip-path="url(#terminal-2814491796-clip-terminal)">
+ <g transform="translate(9, 41)" clip-path="url(#terminal-1020697717-clip-terminal)">
- <g class="terminal-2814491796-matrix">
- <text class="terminal-2814491796-r2" x="1464" y="20" textLength="12.2" clip-path="url(#terminal-2814491796-line-0)">
-</text><text class="terminal-2814491796-r3" x="12.2" y="44.4" textLength="85.4" clip-path="url(#terminal-2814491796-line-1)">Usage: </text><text class="terminal-2814491796-r1" x="97.6" y="44.4" textLength="366" clip-path="url(#terminal-2814491796-line-1)">breeze fix-ownership [OPTIONS]</text><text class="terminal-2814491796-r2" x="1464" y="44.4" textLength="12.2" clip-path="url(#terminal-2814491796-line-1)">
-</text><text class="terminal-2814491796-r2" x="1464" y="68.8" textLength="12.2" clip-path="url(#terminal-2814491796-line-2)">
-</text><text class="terminal-2814491796-r2" x="12.2" y="93.2" textLength="658.8" clip-path="url(#terminal-2814491796-line-3)">Fix ownership of source files to be same as host user.</text><text class="terminal-2814491796-r2" x="1464" y="93.2" textLength="12.2" clip-path="url(#terminal-2814491796-line-3)">
-</text><text class="terminal-2814491796-r2" x="1464" y="117.6" textLength="12.2" clip-path="url(#terminal-2814491796-line-4)">
-</text><text class="terminal-2814491796-r4" x="0" y="142" textLength="24.4" clip-path="url(#terminal-2814491796-line-5)">╭─</text><text class="terminal-2814491796-r4" x="24.4" y="142" textLength="1415.2" clip-path="url(#terminal-2814491796-line-5)"> Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────</text><text class="terminal-2814491796-r4" x="1439.6" y="142" textLength="24.4" clip-path="url(#terminal-2814491796- [...]
-</text><text class="terminal-2814491796-r4" x="0" y="166.4" textLength="12.2" clip-path="url(#terminal-2814491796-line-6)">│</text><text class="terminal-2814491796-r5" x="24.4" y="166.4" textLength="12.2" clip-path="url(#terminal-2814491796-line-6)">-</text><text class="terminal-2814491796-r5" x="36.6" y="166.4" textLength="85.4" clip-path="url(#terminal-2814491796-line-6)">-github</text><text class="terminal-2814491796-r5" x="122" y="166.4" textLength="134.2" clip-path="url(#terminal-28 [...]
-</text><text class="terminal-2814491796-r4" x="0" y="190.8" textLength="12.2" clip-path="url(#terminal-2814491796-line-7)">│</text><text class="terminal-2814491796-r5" x="24.4" y="190.8" textLength="12.2" clip-path="url(#terminal-2814491796-line-7)">-</text><text class="terminal-2814491796-r5" x="36.6" y="190.8" textLength="97.6" clip-path="url(#terminal-2814491796-line-7)">-verbose</text><text class="terminal-2814491796-r6" x="280.6" y="190.8" textLength="24.4" clip-path="url(#terminal- [...]
-</text><text class="terminal-2814491796-r4" x="0" y="215.2" textLength="12.2" clip-path="url(#terminal-2814491796-line-8)">│</text><text class="terminal-2814491796-r5" x="24.4" y="215.2" textLength="12.2" clip-path="url(#terminal-2814491796-line-8)">-</text><text class="terminal-2814491796-r5" x="36.6" y="215.2" textLength="48.8" clip-path="url(#terminal-2814491796-line-8)">-dry</text><text class="terminal-2814491796-r5" x="85.4" y="215.2" textLength="48.8" clip-path="url(#terminal-28144 [...]
-</text><text class="terminal-2814491796-r4" x="0" y="239.6" textLength="12.2" clip-path="url(#terminal-2814491796-line-9)">│</text><text class="terminal-2814491796-r5" x="24.4" y="239.6" textLength="12.2" clip-path="url(#terminal-2814491796-line-9)">-</text><text class="terminal-2814491796-r5" x="36.6" y="239.6" textLength="61" clip-path="url(#terminal-2814491796-line-9)">-help</text><text class="terminal-2814491796-r6" x="280.6" y="239.6" textLength="24.4" clip-path="url(#terminal-28144 [...]
-</text><text class="terminal-2814491796-r4" x="0" y="264" textLength="1464" clip-path="url(#terminal-2814491796-line-10)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="terminal-2814491796-r2" x="1464" y="264" textLength="12.2" clip-path="url(#terminal-2814491796-line-10)">
+ <g class="terminal-1020697717-matrix">
+ <text class="terminal-1020697717-r2" x="1464" y="20" textLength="12.2" clip-path="url(#terminal-1020697717-line-0)">
+</text><text class="terminal-1020697717-r3" x="12.2" y="44.4" textLength="85.4" clip-path="url(#terminal-1020697717-line-1)">Usage: </text><text class="terminal-1020697717-r1" x="97.6" y="44.4" textLength="366" clip-path="url(#terminal-1020697717-line-1)">breeze fix-ownership [OPTIONS]</text><text class="terminal-1020697717-r2" x="1464" y="44.4" textLength="12.2" clip-path="url(#terminal-1020697717-line-1)">
+</text><text class="terminal-1020697717-r2" x="1464" y="68.8" textLength="12.2" clip-path="url(#terminal-1020697717-line-2)">
+</text><text class="terminal-1020697717-r2" x="12.2" y="93.2" textLength="658.8" clip-path="url(#terminal-1020697717-line-3)">Fix ownership of source files to be same as host user.</text><text class="terminal-1020697717-r2" x="1464" y="93.2" textLength="12.2" clip-path="url(#terminal-1020697717-line-3)">
+</text><text class="terminal-1020697717-r2" x="1464" y="117.6" textLength="12.2" clip-path="url(#terminal-1020697717-line-4)">
+</text><text class="terminal-1020697717-r4" x="0" y="142" textLength="24.4" clip-path="url(#terminal-1020697717-line-5)">╭─</text><text class="terminal-1020697717-r4" x="24.4" y="142" textLength="1415.2" clip-path="url(#terminal-1020697717-line-5)"> Fix ownership flags ───────────────────────────────────────────────────────────────────────────────────────────────</text><text class="terminal-1020697717-r4" x="1439.6" y="142" textLength="24.4" clip-path="url(#terminal-1 [...]
+</text><text class="terminal-1020697717-r4" x="0" y="166.4" textLength="12.2" clip-path="url(#terminal-1020697717-line-6)">│</text><text class="terminal-1020697717-r5" x="24.4" y="166.4" textLength="12.2" clip-path="url(#terminal-1020697717-line-6)">-</text><text class="terminal-1020697717-r5" x="36.6" y="166.4" textLength="48.8" clip-path="url(#terminal-1020697717-line-6)">-use</text><text class="terminal-1020697717-r5" x="85.4" y="166.4" textLength="61" clip-path="url(#terminal-1020697 [...]
+</text><text class="terminal-1020697717-r4" x="0" y="190.8" textLength="1464" clip-path="url(#terminal-1020697717-line-7)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="terminal-1020697717-r2" x="1464" y="190.8" textLength="12.2" clip-path="url(#terminal-1020697717-line-7)">
+</text><text class="terminal-1020697717-r4" x="0" y="215.2" textLength="24.4" clip-path="url(#terminal-1020697717-line-8)">╭─</text><text class="terminal-1020697717-r4" x="24.4" y="215.2" textLength="1415.2" clip-path="url(#terminal-1020697717-line-8)"> Options ───────────────────────────────────────────────────────────────────────────────────────────────────────────</text><text class="terminal-1020697717-r4" x="1439.6" y="215.2" textLength="24.4" clip-path="url(#terminal-10206 [...]
+</text><text class="terminal-1020697717-r4" x="0" y="239.6" textLength="12.2" clip-path="url(#terminal-1020697717-line-9)">│</text><text class="terminal-1020697717-r5" x="24.4" y="239.6" textLength="12.2" clip-path="url(#terminal-1020697717-line-9)">-</text><text class="terminal-1020697717-r5" x="36.6" y="239.6" textLength="85.4" clip-path="url(#terminal-1020697717-line-9)">-github</text><text class="terminal-1020697717-r5" x="122" y="239.6" textLength="134.2" clip-path="url(#terminal-10 [...]
+</text><text class="terminal-1020697717-r4" x="0" y="264" textLength="12.2" clip-path="url(#terminal-1020697717-line-10)">│</text><text class="terminal-1020697717-r5" x="24.4" y="264" textLength="12.2" clip-path="url(#terminal-1020697717-line-10)">-</text><text class="terminal-1020697717-r5" x="36.6" y="264" textLength="97.6" clip-path="url(#terminal-1020697717-line-10)">-verbose</text><text class="terminal-1020697717-r6" x="280.6" y="264" textLength="24.4" clip-path="url(#terminal-10206 [...]
+</text><text class="terminal-1020697717-r4" x="0" y="288.4" textLength="12.2" clip-path="url(#terminal-1020697717-line-11)">│</text><text class="terminal-1020697717-r5" x="24.4" y="288.4" textLength="12.2" clip-path="url(#terminal-1020697717-line-11)">-</text><text class="terminal-1020697717-r5" x="36.6" y="288.4" textLength="48.8" clip-path="url(#terminal-1020697717-line-11)">-dry</text><text class="terminal-1020697717-r5" x="85.4" y="288.4" textLength="48.8" clip-path="url(#terminal-10 [...]
+</text><text class="terminal-1020697717-r4" x="0" y="312.8" textLength="12.2" clip-path="url(#terminal-1020697717-line-12)">│</text><text class="terminal-1020697717-r5" x="24.4" y="312.8" textLength="12.2" clip-path="url(#terminal-1020697717-line-12)">-</text><text class="terminal-1020697717-r5" x="36.6" y="312.8" textLength="61" clip-path="url(#terminal-1020697717-line-12)">-help</text><text class="terminal-1020697717-r6" x="280.6" y="312.8" textLength="24.4" clip-path="url(#terminal-10 [...]
+</text><text class="terminal-1020697717-r4" x="0" y="337.2" textLength="1464" clip-path="url(#terminal-1020697717-line-13)">╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯</text><text class="terminal-1020697717-r2" x="1464" y="337.2" textLength="12.2" clip-path="url(#terminal-1020697717-line-13)">
</text>
</g>
</g>