You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/04/05 21:48:02 UTC
[airflow] 03/16: Parallelize build of documentation. (#15062)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 17b89f3514e65c9671c722d40278ca8a0e8f2220
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Wed Mar 31 00:24:09 2021 +0200
Parallelize build of documentation. (#15062)
This is far more complex than it should be because of
autoapi problems with parallel execution. Unfortunately autoapi
does not cope well when several autoapis are run in parallel on
the same code - even if they are run in separate processes and
for different packages. Autoapi uses common _doctree and _api
directories generated in the source code and they override
each other if two or more of them run in parallel.
The solution in this PR is mostly applicable for CI environment.
In this case we have docker images that have been already built
using current sources so we can safely run separate docker
containers without mapping the sources and run generation
of documentation separtely and independently in each container.
This seems to work really well, speeding up docs generation
2x in public GitHub runners and 8x in self-hosted runners.
Public runners:
* 27m -> 15m
Self-hosted runners:
* 27m -> < 8m
(cherry picked from commit 741a54502f0fcb3ad57c17d18edd9a6745b4b78b)
---
.github/workflows/ci.yml | 14 +
.gitignore | 5 +
docs/build_docs.py | 417 ++++++++++++++++++---
docs/exts/docs_build/code_utils.py | 32 ++
docs/exts/docs_build/docs_builder.py | 317 ++++++++++++----
docs/exts/docs_build/errors.py | 39 +-
docs/exts/docs_build/github_action_utils.py | 1 +
docs/exts/docs_build/spelling_checks.py | 47 ++-
provider_packages/README.rst | 53 +++
scripts/ci/docs/ci_docs.sh | 15 +-
scripts/in_container/_in_container_utils.sh | 9 +-
.../{run_fix_ownership.sh => run_anything.sh} | 5 +-
scripts/in_container/run_fix_ownership.sh | 4 +-
13 files changed, 777 insertions(+), 181 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a70632c..49fb2e7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -463,12 +463,26 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
env:
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
+ PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
with:
persist-credentials: false
submodules: recursive
+ - name: "Setup python"
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
+ - uses: actions/cache@v2
+ id: cache-venv-docs
+ with:
+ path: ./.docs-venv/
+ key: ${{ runner.os }}-docs-venv-${{ hashFiles('setup.py', 'setup.cfg') }}
+ restore-keys: |
+ ${{ runner.os }}-docs-venv-
+ - name: "Free space"
+ run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
run: ./scripts/ci/images/ci_prepare_ci_image_on_ci.sh
- name: "Build docs"
diff --git a/.gitignore b/.gitignore
index 67dfd80..5da0b18 100644
--- a/.gitignore
+++ b/.gitignore
@@ -215,3 +215,8 @@ Chart.lock
pip-wheel-metadata
.pypirc
+/.docs-venv
+
+# Dev files
+/dev/packages.txt
+/dev/Dockerfile.pmc
diff --git a/docs/build_docs.py b/docs/build_docs.py
index 4e4786f..59e1681 100755
--- a/docs/build_docs.py
+++ b/docs/build_docs.py
@@ -16,14 +16,25 @@
# specific language governing permissions and limitations
# under the License.
import argparse
+import multiprocessing
import os
+import platform
import sys
from collections import defaultdict
-from typing import Dict, List, Optional, Tuple
+from subprocess import run
+from typing import Dict, List, NamedTuple, Optional, Tuple
+from rich.console import Console
from tabulate import tabulate
from docs.exts.docs_build import dev_index_generator, lint_checks # pylint: disable=no-name-in-module
+from docs.exts.docs_build.code_utils import (
+ CONSOLE_WIDTH,
+ DOCKER_PROJECT_DIR,
+ ROOT_PROJECT_DIR,
+ TEXT_RED,
+ TEXT_RESET,
+)
from docs.exts.docs_build.docs_builder import ( # pylint: disable=no-name-in-module
DOCS_DIR,
AirflowDocsBuilder,
@@ -60,32 +71,32 @@ ERRORS_ELIGIBLE_TO_REBUILD = [
]
ON_GITHUB_ACTIONS = os.environ.get('GITHUB_ACTIONS', 'false') == "true"
-TEXT_BLUE = '\033[94m'
-TEXT_RESET = '\033[0m'
+
+console = Console(force_terminal=True, color_system="standard", width=CONSOLE_WIDTH)
def _promote_new_flags():
- print(TEXT_BLUE)
- print("Tired of waiting for documentation to be built?")
- print()
+ console.print()
+ console.print("[yellow]Still tired of waiting for documentation to be built?[/]")
+ console.print()
if ON_GITHUB_ACTIONS:
- print("You can quickly build documentation locally with just one command.")
- print(" ./breeze build-docs")
- print()
- print("Still too slow?")
- print()
- print("You can only build one documentation package:")
- print(" ./breeze build-docs -- --package-filter <PACKAGE-NAME>")
- print()
- print("This usually takes from 20 seconds to 2 minutes.")
- print()
- print("You can also use other extra flags to iterate faster:")
- print(" --docs-only - Only build documentation")
- print(" --spellcheck-only - Only perform spellchecking")
- print()
- print("For more info:")
- print(" ./breeze build-docs --help")
- print(TEXT_RESET)
+ console.print("You can quickly build documentation locally with just one command.")
+ console.print(" [blue]./breeze build-docs[/]")
+ console.print()
+ console.print("[yellow]Still too slow?[/]")
+ console.print()
+ console.print("You can only build one documentation package:")
+ console.print(" [blue]./breeze build-docs -- --package-filter <PACKAGE-NAME>[/]")
+ console.print()
+ console.print("This usually takes from [yellow]20 seconds[/] to [yellow]2 minutes[/].")
+ console.print()
+ console.print("You can also use other extra flags to iterate faster:")
+ console.print(" [blue]--docs-only - Only build documentation[/]")
+ console.print(" [blue]--spellcheck-only - Only perform spellchecking[/]")
+ console.print()
+ console.print("For more info:")
+ console.print(" [blue]./breeze build-docs --help[/]")
+ console.print()
def _get_parser():
@@ -116,6 +127,34 @@ def _get_parser():
help='Builds documentation for official release i.e. all links point to stable version',
)
parser.add_argument(
+ "-j",
+ "--jobs",
+ dest='jobs',
+ type=int,
+ default=1,
+ help=(
+ """
+ Number of parallel processes that will be spawned to build the docs.
+
+ This is usually used in CI system only. Though you can also use it to run complete check
+ of the documntation locally if you have powerful local machine.
+ Default is 1 - which means that doc check runs sequentially, This is the default behaviour
+ because autoapi extension we use is not capable of running parallel builds at the same time using
+ the same source files.
+
+ In parallel builds we are using dockerised version of image built from local sources but the image
+ has to be prepared locally (similarly as it is in CI) before you run the docs build. Any changes you
+ have done locally after building the image, will not be checked.
+
+ Typically you run parallel build in this way if you want to quickly run complete check for all docs:
+
+ ./breeze build-image --python 3.6
+ ./docs/build-docs.py -j 0
+
+"""
+ ),
+ )
+ parser.add_argument(
"-v",
"--verbose",
dest='verbose',
@@ -129,31 +168,291 @@ def _get_parser():
return parser
+class BuildSpecification(NamedTuple):
+ """Specification of single build."""
+
+ package_name: str
+ for_production: bool
+ verbose: bool
+ dockerized: bool
+
+
+class BuildDocsResult(NamedTuple):
+ """Result of building documentation."""
+
+ package_name: str
+ log_file_name: str
+ errors: List[DocBuildError]
+
+
+class SpellCheckResult(NamedTuple):
+ """Result of spellcheck."""
+
+ package_name: str
+ log_file_name: str
+ errors: List[SpellingError]
+
+
+def perform_docs_build_for_single_package(build_specification: BuildSpecification) -> BuildDocsResult:
+ """Performs single package docs build."""
+ builder = AirflowDocsBuilder(
+ package_name=build_specification.package_name, for_production=build_specification.for_production
+ )
+ console.print(f"[blue]{build_specification.package_name:60}:[/] Building documentation")
+ result = BuildDocsResult(
+ package_name=build_specification.package_name,
+ errors=builder.build_sphinx_docs(
+ dockerized=build_specification.dockerized,
+ verbose=build_specification.verbose,
+ ),
+ log_file_name=builder.log_build_filename,
+ )
+ return result
+
+
+def perform_spell_check_for_single_package(build_specification: BuildSpecification) -> SpellCheckResult:
+ """Performs single package spell check."""
+ builder = AirflowDocsBuilder(
+ package_name=build_specification.package_name, for_production=build_specification.for_production
+ )
+ console.print(f"[blue]{build_specification.package_name:60}:[/] Checking spelling started")
+ result = SpellCheckResult(
+ package_name=build_specification.package_name,
+ errors=builder.check_spelling(
+ dockerized=build_specification.dockerized,
+ verbose=build_specification.verbose,
+ ),
+ log_file_name=builder.log_spelling_filename,
+ )
+ console.print(f"[blue]{build_specification.package_name:60}:[/] Checking spelling completed")
+ return result
+
+
def build_docs_for_packages(
- current_packages: List[str], docs_only: bool, spellcheck_only: bool, for_production: bool, verbose: bool
+ current_packages: List[str],
+ docs_only: bool,
+ spellcheck_only: bool,
+ for_production: bool,
+ jobs: int,
+ verbose: bool,
) -> Tuple[Dict[str, List[DocBuildError]], Dict[str, List[SpellingError]]]:
- """Builds documentation for single package and returns errors"""
+ """Builds documentation for all packages and combines errors."""
all_build_errors: Dict[str, List[DocBuildError]] = defaultdict(list)
all_spelling_errors: Dict[str, List[SpellingError]] = defaultdict(list)
- for package_no, package_name in enumerate(current_packages, start=1):
- print("#" * 20, f"[{package_no}/{len(current_packages)}] {package_name}", "#" * 20)
- builder = AirflowDocsBuilder(package_name=package_name, for_production=for_production)
- builder.clean_files()
- if not docs_only:
- with with_group(f"Check spelling: {package_name}"):
- spelling_errors = builder.check_spelling(verbose=verbose)
- if spelling_errors:
- all_spelling_errors[package_name].extend(spelling_errors)
-
- if not spellcheck_only:
- with with_group(f"Building docs: {package_name}"):
- docs_errors = builder.build_sphinx_docs(verbose=verbose)
- if docs_errors:
- all_build_errors[package_name].extend(docs_errors)
-
+ with with_group("Cleaning documentation files"):
+ for package_name in current_packages:
+ console.print(f"[blue]{package_name:60}:[/] Cleaning files")
+ builder = AirflowDocsBuilder(package_name=package_name, for_production=for_production)
+ builder.clean_files()
+ if jobs > 1:
+ if os.getenv('CI', '') == '':
+ console.print("[yellow] PARALLEL DOCKERIZED EXECUTION REQUIRES IMAGE TO BE BUILD BEFORE !!!![/]")
+ console.print("[yellow] Make sure that you've build the image before runnning docs build.[/]")
+ console.print("[yellow] otherwise local changes you've done will not be used during the check[/]")
+ console.print()
+ run_in_parallel(
+ all_build_errors,
+ all_spelling_errors,
+ current_packages,
+ docs_only,
+ for_production,
+ jobs,
+ spellcheck_only,
+ verbose,
+ )
+ else:
+ run_sequentially(
+ all_build_errors,
+ all_spelling_errors,
+ current_packages,
+ docs_only,
+ for_production,
+ spellcheck_only,
+ verbose,
+ )
return all_build_errors, all_spelling_errors
+def run_sequentially(
+ all_build_errors,
+ all_spelling_errors,
+ current_packages,
+ docs_only,
+ for_production,
+ spellcheck_only,
+ verbose,
+):
+ """Run both - spellcheck and docs build sequentially without multiprocessing"""
+ if not spellcheck_only:
+ for package_name in current_packages:
+ build_result = perform_docs_build_for_single_package(
+ build_specification=BuildSpecification(
+ package_name=package_name,
+ for_production=for_production,
+ dockerized=False,
+ verbose=verbose,
+ )
+ )
+ if build_result.errors:
+ all_build_errors[package_name].extend(build_result.errors)
+ print_build_output(build_result)
+ if not docs_only:
+ for package_name in current_packages:
+ spellcheck_result = perform_spell_check_for_single_package(
+ build_specification=BuildSpecification(
+ package_name=package_name,
+ for_production=for_production,
+ dockerized=False,
+ verbose=verbose,
+ )
+ )
+ if spellcheck_result.errors:
+ all_spelling_errors[package_name].extend(spellcheck_result.errors)
+ print_spelling_output(spellcheck_result)
+
+
+def run_in_parallel(
+ all_build_errors,
+ all_spelling_errors,
+ current_packages,
+ docs_only,
+ for_production,
+ jobs,
+ spellcheck_only,
+ verbose,
+):
+ """Run both - spellcheck and docs build sequentially without multiprocessing"""
+ pool = multiprocessing.Pool(processes=jobs)
+ # until we fix autoapi, we need to run parallel builds as dockerized images
+ dockerized = True
+ if not spellcheck_only:
+ run_docs_build_in_parallel(
+ all_build_errors=all_build_errors,
+ for_production=for_production,
+ current_packages=current_packages,
+ verbose=verbose,
+ dockerized=dockerized,
+ pool=pool,
+ )
+ if not docs_only:
+ run_spell_check_in_parallel(
+ all_spelling_errors=all_spelling_errors,
+ for_production=for_production,
+ current_packages=current_packages,
+ verbose=verbose,
+ dockerized=dockerized,
+ pool=pool,
+ )
+ fix_ownership()
+
+
+def fix_ownership():
+ """Fixes ownership for all files created with root user,"""
+ console.print("Fixing ownership for generated files")
+ python_version = os.getenv('PYTHON_MAJOR_MINOR_VERSION', "3.6")
+ fix_cmd = [
+ "docker",
+ "run",
+ "--entrypoint",
+ "/bin/bash",
+ "--rm",
+ "-e",
+ f"HOST_OS={platform.system()}",
+ "-e" f"HOST_USER_ID={os.getuid()}",
+ "-e",
+ f"HOST_GROUP_ID={os.getgid()}",
+ "-v",
+ f"{ROOT_PROJECT_DIR}:{DOCKER_PROJECT_DIR}",
+ f"apache/airflow:master-python{python_version}-ci",
+ "-c",
+ "/opt/airflow/scripts/in_container/run_fix_ownership.sh",
+ ]
+ run(fix_cmd, check=True)
+
+
+def print_build_output(result: BuildDocsResult):
+ """Prints output of docs build job."""
+ with with_group(f"{TEXT_RED}Output for documentation build {result.package_name}{TEXT_RESET}"):
+ console.print()
+ console.print(f"[blue]{result.package_name:60}: " + "#" * 80)
+ with open(result.log_file_name) as output:
+ for line in output.read().splitlines():
+ console.print(f"{result.package_name:60} {line}")
+ console.print(f"[blue]{result.package_name:60}: " + "#" * 80)
+
+
+def run_docs_build_in_parallel(
+ all_build_errors: Dict[str, List[DocBuildError]],
+ for_production: bool,
+ current_packages: List[str],
+ verbose: bool,
+ dockerized: bool,
+ pool,
+):
+ """Runs documentation building in parallel."""
+ doc_build_specifications: List[BuildSpecification] = []
+ with with_group("Scheduling documentation to build"):
+ for package_name in current_packages:
+ console.print(f"[blue]{package_name:60}:[/] Scheduling documentation to build")
+ doc_build_specifications.append(
+ BuildSpecification(
+ package_name=package_name,
+ for_production=for_production,
+ verbose=verbose,
+ dockerized=dockerized,
+ )
+ )
+ with with_group("Running docs building"):
+ console.print()
+ result_list = pool.map(perform_docs_build_for_single_package, doc_build_specifications)
+ for result in result_list:
+ if result.errors:
+ all_build_errors[result.package_name].extend(result.errors)
+ print_build_output(result)
+
+
+def print_spelling_output(result: SpellCheckResult):
+ """Prints output of spell check job."""
+ with with_group(f"{TEXT_RED}Output for spelling check: {result.package_name}{TEXT_RESET}"):
+ console.print()
+ console.print(f"[blue]{result.package_name:60}: " + "#" * 80)
+ with open(result.log_file_name) as output:
+ for line in output.read().splitlines():
+ console.print(f"{result.package_name:60} {line}")
+ console.print(f"[blue]{result.package_name:60}: " + "#" * 80)
+ console.print()
+
+
+def run_spell_check_in_parallel(
+ all_spelling_errors: Dict[str, List[SpellingError]],
+ for_production: bool,
+ current_packages: List[str],
+ verbose: bool,
+ dockerized: bool,
+ pool,
+):
+ """Runs spell check in parallel."""
+ spell_check_specifications: List[BuildSpecification] = []
+ with with_group("Scheduling spell checking of documentation"):
+ for package_name in current_packages:
+ console.print(f"[blue]{package_name:60}:[/] Scheduling spellchecking")
+ spell_check_specifications.append(
+ BuildSpecification(
+ package_name=package_name,
+ for_production=for_production,
+ verbose=verbose,
+ dockerized=dockerized,
+ )
+ )
+ with with_group("Running spell checking of documentation"):
+ console.print()
+ result_list = pool.map(perform_spell_check_for_single_package, spell_check_specifications)
+ for result in result_list:
+ if result.errors:
+ all_spelling_errors[result.package_name].extend(result.errors)
+ print_spelling_output(result)
+
+
def display_packages_summary(
build_errors: Dict[str, List[DocBuildError]], spelling_errors: Dict[str, List[SpellingError]]
):
@@ -161,15 +460,15 @@ def display_packages_summary(
packages_names = {*build_errors.keys(), *spelling_errors.keys()}
tabular_data = [
{
- "Package name": package_name,
+ "Package name": f"[blue]{package_name}[/]",
"Count of doc build errors": len(build_errors.get(package_name, [])),
"Count of spelling errors": len(spelling_errors.get(package_name, [])),
}
for package_name in sorted(packages_names, key=lambda k: k or '')
]
- print("#" * 20, "Packages errors summary", "#" * 20)
- print(tabulate(tabular_data=tabular_data, headers="keys"))
- print("#" * 50)
+ console.print("#" * 20, " Packages errors summary ", "#" * 20)
+ console.print(tabulate(tabular_data=tabular_data, headers="keys"))
+ console.print("#" * 50)
def print_build_errors_and_exit(
@@ -180,15 +479,17 @@ def print_build_errors_and_exit(
if build_errors or spelling_errors:
if build_errors:
display_errors_summary(build_errors)
- print()
+ console.print()
if spelling_errors:
display_spelling_error_summary(spelling_errors)
- print()
- print("The documentation has errors.")
+ console.print()
+ console.print("The documentation has errors.")
display_packages_summary(build_errors, spelling_errors)
- print()
- print(CHANNEL_INVITATION)
+ console.print()
+ console.print(CHANNEL_INVITATION)
sys.exit(1)
+ else:
+ console.print("[green]Documentation build is successful[/]")
def main():
@@ -201,15 +502,12 @@ def main():
package_filters = args.package_filter
for_production = args.for_production
- if not package_filters:
- _promote_new_flags()
-
with with_group("Available packages"):
for pkg in sorted(available_packages):
- print(f" - {pkg}")
+ console.print(f" - {pkg}")
if package_filters:
- print("Current package filters: ", package_filters)
+ console.print("Current package filters: ", package_filters)
current_packages = process_package_filters(available_packages, package_filters)
with with_group("Fetching inventories"):
@@ -218,9 +516,12 @@ def main():
priority_packages = fetch_inventories()
current_packages = sorted(current_packages, key=lambda d: -1 if d in priority_packages else 1)
- with with_group(f"Documentation will be built for {len(current_packages)} package(s)"):
+ jobs = args.jobs if args.jobs != 0 else os.cpu_count()
+ with with_group(
+ f"Documentation will be built for {len(current_packages)} package(s) with {jobs} parallel jobs"
+ ):
for pkg_no, pkg in enumerate(current_packages, start=1):
- print(f"{pkg_no}. {pkg}")
+ console.print(f"{pkg_no}. {pkg}")
all_build_errors: Dict[Optional[str], List[DocBuildError]] = {}
all_spelling_errors: Dict[Optional[str], List[SpellingError]] = {}
@@ -229,6 +530,7 @@ def main():
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
+ jobs=jobs,
verbose=args.verbose,
)
if package_build_errors:
@@ -252,6 +554,7 @@ def main():
docs_only=docs_only,
spellcheck_only=spellcheck_only,
for_production=for_production,
+ jobs=jobs,
verbose=args.verbose,
)
if package_build_errors:
diff --git a/docs/exts/docs_build/code_utils.py b/docs/exts/docs_build/code_utils.py
index e77d0b6..07fe8d0 100644
--- a/docs/exts/docs_build/code_utils.py
+++ b/docs/exts/docs_build/code_utils.py
@@ -17,6 +17,38 @@
import os
from contextlib import suppress
+from docs.exts.provider_yaml_utils import load_package_data
+
+ROOT_PROJECT_DIR = os.path.abspath(
+ os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, os.pardir)
+)
+DOCS_DIR = os.path.join(ROOT_PROJECT_DIR, "docs")
+AIRFLOW_DIR = os.path.join(ROOT_PROJECT_DIR, "airflow")
+
+DOCKER_PROJECT_DIR = "/opt/airflow"
+DOCKER_DOCS_DIR = os.path.join(DOCKER_PROJECT_DIR, "docs")
+DOCKER_AIRFLOW_DIR = os.path.join(DOCKER_PROJECT_DIR, "/airflow")
+ALL_PROVIDER_YAMLS = load_package_data()
+AIRFLOW_SITE_DIR = os.environ.get('AIRFLOW_SITE_DIRECTORY')
+PROCESS_TIMEOUT = 4 * 60
+
+TEXT_RED = '\033[31m'
+TEXT_RESET = '\033[0m'
+
+CONSOLE_WIDTH = 180
+
+
+def remap_from_docker(file_name: str, dockerized: bool):
+ """
+ Remaps filename from Docker to Host.
+ :param file_name: name of file
+ :param dockerized: whether builds were running in docker environment.
+ :return:
+ """
+ if dockerized and file_name.startswith(DOCKER_PROJECT_DIR):
+ return file_name.replace(DOCKER_PROJECT_DIR, ROOT_PROJECT_DIR)
+ return file_name
+
def prepare_code_snippet(file_path: str, line_no: int, context_lines_count: int = 5) -> str:
"""
diff --git a/docs/exts/docs_build/docs_builder.py b/docs/exts/docs_build/docs_builder.py
index 71e4acb..0669c75 100644
--- a/docs/exts/docs_build/docs_builder.py
+++ b/docs/exts/docs_build/docs_builder.py
@@ -20,24 +20,27 @@ import shlex
import shutil
from glob import glob
from subprocess import run
-from tempfile import NamedTemporaryFile, TemporaryDirectory
from typing import List
-# pylint: disable=no-name-in-module
-from docs.exts.docs_build.code_utils import pretty_format_path
+from rich.console import Console
+
+from docs.exts.docs_build.code_utils import (
+ AIRFLOW_SITE_DIR,
+ ALL_PROVIDER_YAMLS,
+ CONSOLE_WIDTH,
+ DOCKER_DOCS_DIR,
+ DOCS_DIR,
+ PROCESS_TIMEOUT,
+ pretty_format_path,
+)
from docs.exts.docs_build.errors import DocBuildError, parse_sphinx_warnings
+
+# pylint: disable=no-name-in-module
from docs.exts.docs_build.spelling_checks import SpellingError, parse_spelling_warnings
-from docs.exts.provider_yaml_utils import load_package_data
# pylint: enable=no-name-in-module
-ROOT_PROJECT_DIR = os.path.abspath(
- os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, os.pardir)
-)
-DOCS_DIR = os.path.join(ROOT_PROJECT_DIR, "docs")
-ALL_PROVIDER_YAMLS = load_package_data()
-AIRFLOW_SITE_DIR = os.environ.get('AIRFLOW_SITE_DIRECTORY')
-PROCESS_TIMEOUT = 4 * 60
+console = Console(force_terminal=True, color_system="standard", width=CONSOLE_WIDTH)
class AirflowDocsBuilder:
@@ -52,6 +55,18 @@ class AirflowDocsBuilder:
return f"{DOCS_DIR}/_doctrees/docs/{self.package_name}"
@property
+ def _docker_doctree_dir(self) -> str:
+ return f"{DOCKER_DOCS_DIR}/_doctrees/docs/{self.package_name}"
+
+ @property
+ def _inventory_cache_dir(self) -> str:
+ return f"{DOCS_DIR}/_inventory_cache"
+
+ @property
+ def _docker_inventory_cache_dir(self) -> str:
+ return f"{DOCKER_DOCS_DIR}/_inventory_cache"
+
+ @property
def is_versioned(self):
"""Is current documentation package versioned?"""
# Disable versioning. This documentation does not apply to any released product and we can update
@@ -67,6 +82,54 @@ class AirflowDocsBuilder:
return f"{DOCS_DIR}/_build/docs/{self.package_name}"
@property
+ def log_spelling_filename(self) -> str:
+ """Log from spelling job."""
+ return os.path.join(self._build_dir, f"output-spelling-{self.package_name}.log")
+
+ @property
+ def docker_log_spelling_filename(self) -> str:
+ """Log from spelling job in docker."""
+ return os.path.join(self._docker_build_dir, f"output-spelling-{self.package_name}.log")
+
+ @property
+ def log_spelling_output_dir(self) -> str:
+ """Results from spelling job."""
+ return os.path.join(self._build_dir, f"output-spelling-results-{self.package_name}")
+
+ @property
+ def docker_log_spelling_output_dir(self) -> str:
+ """Results from spelling job in docker."""
+ return os.path.join(self._docker_build_dir, f"output-spelling-results-{self.package_name}")
+
+ @property
+ def log_build_filename(self) -> str:
+ """Log from build job."""
+ return os.path.join(self._build_dir, f"output-build-{self.package_name}.log")
+
+ @property
+ def docker_log_build_filename(self) -> str:
+ """Log from build job in docker."""
+ return os.path.join(self._docker_build_dir, f"output-build-{self.package_name}.log")
+
+ @property
+ def log_build_warning_filename(self) -> str:
+ """Warnings from build job."""
+ return os.path.join(self._build_dir, f"warning-build-{self.package_name}.log")
+
+ @property
+ def docker_log_warning_filename(self) -> str:
+ """Warnings from build job in docker."""
+ return os.path.join(self._docker_build_dir, f"warning-build-{self.package_name}.log")
+
+ @property
+ def _docker_build_dir(self) -> str:
+ if self.is_versioned:
+ version = "stable" if self.for_production else "latest"
+ return f"{DOCKER_DOCS_DIR}/_build/docs/{self.package_name}/{version}"
+ else:
+ return f"{DOCKER_DOCS_DIR}/_build/docs/{self.package_name}"
+
+ @property
def _current_version(self):
if not self.is_versioned:
raise Exception("This documentation package is not versioned")
@@ -90,6 +153,10 @@ class AirflowDocsBuilder:
def _src_dir(self) -> str:
return f"{DOCS_DIR}/{self.package_name}"
+ @property
+ def _docker_src_dir(self) -> str:
+ return f"{DOCKER_DOCS_DIR}/{self.package_name}"
+
def clean_files(self) -> None:
"""Cleanup all artifacts generated by previous builds."""
api_dir = os.path.join(self._src_dir, "_api")
@@ -99,11 +166,42 @@ class AirflowDocsBuilder:
os.makedirs(api_dir, exist_ok=True)
os.makedirs(self._build_dir, exist_ok=True)
- def check_spelling(self, verbose):
- """Checks spelling."""
+ def check_spelling(self, verbose: bool, dockerized: bool) -> List[SpellingError]:
+ """
+ Checks spelling
+
+ :param verbose: whether to show output while running
+ :param dockerized: whether to run dockerized build (required for paralllel processing on CI)
+ :return: list of errors
+ """
spelling_errors = []
- with TemporaryDirectory() as tmp_dir, NamedTemporaryFile() as output:
+ os.makedirs(self._build_dir, exist_ok=True)
+ shutil.rmtree(self.log_spelling_output_dir, ignore_errors=True)
+ os.makedirs(self.log_spelling_output_dir, exist_ok=True)
+ if dockerized:
+ python_version = os.getenv('PYTHON_MAJOR_MINOR_VERSION', "3.6")
build_cmd = [
+ "docker",
+ "run",
+ "--rm",
+ "-e",
+ "AIRFLOW_FOR_PRODUCTION",
+ "-e",
+ "AIRFLOW_PACKAGE_NAME",
+ "-v",
+ f"{self._build_dir}:{self._docker_build_dir}",
+ "-v",
+ f"{self._inventory_cache_dir}:{self._docker_inventory_cache_dir}",
+ "-w",
+ DOCKER_DOCS_DIR,
+ f"apache/airflow:master-python{python_version}-ci",
+ "/opt/airflow/scripts/in_container/run_anything.sh",
+ ]
+ else:
+ build_cmd = []
+
+ build_cmd.extend(
+ [
"sphinx-build",
"-W", # turn warnings into errors
"--color", # do emit colored output
@@ -111,19 +209,26 @@ class AirflowDocsBuilder:
"-b", # builder to use
"spelling",
"-c",
- DOCS_DIR,
+ DOCS_DIR if not dockerized else DOCKER_DOCS_DIR,
"-d", # path for the cached environment and doctree files
- self._doctree_dir,
- self._src_dir, # path to documentation source files
- tmp_dir,
+ self._doctree_dir if not dockerized else self._docker_doctree_dir,
+ self._src_dir
+ if not dockerized
+ else self._docker_src_dir, # path to documentation source files
+ self.log_spelling_output_dir if not dockerized else self.docker_log_spelling_output_dir,
]
- print("Executing cmd: ", " ".join([shlex.quote(c) for c in build_cmd]))
- if not verbose:
- print("The output is hidden until an error occurs.")
- env = os.environ.copy()
- env['AIRFLOW_PACKAGE_NAME'] = self.package_name
- if self.for_production:
- env['AIRFLOW_FOR_PRODUCTION'] = 'true'
+ )
+ env = os.environ.copy()
+ env['AIRFLOW_PACKAGE_NAME'] = self.package_name
+ if self.for_production:
+ env['AIRFLOW_FOR_PRODUCTION'] = 'true'
+ if verbose:
+ console.print(
+ f"[blue]{self.package_name:60}:[/] Executing cmd: ",
+ " ".join([shlex.quote(c) for c in build_cmd]),
+ )
+ console.print(f"[blue]{self.package_name:60}:[/] The output is hidden until an error occurs.")
+ with open(self.log_spelling_filename, "wt") as output:
completed_proc = run( # pylint: disable=subprocess-run-check
build_cmd,
cwd=self._src_dir,
@@ -132,58 +237,101 @@ class AirflowDocsBuilder:
stderr=output if not verbose else None,
timeout=PROCESS_TIMEOUT,
)
- if completed_proc.returncode != 0:
- output.seek(0)
- print(output.read().decode())
-
- spelling_errors.append(
- SpellingError(
- file_path=None,
- line_no=None,
- spelling=None,
- suggestion=None,
- context_line=None,
- message=(
- f"Sphinx spellcheck returned non-zero exit status: {completed_proc.returncode}."
- ),
- )
+ if completed_proc.returncode != 0:
+ spelling_errors.append(
+ SpellingError(
+ file_path=None,
+ line_no=None,
+ spelling=None,
+ suggestion=None,
+ context_line=None,
+ message=(
+ f"Sphinx spellcheck returned non-zero exit status: " f"{completed_proc.returncode}."
+ ),
+ )
+ )
+ warning_text = ""
+ for filepath in glob(f"{self.log_spelling_output_dir}/**/*.spelling", recursive=True):
+ with open(filepath) as spelling_file:
+ warning_text += spelling_file.read()
+ spelling_errors.extend(parse_spelling_warnings(warning_text, self._src_dir, dockerized))
+ console.print(f"[blue]{self.package_name:60}:[/] [red]Finished spell-checking with errors[/]")
+ else:
+ if spelling_errors:
+ console.print(
+ f"[blue]{self.package_name:60}:[/] [yellow]Finished spell-checking " f"with warnings[/]"
+ )
+ else:
+ console.print(
+ f"[blue]{self.package_name:60}:[/] [green]Finished spell-checking " f"successfully[/]"
)
- warning_text = ""
- for filepath in glob(f"{tmp_dir}/**/*.spelling", recursive=True):
- with open(filepath) as speeling_file:
- warning_text += speeling_file.read()
-
- spelling_errors.extend(parse_spelling_warnings(warning_text, self._src_dir))
return spelling_errors
- def build_sphinx_docs(self, verbose) -> List[DocBuildError]:
- """Build Sphinx documentation"""
+ def build_sphinx_docs(self, verbose: bool, dockerized: bool) -> List[DocBuildError]:
+ """
+ Build Sphinx documentation.
+
+ :param verbose: whether to show output while running
+ :param dockerized: whether to run dockerized build (required for paralllel processing on CI)
+ :return: list of errors
+ """
build_errors = []
- with NamedTemporaryFile() as tmp_file, NamedTemporaryFile() as output:
+ os.makedirs(self._build_dir, exist_ok=True)
+ if dockerized:
+ python_version = os.getenv('PYTHON_MAJOR_MINOR_VERSION', "3.6")
build_cmd = [
+ "docker",
+ "run",
+ "--rm",
+ "-e",
+ "AIRFLOW_FOR_PRODUCTION",
+ "-e",
+ "AIRFLOW_PACKAGE_NAME",
+ "-v",
+ f"{self._build_dir}:{self._docker_build_dir}",
+ "-v",
+ f"{self._inventory_cache_dir}:{self._docker_inventory_cache_dir}",
+ "-w",
+ DOCKER_DOCS_DIR,
+ f"apache/airflow:master-python{python_version}-ci",
+ "/opt/airflow/scripts/in_container/run_anything.sh",
+ ]
+ else:
+ build_cmd = []
+ build_cmd.extend(
+ [
"sphinx-build",
"-T", # show full traceback on exception
"--color", # do emit colored output
"-b", # builder to use
"html",
"-d", # path for the cached environment and doctree files
- self._doctree_dir,
+ self._doctree_dir if not dockerized else self._docker_doctree_dir,
"-c",
- DOCS_DIR,
+ DOCS_DIR if not dockerized else DOCKER_DOCS_DIR,
"-w", # write warnings (and errors) to given file
- tmp_file.name,
- self._src_dir, # path to documentation source files
- self._build_dir, # path to output directory
+ self.log_build_warning_filename if not dockerized else self.docker_log_warning_filename,
+ self._src_dir
+ if not dockerized
+ else self._docker_src_dir, # path to documentation source files
+ self._build_dir if not dockerized else self._docker_build_dir, # path to output directory
]
- print("Executing cmd: ", " ".join([shlex.quote(c) for c in build_cmd]))
- if not verbose:
- print("The output is hidden until an error occurs.")
-
- env = os.environ.copy()
- env['AIRFLOW_PACKAGE_NAME'] = self.package_name
- if self.for_production:
- env['AIRFLOW_FOR_PRODUCTION'] = 'true'
-
+ )
+ env = os.environ.copy()
+ env['AIRFLOW_PACKAGE_NAME'] = self.package_name
+ if self.for_production:
+ env['AIRFLOW_FOR_PRODUCTION'] = 'true'
+ if verbose:
+ console.print(
+ f"[blue]{self.package_name:60}:[/] Executing cmd: ",
+ " ".join([shlex.quote(c) for c in build_cmd]),
+ )
+ else:
+ console.print(
+ f"[blue]{self.package_name:60}:[/] Running sphinx. "
+ f"The output is hidden until an error occurs."
+ )
+ with open(self.log_build_filename, "wt") as output:
completed_proc = run( # pylint: disable=subprocess-run-check
build_cmd,
cwd=self._src_dir,
@@ -192,35 +340,48 @@ class AirflowDocsBuilder:
stderr=output if not verbose else None,
timeout=PROCESS_TIMEOUT,
)
- if completed_proc.returncode != 0:
- output.seek(0)
- print(output.read().decode())
- build_errors.append(
- DocBuildError(
- file_path=None,
- line_no=None,
- message=f"Sphinx returned non-zero exit status: {completed_proc.returncode}.",
- )
+ if completed_proc.returncode != 0:
+ build_errors.append(
+ DocBuildError(
+ file_path=None,
+ line_no=None,
+ message=f"Sphinx returned non-zero exit status: {completed_proc.returncode}.",
)
- tmp_file.seek(0)
- warning_text = tmp_file.read().decode()
+ )
+ if os.path.isfile(self.log_build_warning_filename):
+ with open(self.log_build_warning_filename) as warning_file:
+ warning_text = warning_file.read()
# Remove 7-bit C1 ANSI escape sequences
warning_text = re.sub(r"\x1B[@-_][0-?]*[ -/]*[@-~]", "", warning_text)
- build_errors.extend(parse_sphinx_warnings(warning_text, self._src_dir))
+ build_errors.extend(parse_sphinx_warnings(warning_text, self._src_dir, dockerized))
+ if build_errors:
+ console.print(f"[blue]{self.package_name:60}:[/] [red]Finished docs building with errors[/]")
+ else:
+ console.print(f"[blue]{self.package_name:60}:[/] [green]Finished docs building successfully[/]")
return build_errors
def publish(self):
"""Copy documentation packages files to airflow-site repository."""
- print(f"Publishing docs for {self.package_name}")
+ console.print(f"Publishing docs for {self.package_name}")
output_dir = os.path.join(AIRFLOW_SITE_DIR, self._publish_dir)
pretty_source = pretty_format_path(self._build_dir, os.getcwd())
pretty_target = pretty_format_path(output_dir, AIRFLOW_SITE_DIR)
- print(f"Copy directory: {pretty_source} => {pretty_target}")
+ console.print(f"Copy directory: {pretty_source} => {pretty_target}")
+ if os.path.exists(output_dir):
+ if self.is_versioned:
+ console.print(
+ f"Skipping previously existing {output_dir}! "
+ f"Delete it manually if you want to regenerate it!"
+ )
+ console.print()
+ return
+ else:
+ shutil.rmtree(output_dir)
shutil.copytree(self._build_dir, output_dir)
if self.is_versioned:
with open(os.path.join(output_dir, "..", "stable.txt"), "w") as stable_file:
stable_file.write(self._current_version)
- print()
+ console.print()
def get_available_providers_packages():
diff --git a/docs/exts/docs_build/errors.py b/docs/exts/docs_build/errors.py
index 21106ce..954262d 100644
--- a/docs/exts/docs_build/errors.py
+++ b/docs/exts/docs_build/errors.py
@@ -18,11 +18,16 @@ import os
from functools import total_ordering
from typing import Dict, List, NamedTuple, Optional
+from rich.console import Console
+
from airflow.utils.code_utils import prepare_code_snippet
+from docs.exts.docs_build.code_utils import CONSOLE_WIDTH, remap_from_docker
CURRENT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__)))
DOCS_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir, os.pardir))
+console = Console(force_terminal=True, color_system="standard", width=CONSOLE_WIDTH)
+
@total_ordering
class DocBuildError(NamedTuple):
@@ -52,28 +57,32 @@ class DocBuildError(NamedTuple):
def display_errors_summary(build_errors: Dict[str, List[DocBuildError]]) -> None:
"""Displays summary of errors"""
- print("#" * 20, "Docs build errors summary", "#" * 20)
-
+ console.print()
+ console.print("[red]" + "#" * 30 + " Start docs build errors summary " + "#" * 30 + "[/]")
+ console.print()
for package_name, errors in build_errors.items():
if package_name:
- print("=" * 20, package_name, "=" * 20)
+ console.print("=" * 30 + f" [blue]{package_name}[/] " + "=" * 30)
else:
- print("=" * 20, "General", "=" * 20)
+ console.print("=" * 30, " [blue]General[/] ", "=" * 30)
for warning_no, error in enumerate(sorted(errors), 1):
- print("-" * 20, f"Error {warning_no:3}", "-" * 20)
- print(error.message)
- print()
+ console.print("-" * 30, f"[red]Error {warning_no:3}[/]", "-" * 20)
+ console.print(error.message)
+ console.print()
if error.file_path and error.file_path != "<unknown>" and error.line_no:
- print(f"File path: {os.path.relpath(error.file_path, start=DOCS_DIR)} ({error.line_no})")
- print()
- print(prepare_code_snippet(error.file_path, error.line_no))
+ console.print(
+ f"File path: {os.path.relpath(error.file_path, start=DOCS_DIR)} ({error.line_no})"
+ )
+ console.print()
+ console.print(prepare_code_snippet(error.file_path, error.line_no))
elif error.file_path:
- print(f"File path: {error.file_path}")
-
- print("#" * 50)
+ console.print(f"File path: {error.file_path}")
+ console.print()
+ console.print("[red]" + "#" * 30 + " End docs build errors summary " + "#" * 30 + "[/]")
+ console.print()
-def parse_sphinx_warnings(warning_text: str, docs_dir: str) -> List[DocBuildError]:
+def parse_sphinx_warnings(warning_text: str, docs_dir: str, dockerized: bool) -> List[DocBuildError]:
"""
Parses warnings from Sphinx.
@@ -89,7 +98,7 @@ def parse_sphinx_warnings(warning_text: str, docs_dir: str) -> List[DocBuildErro
try:
sphinx_build_errors.append(
DocBuildError(
- file_path=os.path.join(docs_dir, warning_parts[0]),
+ file_path=remap_from_docker(os.path.join(docs_dir, warning_parts[0]), dockerized),
line_no=int(warning_parts[1]),
message=warning_parts[2],
)
diff --git a/docs/exts/docs_build/github_action_utils.py b/docs/exts/docs_build/github_action_utils.py
index 4b21b03..f0fc483 100644
--- a/docs/exts/docs_build/github_action_utils.py
+++ b/docs/exts/docs_build/github_action_utils.py
@@ -33,6 +33,7 @@ def with_group(title):
yield
return
print(f"::group::{title}")
+ print()
yield
print("\033[0m")
print("::endgroup::")
diff --git a/docs/exts/docs_build/spelling_checks.py b/docs/exts/docs_build/spelling_checks.py
index c2b7ca9..2be9cca 100644
--- a/docs/exts/docs_build/spelling_checks.py
+++ b/docs/exts/docs_build/spelling_checks.py
@@ -20,11 +20,16 @@ import re
from functools import total_ordering
from typing import Dict, List, NamedTuple, Optional
+from rich.console import Console
+
from airflow.utils.code_utils import prepare_code_snippet
+from docs.exts.docs_build.code_utils import CONSOLE_WIDTH, remap_from_docker
CURRENT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__)))
DOCS_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir, os.pardir))
+console = Console(force_terminal=True, color_system="standard", width=CONSOLE_WIDTH)
+
@total_ordering
class SpellingError(NamedTuple):
@@ -75,7 +80,7 @@ class SpellingError(NamedTuple):
return left < right
-def parse_spelling_warnings(warning_text: str, docs_dir) -> List[SpellingError]:
+def parse_spelling_warnings(warning_text: str, docs_dir: str, dockerized: bool) -> List[SpellingError]:
"""
Parses warnings from Sphinx.
@@ -94,7 +99,7 @@ def parse_spelling_warnings(warning_text: str, docs_dir) -> List[SpellingError]:
try:
sphinx_spelling_errors.append(
SpellingError(
- file_path=os.path.join(docs_dir, warning_parts[0]),
+ file_path=remap_from_docker(os.path.join(docs_dir, warning_parts[0]), dockerized),
line_no=int(warning_parts[1]) if warning_parts[1] not in ('None', '') else None,
spelling=warning_parts[2],
suggestion=warning_parts[3] if warning_parts[3] else None,
@@ -130,43 +135,47 @@ def parse_spelling_warnings(warning_text: str, docs_dir) -> List[SpellingError]:
def display_spelling_error_summary(spelling_errors: Dict[str, List[SpellingError]]) -> None:
"""Displays summary of Spelling errors"""
- print("#" * 20, "Spelling errors summary", "#" * 20)
+ console.print()
+ console.print("[red]" + "#" * 30 + " Start spelling errors summary " + "#" * 30 + "[/]")
+ console.print()
for package_name, errors in sorted(spelling_errors.items()):
if package_name:
- print("=" * 20, package_name, "=" * 20)
+ console.print("=" * 30, f" [blue]{package_name}[/] ", "=" * 30)
else:
- print("=" * 20, "General", "=" * 20)
+ console.print("=" * 30, " [blue]General[/] ", "=" * 30)
for warning_no, error in enumerate(sorted(errors), 1):
- print("-" * 20, f"Error {warning_no:3}", "-" * 20)
+ console.print("-" * 30, f"Error {warning_no:3}", "-" * 30)
_display_error(error)
- print("=" * 50)
- print()
+ console.print("=" * 100)
+ console.print()
msg = """
If the spelling is correct, add the spelling to docs/spelling_wordlist.txt
or use the spelling directive.
Check https://sphinxcontrib-spelling.readthedocs.io/en/latest/customize.html#private-dictionaries
for more details.
"""
- print(msg)
- print()
- print("#" * 50)
+ console.print(msg)
+ console.print()
+ console.print
+ console.print("[red]" + "#" * 30 + " End docs build errors summary " + "#" * 30 + "[/]")
+ console.print
def _display_error(error: SpellingError):
- print(error.message)
- print()
+ console.print(error.message)
+ console.print()
if error.file_path:
- print(f"File path: {os.path.relpath(error.file_path, start=DOCS_DIR)}")
+ console.print(f"File path: {os.path.relpath(error.file_path, start=DOCS_DIR)}")
if error.spelling:
- print(f"Incorrect Spelling: '{error.spelling}'")
+ console.print(f"Incorrect Spelling: '{error.spelling}'")
if error.suggestion:
- print(f"Suggested Spelling: '{error.suggestion}'")
+ console.print(f"Suggested Spelling: '{error.suggestion}'")
if error.context_line:
- print(f"Line with Error: '{error.context_line}'")
+ console.print(f"Line with Error: '{error.context_line}'")
if error.line_no:
- print(f"Line Number: {error.line_no}")
- print(prepare_code_snippet(error.file_path, error.line_no))
+ console.print(f"Line Number: {error.line_no}")
+ console.print(prepare_code_snippet(error.file_path, error.line_no))
diff --git a/provider_packages/README.rst b/provider_packages/README.rst
new file mode 100644
index 0000000..9761c5e
--- /dev/null
+++ b/provider_packages/README.rst
@@ -0,0 +1,53 @@
+
+.. Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+.. http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+
+Package ``apache-airflow-providers-ssh``
+
+Release: ``1.0.0dev``
+
+
+`Secure Shell (SSH) <https://tools.ietf.org/html/rfc4251>`__
+
+
+Provider package
+================
+
+This is a provider package for ``ssh`` provider. All classes for this provider package
+are in ``airflow.providers.ssh`` python package.
+
+You can find package information and changelog for the provider
+in the `documentation <https://airflow.apache.org/docs/apache-airflow-providers-ssh/1.0.0/>`_.
+
+
+Installation
+============
+
+You can install this package on top of an existing airflow 2.* installation via
+``pip install apache-airflow-providers-ssh``
+
+PIP requirements
+================
+
+============= ==================
+PIP package Version required
+============= ==================
+``paramiko`` ``>=2.6.0``
+``pysftp`` ``>=0.2.9``
+``sshtunnel`` ``>=0.1.4,<0.2``
+============= ==================
diff --git a/scripts/ci/docs/ci_docs.sh b/scripts/ci/docs/ci_docs.sh
index be0d2ed..003a8c2 100755
--- a/scripts/ci/docs/ci_docs.sh
+++ b/scripts/ci/docs/ci_docs.sh
@@ -22,4 +22,17 @@ build_images::prepare_ci_build
build_images::rebuild_ci_image_if_needed_with_group
-runs::run_docs "${@}"
+start_end::group_start "Preparing venv for doc building"
+
+python3 -m venv .docs-venv
+source .docs-venv/bin/activate
+export PYTHONPATH=${AIRFLOW_SOURCES}
+
+pip install --upgrade pip==20.2.4
+
+pip install .[doc] --upgrade --constraint \
+ "https://raw.githubusercontent.com/apache/airflow/constraints-${DEFAULT_BRANCH}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"
+
+start_end::group_end
+
+"${AIRFLOW_SOURCES}/docs/build_docs.py" -j 0 "${@}"
diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh
index f0006e1..7d80f00 100644
--- a/scripts/in_container/_in_container_utils.sh
+++ b/scripts/in_container/_in_container_utils.sh
@@ -137,17 +137,16 @@ function in_container_cleanup_pycache() {
function in_container_fix_ownership() {
if [[ ${HOST_OS:=} == "Linux" ]]; then
DIRECTORIES_TO_FIX=(
- "/tmp"
"/files"
"/root/.aws"
"/root/.azure"
"/root/.config/gcloud"
"/root/.docker"
- "${AIRFLOW_SOURCES}"
+ "/opt/airflow/logs"
+ "/opt/airflow/docs"
)
- sudo find "${DIRECTORIES_TO_FIX[@]}" -print0 -user root 2>/dev/null |
- sudo xargs --null chown "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference ||
- true >/dev/null 2>&1
+ find "${DIRECTORIES_TO_FIX[@]}" -print0 -user root 2>/dev/null |
+ xargs --null chown "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference || true >/dev/null 2>&1
fi
}
diff --git a/scripts/in_container/run_fix_ownership.sh b/scripts/in_container/run_anything.sh
similarity index 83%
copy from scripts/in_container/run_fix_ownership.sh
copy to scripts/in_container/run_anything.sh
index eaaee77..233cb47 100755
--- a/scripts/in_container/run_fix_ownership.sh
+++ b/scripts/in_container/run_anything.sh
@@ -15,7 +15,4 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-# shellcheck source=scripts/in_container/_in_container_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
-
-in_container_fix_ownership
+"${@}"
diff --git a/scripts/in_container/run_fix_ownership.sh b/scripts/in_container/run_fix_ownership.sh
index eaaee77..d9e98ff 100755
--- a/scripts/in_container/run_fix_ownership.sh
+++ b/scripts/in_container/run_fix_ownership.sh
@@ -15,7 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-# shellcheck source=scripts/in_container/_in_container_script_init.sh
-. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_script_init.sh"
+# shellcheck source=scripts/in_container/_in_container_utils.sh
+. "$( dirname "${BASH_SOURCE[0]}" )/_in_container_utils.sh"
in_container_fix_ownership