You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2021/04/06 01:39:14 UTC
[airflow] branch v2-0-test updated (601193f -> 5ce558c)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to branch v2-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git.
discard 601193f Less docker magic in docs building (#15176)
new 5ce558c Less docker magic in docs building (#15176)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (601193f)
\
N -- N -- N refs/heads/v2-0-test (5ce558c)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
[airflow] 01/01: Less docker magic in docs building (#15176)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v2-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 5ce558c3f91afe1ffb488bf5ec756a4f18d4c883
Author: Kamil BreguĊa <mi...@users.noreply.github.com>
AuthorDate: Tue Apr 6 03:10:42 2021 +0200
Less docker magic in docs building (#15176)
(cherry picked from commit 3bd11631ff0fbff4859452513efe03674b04b141)
---
.github/workflows/ci.yml | 12 --
docs/build_docs.py | 91 +++------------
docs/conf.py | 2 +-
docs/exts/docs_build/code_utils.py | 16 +--
docs/exts/docs_build/docs_builder.py | 171 +++++++----------------------
docs/exts/docs_build/errors.py | 6 +-
docs/exts/docs_build/run_patched_sphinx.py | 105 ++++++++++++++++++
docs/exts/docs_build/spelling_checks.py | 6 +-
docs/exts/provider_init_hack.py | 10 +-
scripts/ci/docs/ci_docs.sh | 15 +--
10 files changed, 170 insertions(+), 264 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index dc98f5c..86bc960 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -463,24 +463,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
env:
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
- PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
with:
persist-credentials: false
submodules: recursive
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
- - uses: actions/cache@v2
- id: cache-venv-docs
- with:
- path: ./.docs-venv/
- key: ${{ runner.os }}-docs-venv-${{ hashFiles('setup.py', 'setup.cfg') }}
- restore-keys: |
- ${{ runner.os }}-docs-venv-
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
diff --git a/docs/build_docs.py b/docs/build_docs.py
index 59e1681..5f1a534 100755
--- a/docs/build_docs.py
+++ b/docs/build_docs.py
@@ -18,23 +18,15 @@
import argparse
import multiprocessing
import os
-import platform
import sys
from collections import defaultdict
-from subprocess import run
from typing import Dict, List, NamedTuple, Optional, Tuple
from rich.console import Console
from tabulate import tabulate
from docs.exts.docs_build import dev_index_generator, lint_checks # pylint: disable=no-name-in-module
-from docs.exts.docs_build.code_utils import (
- CONSOLE_WIDTH,
- DOCKER_PROJECT_DIR,
- ROOT_PROJECT_DIR,
- TEXT_RED,
- TEXT_RESET,
-)
+from docs.exts.docs_build.code_utils import CONSOLE_WIDTH, PROVIDER_INIT_FILE, TEXT_RED, TEXT_RESET
from docs.exts.docs_build.docs_builder import ( # pylint: disable=no-name-in-module
DOCS_DIR,
AirflowDocsBuilder,
@@ -52,7 +44,7 @@ from docs.exts.docs_build.spelling_checks import ( # pylint: disable=no-name-in
display_spelling_error_summary,
)
-if __name__ != "__main__":
+if __name__ not in ("__main__", "__mp_main__"):
raise SystemExit(
"This file is intended to be executed as an executable program. You cannot use it as a module."
"To run this script, run the ./build_docs.py command"
@@ -131,27 +123,13 @@ def _get_parser():
"--jobs",
dest='jobs',
type=int,
- default=1,
+ default=0,
help=(
- """
- Number of parallel processes that will be spawned to build the docs.
-
- This is usually used in CI system only. Though you can also use it to run complete check
- of the documntation locally if you have powerful local machine.
- Default is 1 - which means that doc check runs sequentially, This is the default behaviour
- because autoapi extension we use is not capable of running parallel builds at the same time using
- the same source files.
-
- In parallel builds we are using dockerised version of image built from local sources but the image
- has to be prepared locally (similarly as it is in CI) before you run the docs build. Any changes you
- have done locally after building the image, will not be checked.
-
- Typically you run parallel build in this way if you want to quickly run complete check for all docs:
+ """\
+ Number of parallel processes that will be spawned to build the docs.
- ./breeze build-image --python 3.6
- ./docs/build-docs.py -j 0
-
-"""
+ If passed 0, the value will be determined based on the number of CPUs.
+ """
),
)
parser.add_argument(
@@ -174,7 +152,6 @@ class BuildSpecification(NamedTuple):
package_name: str
for_production: bool
verbose: bool
- dockerized: bool
class BuildDocsResult(NamedTuple):
@@ -202,7 +179,6 @@ def perform_docs_build_for_single_package(build_specification: BuildSpecificatio
result = BuildDocsResult(
package_name=build_specification.package_name,
errors=builder.build_sphinx_docs(
- dockerized=build_specification.dockerized,
verbose=build_specification.verbose,
),
log_file_name=builder.log_build_filename,
@@ -219,7 +195,6 @@ def perform_spell_check_for_single_package(build_specification: BuildSpecificati
result = SpellCheckResult(
package_name=build_specification.package_name,
errors=builder.check_spelling(
- dockerized=build_specification.dockerized,
verbose=build_specification.verbose,
),
log_file_name=builder.log_spelling_filename,
@@ -245,11 +220,6 @@ def build_docs_for_packages(
builder = AirflowDocsBuilder(package_name=package_name, for_production=for_production)
builder.clean_files()
if jobs > 1:
- if os.getenv('CI', '') == '':
- console.print("[yellow] PARALLEL DOCKERIZED EXECUTION REQUIRES IMAGE TO BE BUILD BEFORE !!!![/]")
- console.print("[yellow] Make sure that you've build the image before runnning docs build.[/]")
- console.print("[yellow] otherwise local changes you've done will not be used during the check[/]")
- console.print()
run_in_parallel(
all_build_errors,
all_spelling_errors,
@@ -289,7 +259,6 @@ def run_sequentially(
build_specification=BuildSpecification(
package_name=package_name,
for_production=for_production,
- dockerized=False,
verbose=verbose,
)
)
@@ -302,7 +271,6 @@ def run_sequentially(
build_specification=BuildSpecification(
package_name=package_name,
for_production=for_production,
- dockerized=False,
verbose=verbose,
)
)
@@ -323,15 +291,12 @@ def run_in_parallel(
):
"""Run both - spellcheck and docs build sequentially without multiprocessing"""
pool = multiprocessing.Pool(processes=jobs)
- # until we fix autoapi, we need to run parallel builds as dockerized images
- dockerized = True
if not spellcheck_only:
run_docs_build_in_parallel(
all_build_errors=all_build_errors,
for_production=for_production,
current_packages=current_packages,
verbose=verbose,
- dockerized=dockerized,
pool=pool,
)
if not docs_only:
@@ -340,34 +305,8 @@ def run_in_parallel(
for_production=for_production,
current_packages=current_packages,
verbose=verbose,
- dockerized=dockerized,
pool=pool,
)
- fix_ownership()
-
-
-def fix_ownership():
- """Fixes ownership for all files created with root user,"""
- console.print("Fixing ownership for generated files")
- python_version = os.getenv('PYTHON_MAJOR_MINOR_VERSION', "3.6")
- fix_cmd = [
- "docker",
- "run",
- "--entrypoint",
- "/bin/bash",
- "--rm",
- "-e",
- f"HOST_OS={platform.system()}",
- "-e" f"HOST_USER_ID={os.getuid()}",
- "-e",
- f"HOST_GROUP_ID={os.getgid()}",
- "-v",
- f"{ROOT_PROJECT_DIR}:{DOCKER_PROJECT_DIR}",
- f"apache/airflow:master-python{python_version}-ci",
- "-c",
- "/opt/airflow/scripts/in_container/run_fix_ownership.sh",
- ]
- run(fix_cmd, check=True)
def print_build_output(result: BuildDocsResult):
@@ -386,7 +325,6 @@ def run_docs_build_in_parallel(
for_production: bool,
current_packages: List[str],
verbose: bool,
- dockerized: bool,
pool,
):
"""Runs documentation building in parallel."""
@@ -399,7 +337,6 @@ def run_docs_build_in_parallel(
package_name=package_name,
for_production=for_production,
verbose=verbose,
- dockerized=dockerized,
)
)
with with_group("Running docs building"):
@@ -428,7 +365,6 @@ def run_spell_check_in_parallel(
for_production: bool,
current_packages: List[str],
verbose: bool,
- dockerized: bool,
pool,
):
"""Runs spell check in parallel."""
@@ -437,12 +373,7 @@ def run_spell_check_in_parallel(
for package_name in current_packages:
console.print(f"[blue]{package_name:60}:[/] Scheduling spellchecking")
spell_check_specifications.append(
- BuildSpecification(
- package_name=package_name,
- for_production=for_production,
- verbose=verbose,
- dockerized=dockerized,
- )
+ BuildSpecification(package_name=package_name, for_production=for_production, verbose=verbose)
)
with with_group("Running spell checking of documentation"):
console.print()
@@ -572,10 +503,14 @@ def main():
if not package_filters:
_promote_new_flags()
+ if os.path.exists(PROVIDER_INIT_FILE):
+ os.remove(PROVIDER_INIT_FILE)
+
print_build_errors_and_exit(
all_build_errors,
all_spelling_errors,
)
-main()
+if __name__ == "__main__":
+ main()
diff --git a/docs/conf.py b/docs/conf.py
index 678f053..11708f9 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -506,7 +506,7 @@ autoapi_keep_files = True
# Relative path to output the AutoAPI files into. This can also be used to place the generated documentation
# anywhere in your documentation hierarchy.
-autoapi_root = f'{PACKAGE_NAME}/_api'
+autoapi_root = '_api'
# Whether to insert the generated documentation into the TOC tree. If this is False, the default AutoAPI
# index page is not generated and you will need to include the generated documentation in a
diff --git a/docs/exts/docs_build/code_utils.py b/docs/exts/docs_build/code_utils.py
index 5c88797..adab5c2 100644
--- a/docs/exts/docs_build/code_utils.py
+++ b/docs/exts/docs_build/code_utils.py
@@ -22,12 +22,10 @@ from docs.exts.provider_yaml_utils import load_package_data
ROOT_PROJECT_DIR = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, os.pardir)
)
+PROVIDER_INIT_FILE = os.path.join(ROOT_PROJECT_DIR, "airflow", "providers", "__init__.py")
DOCS_DIR = os.path.join(ROOT_PROJECT_DIR, "docs")
AIRFLOW_DIR = os.path.join(ROOT_PROJECT_DIR, "airflow")
-DOCKER_PROJECT_DIR = "/opt/airflow"
-DOCKER_DOCS_DIR = os.path.join(DOCKER_PROJECT_DIR, "docs")
-DOCKER_AIRFLOW_DIR = os.path.join(DOCKER_PROJECT_DIR, "/airflow")
ALL_PROVIDER_YAMLS = load_package_data()
AIRFLOW_SITE_DIR = os.environ.get('AIRFLOW_SITE_DIRECTORY')
PROCESS_TIMEOUT = 8 * 60 # 400 seconds
@@ -38,18 +36,6 @@ TEXT_RESET = '\033[0m'
CONSOLE_WIDTH = 180
-def remap_from_docker(file_name: str, dockerized: bool):
- """
- Remaps filename from Docker to Host.
- :param file_name: name of file
- :param dockerized: whether builds were running in docker environment.
- :return:
- """
- if dockerized and file_name.startswith(DOCKER_PROJECT_DIR):
- return file_name.replace(DOCKER_PROJECT_DIR, ROOT_PROJECT_DIR)
- return file_name
-
-
def prepare_code_snippet(file_path: str, line_no: int, context_lines_count: int = 5) -> str:
"""
Prepares code snippet.
diff --git a/docs/exts/docs_build/docs_builder.py b/docs/exts/docs_build/docs_builder.py
index 0669c75..669d76d 100644
--- a/docs/exts/docs_build/docs_builder.py
+++ b/docs/exts/docs_build/docs_builder.py
@@ -28,9 +28,9 @@ from docs.exts.docs_build.code_utils import (
AIRFLOW_SITE_DIR,
ALL_PROVIDER_YAMLS,
CONSOLE_WIDTH,
- DOCKER_DOCS_DIR,
DOCS_DIR,
PROCESS_TIMEOUT,
+ ROOT_PROJECT_DIR,
pretty_format_path,
)
from docs.exts.docs_build.errors import DocBuildError, parse_sphinx_warnings
@@ -55,18 +55,10 @@ class AirflowDocsBuilder:
return f"{DOCS_DIR}/_doctrees/docs/{self.package_name}"
@property
- def _docker_doctree_dir(self) -> str:
- return f"{DOCKER_DOCS_DIR}/_doctrees/docs/{self.package_name}"
-
- @property
def _inventory_cache_dir(self) -> str:
return f"{DOCS_DIR}/_inventory_cache"
@property
- def _docker_inventory_cache_dir(self) -> str:
- return f"{DOCKER_DOCS_DIR}/_inventory_cache"
-
- @property
def is_versioned(self):
"""Is current documentation package versioned?"""
# Disable versioning. This documentation does not apply to any released product and we can update
@@ -87,49 +79,21 @@ class AirflowDocsBuilder:
return os.path.join(self._build_dir, f"output-spelling-{self.package_name}.log")
@property
- def docker_log_spelling_filename(self) -> str:
- """Log from spelling job in docker."""
- return os.path.join(self._docker_build_dir, f"output-spelling-{self.package_name}.log")
-
- @property
def log_spelling_output_dir(self) -> str:
"""Results from spelling job."""
return os.path.join(self._build_dir, f"output-spelling-results-{self.package_name}")
@property
- def docker_log_spelling_output_dir(self) -> str:
- """Results from spelling job in docker."""
- return os.path.join(self._docker_build_dir, f"output-spelling-results-{self.package_name}")
-
- @property
def log_build_filename(self) -> str:
"""Log from build job."""
return os.path.join(self._build_dir, f"output-build-{self.package_name}.log")
@property
- def docker_log_build_filename(self) -> str:
- """Log from build job in docker."""
- return os.path.join(self._docker_build_dir, f"output-build-{self.package_name}.log")
-
- @property
def log_build_warning_filename(self) -> str:
"""Warnings from build job."""
return os.path.join(self._build_dir, f"warning-build-{self.package_name}.log")
@property
- def docker_log_warning_filename(self) -> str:
- """Warnings from build job in docker."""
- return os.path.join(self._docker_build_dir, f"warning-build-{self.package_name}.log")
-
- @property
- def _docker_build_dir(self) -> str:
- if self.is_versioned:
- version = "stable" if self.for_production else "latest"
- return f"{DOCKER_DOCS_DIR}/_build/docs/{self.package_name}/{version}"
- else:
- return f"{DOCKER_DOCS_DIR}/_build/docs/{self.package_name}"
-
- @property
def _current_version(self):
if not self.is_versioned:
raise Exception("This documentation package is not versioned")
@@ -153,10 +117,6 @@ class AirflowDocsBuilder:
def _src_dir(self) -> str:
return f"{DOCS_DIR}/{self.package_name}"
- @property
- def _docker_src_dir(self) -> str:
- return f"{DOCKER_DOCS_DIR}/{self.package_name}"
-
def clean_files(self) -> None:
"""Cleanup all artifacts generated by previous builds."""
api_dir = os.path.join(self._src_dir, "_api")
@@ -166,58 +126,33 @@ class AirflowDocsBuilder:
os.makedirs(api_dir, exist_ok=True)
os.makedirs(self._build_dir, exist_ok=True)
- def check_spelling(self, verbose: bool, dockerized: bool) -> List[SpellingError]:
+ def check_spelling(self, verbose: bool) -> List[SpellingError]:
"""
Checks spelling
:param verbose: whether to show output while running
- :param dockerized: whether to run dockerized build (required for paralllel processing on CI)
:return: list of errors
"""
spelling_errors = []
os.makedirs(self._build_dir, exist_ok=True)
shutil.rmtree(self.log_spelling_output_dir, ignore_errors=True)
os.makedirs(self.log_spelling_output_dir, exist_ok=True)
- if dockerized:
- python_version = os.getenv('PYTHON_MAJOR_MINOR_VERSION', "3.6")
- build_cmd = [
- "docker",
- "run",
- "--rm",
- "-e",
- "AIRFLOW_FOR_PRODUCTION",
- "-e",
- "AIRFLOW_PACKAGE_NAME",
- "-v",
- f"{self._build_dir}:{self._docker_build_dir}",
- "-v",
- f"{self._inventory_cache_dir}:{self._docker_inventory_cache_dir}",
- "-w",
- DOCKER_DOCS_DIR,
- f"apache/airflow:master-python{python_version}-ci",
- "/opt/airflow/scripts/in_container/run_anything.sh",
- ]
- else:
- build_cmd = []
-
- build_cmd.extend(
- [
- "sphinx-build",
- "-W", # turn warnings into errors
- "--color", # do emit colored output
- "-T", # show full traceback on exception
- "-b", # builder to use
- "spelling",
- "-c",
- DOCS_DIR if not dockerized else DOCKER_DOCS_DIR,
- "-d", # path for the cached environment and doctree files
- self._doctree_dir if not dockerized else self._docker_doctree_dir,
- self._src_dir
- if not dockerized
- else self._docker_src_dir, # path to documentation source files
- self.log_spelling_output_dir if not dockerized else self.docker_log_spelling_output_dir,
- ]
- )
+
+ build_cmd = [
+ os.path.join(ROOT_PROJECT_DIR, "docs", "exts", "docs_build", "run_patched_sphinx.py"),
+ "-W", # turn warnings into errors
+ "--color", # do emit colored output
+ "-T", # show full traceback on exception
+ "-b", # builder to use
+ "spelling",
+ "-c",
+ DOCS_DIR,
+ "-d", # path for the cached environment and doctree files
+ self._doctree_dir,
+ self._src_dir, # path to documentation source files
+ self.log_spelling_output_dir,
+ ]
+
env = os.environ.copy()
env['AIRFLOW_PACKAGE_NAME'] = self.package_name
if self.for_production:
@@ -246,7 +181,7 @@ class AirflowDocsBuilder:
suggestion=None,
context_line=None,
message=(
- f"Sphinx spellcheck returned non-zero exit status: " f"{completed_proc.returncode}."
+ f"Sphinx spellcheck returned non-zero exit status: {completed_proc.returncode}."
),
)
)
@@ -254,69 +189,45 @@ class AirflowDocsBuilder:
for filepath in glob(f"{self.log_spelling_output_dir}/**/*.spelling", recursive=True):
with open(filepath) as spelling_file:
warning_text += spelling_file.read()
- spelling_errors.extend(parse_spelling_warnings(warning_text, self._src_dir, dockerized))
+
+ spelling_errors.extend(parse_spelling_warnings(warning_text, self._src_dir))
console.print(f"[blue]{self.package_name:60}:[/] [red]Finished spell-checking with errors[/]")
else:
if spelling_errors:
console.print(
- f"[blue]{self.package_name:60}:[/] [yellow]Finished spell-checking " f"with warnings[/]"
+ f"[blue]{self.package_name:60}:[/] [yellow]Finished spell-checking with warnings[/]"
)
else:
console.print(
- f"[blue]{self.package_name:60}:[/] [green]Finished spell-checking " f"successfully[/]"
+ f"[blue]{self.package_name:60}:[/] [green]Finished spell-checking successfully[/]"
)
return spelling_errors
- def build_sphinx_docs(self, verbose: bool, dockerized: bool) -> List[DocBuildError]:
+ def build_sphinx_docs(self, verbose: bool) -> List[DocBuildError]:
"""
Build Sphinx documentation.
:param verbose: whether to show output while running
- :param dockerized: whether to run dockerized build (required for paralllel processing on CI)
:return: list of errors
"""
build_errors = []
os.makedirs(self._build_dir, exist_ok=True)
- if dockerized:
- python_version = os.getenv('PYTHON_MAJOR_MINOR_VERSION', "3.6")
- build_cmd = [
- "docker",
- "run",
- "--rm",
- "-e",
- "AIRFLOW_FOR_PRODUCTION",
- "-e",
- "AIRFLOW_PACKAGE_NAME",
- "-v",
- f"{self._build_dir}:{self._docker_build_dir}",
- "-v",
- f"{self._inventory_cache_dir}:{self._docker_inventory_cache_dir}",
- "-w",
- DOCKER_DOCS_DIR,
- f"apache/airflow:master-python{python_version}-ci",
- "/opt/airflow/scripts/in_container/run_anything.sh",
- ]
- else:
- build_cmd = []
- build_cmd.extend(
- [
- "sphinx-build",
- "-T", # show full traceback on exception
- "--color", # do emit colored output
- "-b", # builder to use
- "html",
- "-d", # path for the cached environment and doctree files
- self._doctree_dir if not dockerized else self._docker_doctree_dir,
- "-c",
- DOCS_DIR if not dockerized else DOCKER_DOCS_DIR,
- "-w", # write warnings (and errors) to given file
- self.log_build_warning_filename if not dockerized else self.docker_log_warning_filename,
- self._src_dir
- if not dockerized
- else self._docker_src_dir, # path to documentation source files
- self._build_dir if not dockerized else self._docker_build_dir, # path to output directory
- ]
- )
+
+ build_cmd = [
+ os.path.join(ROOT_PROJECT_DIR, "docs", "exts", "docs_build", "run_patched_sphinx.py"),
+ "-T", # show full traceback on exception
+ "--color", # do emit colored output
+ "-b", # builder to use
+ "html",
+ "-d", # path for the cached environment and doctree files
+ self._doctree_dir,
+ "-c",
+ DOCS_DIR,
+ "-w", # write warnings (and errors) to given file
+ self.log_build_warning_filename,
+ self._src_dir,
+ self._build_dir, # path to output directory
+ ]
env = os.environ.copy()
env['AIRFLOW_PACKAGE_NAME'] = self.package_name
if self.for_production:
@@ -353,7 +264,7 @@ class AirflowDocsBuilder:
warning_text = warning_file.read()
# Remove 7-bit C1 ANSI escape sequences
warning_text = re.sub(r"\x1B[@-_][0-?]*[ -/]*[@-~]", "", warning_text)
- build_errors.extend(parse_sphinx_warnings(warning_text, self._src_dir, dockerized))
+ build_errors.extend(parse_sphinx_warnings(warning_text, self._src_dir))
if build_errors:
console.print(f"[blue]{self.package_name:60}:[/] [red]Finished docs building with errors[/]")
else:
diff --git a/docs/exts/docs_build/errors.py b/docs/exts/docs_build/errors.py
index 954262d..1a2ae06 100644
--- a/docs/exts/docs_build/errors.py
+++ b/docs/exts/docs_build/errors.py
@@ -21,7 +21,7 @@ from typing import Dict, List, NamedTuple, Optional
from rich.console import Console
from airflow.utils.code_utils import prepare_code_snippet
-from docs.exts.docs_build.code_utils import CONSOLE_WIDTH, remap_from_docker
+from docs.exts.docs_build.code_utils import CONSOLE_WIDTH
CURRENT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__)))
DOCS_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir, os.pardir))
@@ -82,7 +82,7 @@ def display_errors_summary(build_errors: Dict[str, List[DocBuildError]]) -> None
console.print()
-def parse_sphinx_warnings(warning_text: str, docs_dir: str, dockerized: bool) -> List[DocBuildError]:
+def parse_sphinx_warnings(warning_text: str, docs_dir: str) -> List[DocBuildError]:
"""
Parses warnings from Sphinx.
@@ -98,7 +98,7 @@ def parse_sphinx_warnings(warning_text: str, docs_dir: str, dockerized: bool) ->
try:
sphinx_build_errors.append(
DocBuildError(
- file_path=remap_from_docker(os.path.join(docs_dir, warning_parts[0]), dockerized),
+ file_path=os.path.join(docs_dir, warning_parts[0]),
line_no=int(warning_parts[1]),
message=warning_parts[2],
)
diff --git a/docs/exts/docs_build/run_patched_sphinx.py b/docs/exts/docs_build/run_patched_sphinx.py
new file mode 100755
index 0000000..887b982
--- /dev/null
+++ b/docs/exts/docs_build/run_patched_sphinx.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import sys
+
+import autoapi
+from autoapi.extension import (
+ LOGGER,
+ ExtensionError,
+ bold,
+ darkgreen,
+ default_backend_mapping,
+ default_file_mapping,
+ default_ignore_patterns,
+)
+from sphinx.cmd.build import main
+
+
+def run_autoapi(app):
+ """Load AutoAPI data from the filesystem."""
+ if not app.config.autoapi_dirs:
+ raise ExtensionError("You must configure an autoapi_dirs setting")
+
+ # Make sure the paths are full
+ normalized_dirs = []
+ autoapi_dirs = app.config.autoapi_dirs
+ if isinstance(autoapi_dirs, str):
+ autoapi_dirs = [autoapi_dirs]
+ for path in autoapi_dirs:
+ if os.path.isabs(path):
+ normalized_dirs.append(path)
+ else:
+ normalized_dirs.append(os.path.normpath(os.path.join(app.confdir, path)))
+
+ for _dir in normalized_dirs:
+ if not os.path.exists(_dir):
+ raise ExtensionError(
+ "AutoAPI Directory `{dir}` not found. "
+ "Please check your `autoapi_dirs` setting.".format(dir=_dir)
+ )
+
+ # Change from app.confdir to app.srcdir.
+ # Before:
+ # - normalized_root = os.path.normpath(
+ # - os.path.join(app.confdir, app.config.autoapi_root)
+ # -)
+ normalized_root = os.path.normpath(os.path.join(app.srcdir, app.config.autoapi_root))
+ url_root = os.path.join("/", app.config.autoapi_root)
+ sphinx_mapper = default_backend_mapping[app.config.autoapi_type]
+ sphinx_mapper_obj = sphinx_mapper(app, template_dir=app.config.autoapi_template_dir, url_root=url_root)
+ app.env.autoapi_mapper = sphinx_mapper_obj
+
+ if app.config.autoapi_file_patterns:
+ file_patterns = app.config.autoapi_file_patterns
+ else:
+ file_patterns = default_file_mapping.get(app.config.autoapi_type, [])
+
+ if app.config.autoapi_ignore:
+ ignore_patterns = app.config.autoapi_ignore
+ else:
+ ignore_patterns = default_ignore_patterns.get(app.config.autoapi_type, [])
+
+ if ".rst" in app.config.source_suffix:
+ out_suffix = ".rst"
+ elif ".txt" in app.config.source_suffix:
+ out_suffix = ".txt"
+ else:
+ # Fallback to first suffix listed
+ out_suffix = app.config.source_suffix[0]
+
+ # Actual meat of the run.
+ LOGGER.info(bold("[AutoAPI] ") + darkgreen("Loading Data"))
+ sphinx_mapper_obj.load(patterns=file_patterns, dirs=normalized_dirs, ignore=ignore_patterns)
+
+ LOGGER.info(bold("[AutoAPI] ") + darkgreen("Mapping Data"))
+ sphinx_mapper_obj.map(options=app.config.autoapi_options)
+
+ if app.config.autoapi_generate_api_docs:
+ LOGGER.info(bold("[AutoAPI] ") + darkgreen("Rendering Data"))
+ sphinx_mapper_obj.output_rst(root=normalized_root, source_suffix=out_suffix)
+
+
+# HACK: sphinx-auto map did not correctly use the confdir attribute instead of srcdir when specifying the
+# directory to contain the generated files.
+# Unfortunately we have a problem updating to a newer version of this library and we have to use
+# sphinx-autoapi v1.0.0, so I am monkeypatching this library to fix this one problem.
+autoapi.extension.run_autoapi = run_autoapi
+
+sys.exit(main(sys.argv[1:]))
diff --git a/docs/exts/docs_build/spelling_checks.py b/docs/exts/docs_build/spelling_checks.py
index 2be9cca..4d3c26d 100644
--- a/docs/exts/docs_build/spelling_checks.py
+++ b/docs/exts/docs_build/spelling_checks.py
@@ -23,7 +23,7 @@ from typing import Dict, List, NamedTuple, Optional
from rich.console import Console
from airflow.utils.code_utils import prepare_code_snippet
-from docs.exts.docs_build.code_utils import CONSOLE_WIDTH, remap_from_docker
+from docs.exts.docs_build.code_utils import CONSOLE_WIDTH
CURRENT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__)))
DOCS_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir, os.pardir))
@@ -80,7 +80,7 @@ class SpellingError(NamedTuple):
return left < right
-def parse_spelling_warnings(warning_text: str, docs_dir: str, dockerized: bool) -> List[SpellingError]:
+def parse_spelling_warnings(warning_text: str, docs_dir: str) -> List[SpellingError]:
"""
Parses warnings from Sphinx.
@@ -99,7 +99,7 @@ def parse_spelling_warnings(warning_text: str, docs_dir: str, dockerized: bool)
try:
sphinx_spelling_errors.append(
SpellingError(
- file_path=remap_from_docker(os.path.join(docs_dir, warning_parts[0]), dockerized),
+ file_path=os.path.join(docs_dir, warning_parts[0]),
line_no=int(warning_parts[1]) if warning_parts[1] not in ('None', '') else None,
spelling=warning_parts[2],
suggestion=warning_parts[3] if warning_parts[3] else None,
diff --git a/docs/exts/provider_init_hack.py b/docs/exts/provider_init_hack.py
index 0d88559..40f7fef 100644
--- a/docs/exts/provider_init_hack.py
+++ b/docs/exts/provider_init_hack.py
@@ -34,17 +34,12 @@ PROVIDER_INIT_FILE = os.path.join(ROOT_PROJECT_DIR, "airflow", "providers", "__i
def _create_init_py(app, config):
del app
del config
+ # This file is deleted by /docs/build_docs.py. If you are not using the script, the file will be
+ # deleted by pre-commit.
with open(PROVIDER_INIT_FILE, "wt"):
pass
-def _delete_init_py(app, exception):
- del app
- del exception
- if os.path.exists(PROVIDER_INIT_FILE):
- os.remove(PROVIDER_INIT_FILE)
-
-
def setup(app: Sphinx):
"""
Sets the plugin up and returns configuration of the plugin.
@@ -53,6 +48,5 @@ def setup(app: Sphinx):
:return json description of the configuration that is needed by the plugin.
"""
app.connect("config-inited", _create_init_py)
- app.connect("build-finished", _delete_init_py)
return {"version": "builtin", "parallel_read_safe": True, "parallel_write_safe": True}
diff --git a/scripts/ci/docs/ci_docs.sh b/scripts/ci/docs/ci_docs.sh
index 003a8c2..be0d2ed 100755
--- a/scripts/ci/docs/ci_docs.sh
+++ b/scripts/ci/docs/ci_docs.sh
@@ -22,17 +22,4 @@ build_images::prepare_ci_build
build_images::rebuild_ci_image_if_needed_with_group
-start_end::group_start "Preparing venv for doc building"
-
-python3 -m venv .docs-venv
-source .docs-venv/bin/activate
-export PYTHONPATH=${AIRFLOW_SOURCES}
-
-pip install --upgrade pip==20.2.4
-
-pip install .[doc] --upgrade --constraint \
- "https://raw.githubusercontent.com/apache/airflow/constraints-${DEFAULT_BRANCH}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"
-
-start_end::group_end
-
-"${AIRFLOW_SOURCES}/docs/build_docs.py" -j 0 "${@}"
+runs::run_docs "${@}"