You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by as...@apache.org on 2021/04/15 12:07:27 UTC
[airflow] 08/36: Less docker magic in docs building (#15176)
This is an automated email from the ASF dual-hosted git repository.
ash pushed a commit to branch v2-0-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit b61feb61233c2b2f11dfc48eb85ba3370d4f6b6d
Author: Kamil BreguĊa <mi...@users.noreply.github.com>
AuthorDate: Tue Apr 6 03:10:42 2021 +0200
Less docker magic in docs building (#15176)
(cherry picked from commit 3bd11631ff0fbff4859452513efe03674b04b141)
---
.github/workflows/ci.yml | 12 --
docs/build_docs.py | 91 +++------------
docs/conf.py | 2 +-
docs/exts/docs_build/code_utils.py | 16 +--
docs/exts/docs_build/docs_builder.py | 171 +++++++----------------------
docs/exts/docs_build/errors.py | 6 +-
docs/exts/docs_build/run_patched_sphinx.py | 105 ++++++++++++++++++
docs/exts/docs_build/spelling_checks.py | 6 +-
docs/exts/provider_init_hack.py | 10 +-
scripts/ci/docs/ci_docs.sh | 15 +--
10 files changed, 170 insertions(+), 264 deletions(-)
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index dc98f5c..86bc960 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -463,24 +463,12 @@ ${{ hashFiles('.pre-commit-config.yaml') }}"
env:
RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn) }}
GITHUB_REGISTRY: ${{ needs.ci-images.outputs.githubRegistry }}
- PYTHON_MAJOR_MINOR_VERSION: ${{needs.build-info.outputs.defaultPythonVersion}}
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v2
with:
persist-credentials: false
submodules: recursive
- - name: "Setup python"
- uses: actions/setup-python@v2
- with:
- python-version: ${{needs.build-info.outputs.defaultPythonVersion}}
- - uses: actions/cache@v2
- id: cache-venv-docs
- with:
- path: ./.docs-venv/
- key: ${{ runner.os }}-docs-venv-${{ hashFiles('setup.py', 'setup.cfg') }}
- restore-keys: |
- ${{ runner.os }}-docs-venv-
- name: "Free space"
run: ./scripts/ci/tools/ci_free_space_on_ci.sh
- name: "Prepare CI image ${{env.PYTHON_MAJOR_MINOR_VERSION}}:${{ env.GITHUB_REGISTRY_PULL_IMAGE_TAG }}"
diff --git a/docs/build_docs.py b/docs/build_docs.py
index 59e1681..5f1a534 100755
--- a/docs/build_docs.py
+++ b/docs/build_docs.py
@@ -18,23 +18,15 @@
import argparse
import multiprocessing
import os
-import platform
import sys
from collections import defaultdict
-from subprocess import run
from typing import Dict, List, NamedTuple, Optional, Tuple
from rich.console import Console
from tabulate import tabulate
from docs.exts.docs_build import dev_index_generator, lint_checks # pylint: disable=no-name-in-module
-from docs.exts.docs_build.code_utils import (
- CONSOLE_WIDTH,
- DOCKER_PROJECT_DIR,
- ROOT_PROJECT_DIR,
- TEXT_RED,
- TEXT_RESET,
-)
+from docs.exts.docs_build.code_utils import CONSOLE_WIDTH, PROVIDER_INIT_FILE, TEXT_RED, TEXT_RESET
from docs.exts.docs_build.docs_builder import ( # pylint: disable=no-name-in-module
DOCS_DIR,
AirflowDocsBuilder,
@@ -52,7 +44,7 @@ from docs.exts.docs_build.spelling_checks import ( # pylint: disable=no-name-in
display_spelling_error_summary,
)
-if __name__ != "__main__":
+if __name__ not in ("__main__", "__mp_main__"):
raise SystemExit(
"This file is intended to be executed as an executable program. You cannot use it as a module."
"To run this script, run the ./build_docs.py command"
@@ -131,27 +123,13 @@ def _get_parser():
"--jobs",
dest='jobs',
type=int,
- default=1,
+ default=0,
help=(
- """
- Number of parallel processes that will be spawned to build the docs.
-
- This is usually used in CI system only. Though you can also use it to run complete check
- of the documntation locally if you have powerful local machine.
- Default is 1 - which means that doc check runs sequentially, This is the default behaviour
- because autoapi extension we use is not capable of running parallel builds at the same time using
- the same source files.
-
- In parallel builds we are using dockerised version of image built from local sources but the image
- has to be prepared locally (similarly as it is in CI) before you run the docs build. Any changes you
- have done locally after building the image, will not be checked.
-
- Typically you run parallel build in this way if you want to quickly run complete check for all docs:
+ """\
+ Number of parallel processes that will be spawned to build the docs.
- ./breeze build-image --python 3.6
- ./docs/build-docs.py -j 0
-
-"""
+ If passed 0, the value will be determined based on the number of CPUs.
+ """
),
)
parser.add_argument(
@@ -174,7 +152,6 @@ class BuildSpecification(NamedTuple):
package_name: str
for_production: bool
verbose: bool
- dockerized: bool
class BuildDocsResult(NamedTuple):
@@ -202,7 +179,6 @@ def perform_docs_build_for_single_package(build_specification: BuildSpecificatio
result = BuildDocsResult(
package_name=build_specification.package_name,
errors=builder.build_sphinx_docs(
- dockerized=build_specification.dockerized,
verbose=build_specification.verbose,
),
log_file_name=builder.log_build_filename,
@@ -219,7 +195,6 @@ def perform_spell_check_for_single_package(build_specification: BuildSpecificati
result = SpellCheckResult(
package_name=build_specification.package_name,
errors=builder.check_spelling(
- dockerized=build_specification.dockerized,
verbose=build_specification.verbose,
),
log_file_name=builder.log_spelling_filename,
@@ -245,11 +220,6 @@ def build_docs_for_packages(
builder = AirflowDocsBuilder(package_name=package_name, for_production=for_production)
builder.clean_files()
if jobs > 1:
- if os.getenv('CI', '') == '':
- console.print("[yellow] PARALLEL DOCKERIZED EXECUTION REQUIRES IMAGE TO BE BUILD BEFORE !!!![/]")
- console.print("[yellow] Make sure that you've build the image before runnning docs build.[/]")
- console.print("[yellow] otherwise local changes you've done will not be used during the check[/]")
- console.print()
run_in_parallel(
all_build_errors,
all_spelling_errors,
@@ -289,7 +259,6 @@ def run_sequentially(
build_specification=BuildSpecification(
package_name=package_name,
for_production=for_production,
- dockerized=False,
verbose=verbose,
)
)
@@ -302,7 +271,6 @@ def run_sequentially(
build_specification=BuildSpecification(
package_name=package_name,
for_production=for_production,
- dockerized=False,
verbose=verbose,
)
)
@@ -323,15 +291,12 @@ def run_in_parallel(
):
"""Run both - spellcheck and docs build sequentially without multiprocessing"""
pool = multiprocessing.Pool(processes=jobs)
- # until we fix autoapi, we need to run parallel builds as dockerized images
- dockerized = True
if not spellcheck_only:
run_docs_build_in_parallel(
all_build_errors=all_build_errors,
for_production=for_production,
current_packages=current_packages,
verbose=verbose,
- dockerized=dockerized,
pool=pool,
)
if not docs_only:
@@ -340,34 +305,8 @@ def run_in_parallel(
for_production=for_production,
current_packages=current_packages,
verbose=verbose,
- dockerized=dockerized,
pool=pool,
)
- fix_ownership()
-
-
-def fix_ownership():
- """Fixes ownership for all files created with root user,"""
- console.print("Fixing ownership for generated files")
- python_version = os.getenv('PYTHON_MAJOR_MINOR_VERSION', "3.6")
- fix_cmd = [
- "docker",
- "run",
- "--entrypoint",
- "/bin/bash",
- "--rm",
- "-e",
- f"HOST_OS={platform.system()}",
- "-e" f"HOST_USER_ID={os.getuid()}",
- "-e",
- f"HOST_GROUP_ID={os.getgid()}",
- "-v",
- f"{ROOT_PROJECT_DIR}:{DOCKER_PROJECT_DIR}",
- f"apache/airflow:master-python{python_version}-ci",
- "-c",
- "/opt/airflow/scripts/in_container/run_fix_ownership.sh",
- ]
- run(fix_cmd, check=True)
def print_build_output(result: BuildDocsResult):
@@ -386,7 +325,6 @@ def run_docs_build_in_parallel(
for_production: bool,
current_packages: List[str],
verbose: bool,
- dockerized: bool,
pool,
):
"""Runs documentation building in parallel."""
@@ -399,7 +337,6 @@ def run_docs_build_in_parallel(
package_name=package_name,
for_production=for_production,
verbose=verbose,
- dockerized=dockerized,
)
)
with with_group("Running docs building"):
@@ -428,7 +365,6 @@ def run_spell_check_in_parallel(
for_production: bool,
current_packages: List[str],
verbose: bool,
- dockerized: bool,
pool,
):
"""Runs spell check in parallel."""
@@ -437,12 +373,7 @@ def run_spell_check_in_parallel(
for package_name in current_packages:
console.print(f"[blue]{package_name:60}:[/] Scheduling spellchecking")
spell_check_specifications.append(
- BuildSpecification(
- package_name=package_name,
- for_production=for_production,
- verbose=verbose,
- dockerized=dockerized,
- )
+ BuildSpecification(package_name=package_name, for_production=for_production, verbose=verbose)
)
with with_group("Running spell checking of documentation"):
console.print()
@@ -572,10 +503,14 @@ def main():
if not package_filters:
_promote_new_flags()
+ if os.path.exists(PROVIDER_INIT_FILE):
+ os.remove(PROVIDER_INIT_FILE)
+
print_build_errors_and_exit(
all_build_errors,
all_spelling_errors,
)
-main()
+if __name__ == "__main__":
+ main()
diff --git a/docs/conf.py b/docs/conf.py
index 678f053..11708f9 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -506,7 +506,7 @@ autoapi_keep_files = True
# Relative path to output the AutoAPI files into. This can also be used to place the generated documentation
# anywhere in your documentation hierarchy.
-autoapi_root = f'{PACKAGE_NAME}/_api'
+autoapi_root = '_api'
# Whether to insert the generated documentation into the TOC tree. If this is False, the default AutoAPI
# index page is not generated and you will need to include the generated documentation in a
diff --git a/docs/exts/docs_build/code_utils.py b/docs/exts/docs_build/code_utils.py
index 5c88797..adab5c2 100644
--- a/docs/exts/docs_build/code_utils.py
+++ b/docs/exts/docs_build/code_utils.py
@@ -22,12 +22,10 @@ from docs.exts.provider_yaml_utils import load_package_data
ROOT_PROJECT_DIR = os.path.abspath(
os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, os.pardir, os.pardir)
)
+PROVIDER_INIT_FILE = os.path.join(ROOT_PROJECT_DIR, "airflow", "providers", "__init__.py")
DOCS_DIR = os.path.join(ROOT_PROJECT_DIR, "docs")
AIRFLOW_DIR = os.path.join(ROOT_PROJECT_DIR, "airflow")
-DOCKER_PROJECT_DIR = "/opt/airflow"
-DOCKER_DOCS_DIR = os.path.join(DOCKER_PROJECT_DIR, "docs")
-DOCKER_AIRFLOW_DIR = os.path.join(DOCKER_PROJECT_DIR, "/airflow")
ALL_PROVIDER_YAMLS = load_package_data()
AIRFLOW_SITE_DIR = os.environ.get('AIRFLOW_SITE_DIRECTORY')
PROCESS_TIMEOUT = 8 * 60 # 400 seconds
@@ -38,18 +36,6 @@ TEXT_RESET = '\033[0m'
CONSOLE_WIDTH = 180
-def remap_from_docker(file_name: str, dockerized: bool):
- """
- Remaps filename from Docker to Host.
- :param file_name: name of file
- :param dockerized: whether builds were running in docker environment.
- :return:
- """
- if dockerized and file_name.startswith(DOCKER_PROJECT_DIR):
- return file_name.replace(DOCKER_PROJECT_DIR, ROOT_PROJECT_DIR)
- return file_name
-
-
def prepare_code_snippet(file_path: str, line_no: int, context_lines_count: int = 5) -> str:
"""
Prepares code snippet.
diff --git a/docs/exts/docs_build/docs_builder.py b/docs/exts/docs_build/docs_builder.py
index 0669c75..669d76d 100644
--- a/docs/exts/docs_build/docs_builder.py
+++ b/docs/exts/docs_build/docs_builder.py
@@ -28,9 +28,9 @@ from docs.exts.docs_build.code_utils import (
AIRFLOW_SITE_DIR,
ALL_PROVIDER_YAMLS,
CONSOLE_WIDTH,
- DOCKER_DOCS_DIR,
DOCS_DIR,
PROCESS_TIMEOUT,
+ ROOT_PROJECT_DIR,
pretty_format_path,
)
from docs.exts.docs_build.errors import DocBuildError, parse_sphinx_warnings
@@ -55,18 +55,10 @@ class AirflowDocsBuilder:
return f"{DOCS_DIR}/_doctrees/docs/{self.package_name}"
@property
- def _docker_doctree_dir(self) -> str:
- return f"{DOCKER_DOCS_DIR}/_doctrees/docs/{self.package_name}"
-
- @property
def _inventory_cache_dir(self) -> str:
return f"{DOCS_DIR}/_inventory_cache"
@property
- def _docker_inventory_cache_dir(self) -> str:
- return f"{DOCKER_DOCS_DIR}/_inventory_cache"
-
- @property
def is_versioned(self):
"""Is current documentation package versioned?"""
# Disable versioning. This documentation does not apply to any released product and we can update
@@ -87,49 +79,21 @@ class AirflowDocsBuilder:
return os.path.join(self._build_dir, f"output-spelling-{self.package_name}.log")
@property
- def docker_log_spelling_filename(self) -> str:
- """Log from spelling job in docker."""
- return os.path.join(self._docker_build_dir, f"output-spelling-{self.package_name}.log")
-
- @property
def log_spelling_output_dir(self) -> str:
"""Results from spelling job."""
return os.path.join(self._build_dir, f"output-spelling-results-{self.package_name}")
@property
- def docker_log_spelling_output_dir(self) -> str:
- """Results from spelling job in docker."""
- return os.path.join(self._docker_build_dir, f"output-spelling-results-{self.package_name}")
-
- @property
def log_build_filename(self) -> str:
"""Log from build job."""
return os.path.join(self._build_dir, f"output-build-{self.package_name}.log")
@property
- def docker_log_build_filename(self) -> str:
- """Log from build job in docker."""
- return os.path.join(self._docker_build_dir, f"output-build-{self.package_name}.log")
-
- @property
def log_build_warning_filename(self) -> str:
"""Warnings from build job."""
return os.path.join(self._build_dir, f"warning-build-{self.package_name}.log")
@property
- def docker_log_warning_filename(self) -> str:
- """Warnings from build job in docker."""
- return os.path.join(self._docker_build_dir, f"warning-build-{self.package_name}.log")
-
- @property
- def _docker_build_dir(self) -> str:
- if self.is_versioned:
- version = "stable" if self.for_production else "latest"
- return f"{DOCKER_DOCS_DIR}/_build/docs/{self.package_name}/{version}"
- else:
- return f"{DOCKER_DOCS_DIR}/_build/docs/{self.package_name}"
-
- @property
def _current_version(self):
if not self.is_versioned:
raise Exception("This documentation package is not versioned")
@@ -153,10 +117,6 @@ class AirflowDocsBuilder:
def _src_dir(self) -> str:
return f"{DOCS_DIR}/{self.package_name}"
- @property
- def _docker_src_dir(self) -> str:
- return f"{DOCKER_DOCS_DIR}/{self.package_name}"
-
def clean_files(self) -> None:
"""Cleanup all artifacts generated by previous builds."""
api_dir = os.path.join(self._src_dir, "_api")
@@ -166,58 +126,33 @@ class AirflowDocsBuilder:
os.makedirs(api_dir, exist_ok=True)
os.makedirs(self._build_dir, exist_ok=True)
- def check_spelling(self, verbose: bool, dockerized: bool) -> List[SpellingError]:
+ def check_spelling(self, verbose: bool) -> List[SpellingError]:
"""
Checks spelling
:param verbose: whether to show output while running
- :param dockerized: whether to run dockerized build (required for paralllel processing on CI)
:return: list of errors
"""
spelling_errors = []
os.makedirs(self._build_dir, exist_ok=True)
shutil.rmtree(self.log_spelling_output_dir, ignore_errors=True)
os.makedirs(self.log_spelling_output_dir, exist_ok=True)
- if dockerized:
- python_version = os.getenv('PYTHON_MAJOR_MINOR_VERSION', "3.6")
- build_cmd = [
- "docker",
- "run",
- "--rm",
- "-e",
- "AIRFLOW_FOR_PRODUCTION",
- "-e",
- "AIRFLOW_PACKAGE_NAME",
- "-v",
- f"{self._build_dir}:{self._docker_build_dir}",
- "-v",
- f"{self._inventory_cache_dir}:{self._docker_inventory_cache_dir}",
- "-w",
- DOCKER_DOCS_DIR,
- f"apache/airflow:master-python{python_version}-ci",
- "/opt/airflow/scripts/in_container/run_anything.sh",
- ]
- else:
- build_cmd = []
-
- build_cmd.extend(
- [
- "sphinx-build",
- "-W", # turn warnings into errors
- "--color", # do emit colored output
- "-T", # show full traceback on exception
- "-b", # builder to use
- "spelling",
- "-c",
- DOCS_DIR if not dockerized else DOCKER_DOCS_DIR,
- "-d", # path for the cached environment and doctree files
- self._doctree_dir if not dockerized else self._docker_doctree_dir,
- self._src_dir
- if not dockerized
- else self._docker_src_dir, # path to documentation source files
- self.log_spelling_output_dir if not dockerized else self.docker_log_spelling_output_dir,
- ]
- )
+
+ build_cmd = [
+ os.path.join(ROOT_PROJECT_DIR, "docs", "exts", "docs_build", "run_patched_sphinx.py"),
+ "-W", # turn warnings into errors
+ "--color", # do emit colored output
+ "-T", # show full traceback on exception
+ "-b", # builder to use
+ "spelling",
+ "-c",
+ DOCS_DIR,
+ "-d", # path for the cached environment and doctree files
+ self._doctree_dir,
+ self._src_dir, # path to documentation source files
+ self.log_spelling_output_dir,
+ ]
+
env = os.environ.copy()
env['AIRFLOW_PACKAGE_NAME'] = self.package_name
if self.for_production:
@@ -246,7 +181,7 @@ class AirflowDocsBuilder:
suggestion=None,
context_line=None,
message=(
- f"Sphinx spellcheck returned non-zero exit status: " f"{completed_proc.returncode}."
+ f"Sphinx spellcheck returned non-zero exit status: {completed_proc.returncode}."
),
)
)
@@ -254,69 +189,45 @@ class AirflowDocsBuilder:
for filepath in glob(f"{self.log_spelling_output_dir}/**/*.spelling", recursive=True):
with open(filepath) as spelling_file:
warning_text += spelling_file.read()
- spelling_errors.extend(parse_spelling_warnings(warning_text, self._src_dir, dockerized))
+
+ spelling_errors.extend(parse_spelling_warnings(warning_text, self._src_dir))
console.print(f"[blue]{self.package_name:60}:[/] [red]Finished spell-checking with errors[/]")
else:
if spelling_errors:
console.print(
- f"[blue]{self.package_name:60}:[/] [yellow]Finished spell-checking " f"with warnings[/]"
+ f"[blue]{self.package_name:60}:[/] [yellow]Finished spell-checking with warnings[/]"
)
else:
console.print(
- f"[blue]{self.package_name:60}:[/] [green]Finished spell-checking " f"successfully[/]"
+ f"[blue]{self.package_name:60}:[/] [green]Finished spell-checking successfully[/]"
)
return spelling_errors
- def build_sphinx_docs(self, verbose: bool, dockerized: bool) -> List[DocBuildError]:
+ def build_sphinx_docs(self, verbose: bool) -> List[DocBuildError]:
"""
Build Sphinx documentation.
:param verbose: whether to show output while running
- :param dockerized: whether to run dockerized build (required for paralllel processing on CI)
:return: list of errors
"""
build_errors = []
os.makedirs(self._build_dir, exist_ok=True)
- if dockerized:
- python_version = os.getenv('PYTHON_MAJOR_MINOR_VERSION', "3.6")
- build_cmd = [
- "docker",
- "run",
- "--rm",
- "-e",
- "AIRFLOW_FOR_PRODUCTION",
- "-e",
- "AIRFLOW_PACKAGE_NAME",
- "-v",
- f"{self._build_dir}:{self._docker_build_dir}",
- "-v",
- f"{self._inventory_cache_dir}:{self._docker_inventory_cache_dir}",
- "-w",
- DOCKER_DOCS_DIR,
- f"apache/airflow:master-python{python_version}-ci",
- "/opt/airflow/scripts/in_container/run_anything.sh",
- ]
- else:
- build_cmd = []
- build_cmd.extend(
- [
- "sphinx-build",
- "-T", # show full traceback on exception
- "--color", # do emit colored output
- "-b", # builder to use
- "html",
- "-d", # path for the cached environment and doctree files
- self._doctree_dir if not dockerized else self._docker_doctree_dir,
- "-c",
- DOCS_DIR if not dockerized else DOCKER_DOCS_DIR,
- "-w", # write warnings (and errors) to given file
- self.log_build_warning_filename if not dockerized else self.docker_log_warning_filename,
- self._src_dir
- if not dockerized
- else self._docker_src_dir, # path to documentation source files
- self._build_dir if not dockerized else self._docker_build_dir, # path to output directory
- ]
- )
+
+ build_cmd = [
+ os.path.join(ROOT_PROJECT_DIR, "docs", "exts", "docs_build", "run_patched_sphinx.py"),
+ "-T", # show full traceback on exception
+ "--color", # do emit colored output
+ "-b", # builder to use
+ "html",
+ "-d", # path for the cached environment and doctree files
+ self._doctree_dir,
+ "-c",
+ DOCS_DIR,
+ "-w", # write warnings (and errors) to given file
+ self.log_build_warning_filename,
+ self._src_dir,
+ self._build_dir, # path to output directory
+ ]
env = os.environ.copy()
env['AIRFLOW_PACKAGE_NAME'] = self.package_name
if self.for_production:
@@ -353,7 +264,7 @@ class AirflowDocsBuilder:
warning_text = warning_file.read()
# Remove 7-bit C1 ANSI escape sequences
warning_text = re.sub(r"\x1B[@-_][0-?]*[ -/]*[@-~]", "", warning_text)
- build_errors.extend(parse_sphinx_warnings(warning_text, self._src_dir, dockerized))
+ build_errors.extend(parse_sphinx_warnings(warning_text, self._src_dir))
if build_errors:
console.print(f"[blue]{self.package_name:60}:[/] [red]Finished docs building with errors[/]")
else:
diff --git a/docs/exts/docs_build/errors.py b/docs/exts/docs_build/errors.py
index 954262d..1a2ae06 100644
--- a/docs/exts/docs_build/errors.py
+++ b/docs/exts/docs_build/errors.py
@@ -21,7 +21,7 @@ from typing import Dict, List, NamedTuple, Optional
from rich.console import Console
from airflow.utils.code_utils import prepare_code_snippet
-from docs.exts.docs_build.code_utils import CONSOLE_WIDTH, remap_from_docker
+from docs.exts.docs_build.code_utils import CONSOLE_WIDTH
CURRENT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__)))
DOCS_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir, os.pardir))
@@ -82,7 +82,7 @@ def display_errors_summary(build_errors: Dict[str, List[DocBuildError]]) -> None
console.print()
-def parse_sphinx_warnings(warning_text: str, docs_dir: str, dockerized: bool) -> List[DocBuildError]:
+def parse_sphinx_warnings(warning_text: str, docs_dir: str) -> List[DocBuildError]:
"""
Parses warnings from Sphinx.
@@ -98,7 +98,7 @@ def parse_sphinx_warnings(warning_text: str, docs_dir: str, dockerized: bool) ->
try:
sphinx_build_errors.append(
DocBuildError(
- file_path=remap_from_docker(os.path.join(docs_dir, warning_parts[0]), dockerized),
+ file_path=os.path.join(docs_dir, warning_parts[0]),
line_no=int(warning_parts[1]),
message=warning_parts[2],
)
diff --git a/docs/exts/docs_build/run_patched_sphinx.py b/docs/exts/docs_build/run_patched_sphinx.py
new file mode 100755
index 0000000..887b982
--- /dev/null
+++ b/docs/exts/docs_build/run_patched_sphinx.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import sys
+
+import autoapi
+from autoapi.extension import (
+ LOGGER,
+ ExtensionError,
+ bold,
+ darkgreen,
+ default_backend_mapping,
+ default_file_mapping,
+ default_ignore_patterns,
+)
+from sphinx.cmd.build import main
+
+
+def run_autoapi(app):
+ """Load AutoAPI data from the filesystem."""
+ if not app.config.autoapi_dirs:
+ raise ExtensionError("You must configure an autoapi_dirs setting")
+
+ # Make sure the paths are full
+ normalized_dirs = []
+ autoapi_dirs = app.config.autoapi_dirs
+ if isinstance(autoapi_dirs, str):
+ autoapi_dirs = [autoapi_dirs]
+ for path in autoapi_dirs:
+ if os.path.isabs(path):
+ normalized_dirs.append(path)
+ else:
+ normalized_dirs.append(os.path.normpath(os.path.join(app.confdir, path)))
+
+ for _dir in normalized_dirs:
+ if not os.path.exists(_dir):
+ raise ExtensionError(
+ "AutoAPI Directory `{dir}` not found. "
+ "Please check your `autoapi_dirs` setting.".format(dir=_dir)
+ )
+
+ # Change from app.confdir to app.srcdir.
+ # Before:
+ # - normalized_root = os.path.normpath(
+ # - os.path.join(app.confdir, app.config.autoapi_root)
+ # -)
+ normalized_root = os.path.normpath(os.path.join(app.srcdir, app.config.autoapi_root))
+ url_root = os.path.join("/", app.config.autoapi_root)
+ sphinx_mapper = default_backend_mapping[app.config.autoapi_type]
+ sphinx_mapper_obj = sphinx_mapper(app, template_dir=app.config.autoapi_template_dir, url_root=url_root)
+ app.env.autoapi_mapper = sphinx_mapper_obj
+
+ if app.config.autoapi_file_patterns:
+ file_patterns = app.config.autoapi_file_patterns
+ else:
+ file_patterns = default_file_mapping.get(app.config.autoapi_type, [])
+
+ if app.config.autoapi_ignore:
+ ignore_patterns = app.config.autoapi_ignore
+ else:
+ ignore_patterns = default_ignore_patterns.get(app.config.autoapi_type, [])
+
+ if ".rst" in app.config.source_suffix:
+ out_suffix = ".rst"
+ elif ".txt" in app.config.source_suffix:
+ out_suffix = ".txt"
+ else:
+ # Fallback to first suffix listed
+ out_suffix = app.config.source_suffix[0]
+
+ # Actual meat of the run.
+ LOGGER.info(bold("[AutoAPI] ") + darkgreen("Loading Data"))
+ sphinx_mapper_obj.load(patterns=file_patterns, dirs=normalized_dirs, ignore=ignore_patterns)
+
+ LOGGER.info(bold("[AutoAPI] ") + darkgreen("Mapping Data"))
+ sphinx_mapper_obj.map(options=app.config.autoapi_options)
+
+ if app.config.autoapi_generate_api_docs:
+ LOGGER.info(bold("[AutoAPI] ") + darkgreen("Rendering Data"))
+ sphinx_mapper_obj.output_rst(root=normalized_root, source_suffix=out_suffix)
+
+
+# HACK: sphinx-auto map did not correctly use the confdir attribute instead of srcdir when specifying the
+# directory to contain the generated files.
+# Unfortunately we have a problem updating to a newer version of this library and we have to use
+# sphinx-autoapi v1.0.0, so I am monkeypatching this library to fix this one problem.
+autoapi.extension.run_autoapi = run_autoapi
+
+sys.exit(main(sys.argv[1:]))
diff --git a/docs/exts/docs_build/spelling_checks.py b/docs/exts/docs_build/spelling_checks.py
index 2be9cca..4d3c26d 100644
--- a/docs/exts/docs_build/spelling_checks.py
+++ b/docs/exts/docs_build/spelling_checks.py
@@ -23,7 +23,7 @@ from typing import Dict, List, NamedTuple, Optional
from rich.console import Console
from airflow.utils.code_utils import prepare_code_snippet
-from docs.exts.docs_build.code_utils import CONSOLE_WIDTH, remap_from_docker
+from docs.exts.docs_build.code_utils import CONSOLE_WIDTH
CURRENT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__)))
DOCS_DIR = os.path.abspath(os.path.join(CURRENT_DIR, os.pardir, os.pardir))
@@ -80,7 +80,7 @@ class SpellingError(NamedTuple):
return left < right
-def parse_spelling_warnings(warning_text: str, docs_dir: str, dockerized: bool) -> List[SpellingError]:
+def parse_spelling_warnings(warning_text: str, docs_dir: str) -> List[SpellingError]:
"""
Parses warnings from Sphinx.
@@ -99,7 +99,7 @@ def parse_spelling_warnings(warning_text: str, docs_dir: str, dockerized: bool)
try:
sphinx_spelling_errors.append(
SpellingError(
- file_path=remap_from_docker(os.path.join(docs_dir, warning_parts[0]), dockerized),
+ file_path=os.path.join(docs_dir, warning_parts[0]),
line_no=int(warning_parts[1]) if warning_parts[1] not in ('None', '') else None,
spelling=warning_parts[2],
suggestion=warning_parts[3] if warning_parts[3] else None,
diff --git a/docs/exts/provider_init_hack.py b/docs/exts/provider_init_hack.py
index 0d88559..40f7fef 100644
--- a/docs/exts/provider_init_hack.py
+++ b/docs/exts/provider_init_hack.py
@@ -34,17 +34,12 @@ PROVIDER_INIT_FILE = os.path.join(ROOT_PROJECT_DIR, "airflow", "providers", "__i
def _create_init_py(app, config):
del app
del config
+ # This file is deleted by /docs/build_docs.py. If you are not using the script, the file will be
+ # deleted by pre-commit.
with open(PROVIDER_INIT_FILE, "wt"):
pass
-def _delete_init_py(app, exception):
- del app
- del exception
- if os.path.exists(PROVIDER_INIT_FILE):
- os.remove(PROVIDER_INIT_FILE)
-
-
def setup(app: Sphinx):
"""
Sets the plugin up and returns configuration of the plugin.
@@ -53,6 +48,5 @@ def setup(app: Sphinx):
:return json description of the configuration that is needed by the plugin.
"""
app.connect("config-inited", _create_init_py)
- app.connect("build-finished", _delete_init_py)
return {"version": "builtin", "parallel_read_safe": True, "parallel_write_safe": True}
diff --git a/scripts/ci/docs/ci_docs.sh b/scripts/ci/docs/ci_docs.sh
index 003a8c2..be0d2ed 100755
--- a/scripts/ci/docs/ci_docs.sh
+++ b/scripts/ci/docs/ci_docs.sh
@@ -22,17 +22,4 @@ build_images::prepare_ci_build
build_images::rebuild_ci_image_if_needed_with_group
-start_end::group_start "Preparing venv for doc building"
-
-python3 -m venv .docs-venv
-source .docs-venv/bin/activate
-export PYTHONPATH=${AIRFLOW_SOURCES}
-
-pip install --upgrade pip==20.2.4
-
-pip install .[doc] --upgrade --constraint \
- "https://raw.githubusercontent.com/apache/airflow/constraints-${DEFAULT_BRANCH}/constraints-${PYTHON_MAJOR_MINOR_VERSION}.txt"
-
-start_end::group_end
-
-"${AIRFLOW_SOURCES}/docs/build_docs.py" -j 0 "${@}"
+runs::run_docs "${@}"