You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2025/05/16 03:51:59 UTC
(airflow-site-archive) branch consistency-changes-for-syncing-scripts updated (293da9fbd5 -> 54521b92f8)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to branch consistency-changes-for-syncing-scripts
in repository https://gitbox.apache.org/repos/asf/airflow-site-archive.git
discard 293da9fbd5 Add consistency changes for the syncing scripts and workflows
new 54521b92f8 Add consistency changes for the syncing scripts and workflows
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (293da9fbd5)
\
N -- N -- N refs/heads/consistency-changes-for-syncing-scripts (54521b92f8)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.github/workflows/github-to-s3.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
(airflow-site-archive) 01/01: Add consistency changes for the syncing scripts and workflows
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch consistency-changes-for-syncing-scripts
in repository https://gitbox.apache.org/repos/asf/airflow-site-archive.git
commit 54521b92f874fea180e299c48e0bd4dc705b74e3
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Thu May 15 22:44:25 2025 -0400
Add consistency changes for the syncing scripts and workflows
* folders -> packages (which is consistent with breeze naming)
* more logical parameters for GH -> S3 sync (full sync boolean,
better sequence of parameters)
* commit-sha -> commit-ref (because it might be not only sha)
* consistent plural in document_packages
* updated README with added information about workflows and
--help produced by runing --help
---
.github/workflows/github-to-s3.yml | 114 +++++++++++++++++++------------------
.github/workflows/s3-to-github.yml | 54 +++++++++---------
README.md | 78 ++++++++++++++++---------
scripts/github_to_s3.py | 83 ++++++++++++++-------------
scripts/s3_to_github.py | 36 ++++++------
scripts/transfer_utils.py | 42 +++++++-------
6 files changed, 219 insertions(+), 188 deletions(-)
diff --git a/.github/workflows/github-to-s3.yml b/.github/workflows/github-to-s3.yml
index e02de60a30..17ea8cd229 100644
--- a/.github/workflows/github-to-s3.yml
+++ b/.github/workflows/github-to-s3.yml
@@ -28,23 +28,20 @@ on: # yamllint disable-line rule:truthy
- live
- staging
default: live
- sync-type:
- description: "Perform a full sync or just sync the last commit"
+ document-packages:
+ description: "Packages (long or short) separated with spaces"
required: false
- default: "single_commit"
- type: choice
- options:
- - single_commit
- - full_sync
- commit-sha:
- description: "For single-commit - SHA/tag/branch (default: latest in selected branch)"
- required: false
- default: ""
+ default: "all"
type: string
- document-folders:
- description: "For full-sync, you can select which packages to upload - space separated"
+ full-sync:
+ description: "If specified, whole repo will be synced (not only single commit)."
required: false
- default: "all"
+ default: false
+ type: boolean
+ commit-reference:
+ description: "Commit used to sync if np full-sync (default HEAD of branch chosen)."
+ required: false
+ default: "HEAD"
type: string
processes:
description: "Number of processes to use for syncing"
@@ -59,24 +56,31 @@ jobs:
- name: Summarize parameters
id: parameters
env:
- DOCUMENT_FOLDERS: ${{ inputs.document-folders }}
- SYNC_TYPE: ${{ inputs.sync-type }}
- COMMIT_SHA: ${{ inputs.commit-sha }}
+ DOCUMENT_PACKAGES: ${{ inputs.document-packages }}
PROCESSES: ${{ inputs.processes }}
DESTINATION: ${{ inputs.destination }}}
+ COMMIT_REFERENCE: ${{ inputs.commit-reference }}
+ FULL_SYNC: ${{ inputs.full-sync }}
run: |
echo "Input parameters summary"
echo "========================="
- echo "Document folders: ${DOCUMENT_FOLDER}"
- echo "Sync type: ${SYNC_TYPE}"
- echo "Commit SHA: ${COMMIT_SHA}"
- echo "Processes: ${PROCESSES}"
echo "Destination: ${DESTINATION}"
+ echo "Document packages: ${DOCUMENT_PACKAGES}"
+ echo "Full sync: ${FULL_SYNC}"
+ echo "Commit reference: ${COMMIT_REFERENCE}"
+ echo "Processes: ${PROCESSES}"
+ set -x
if [[ "${DESTINATION}" == "live" ]]; then
echo "destination-location=s3://live-docs-airflow-apache-org/docs/" >> ${GITHUB_OUTPUT}
else
echo "destination-location=s3://staging-docs-airflow-apache-org/docs/" >> ${GITHUB_OUTPUT}
fi
+ if [[ "${FULL_SYNC}" == "true" ]] ; then
+ echo "sync-type=full-sync" >> ${GITHUB_OUTPUT}
+ else
+ echo "sync-type=single-commit" >> ${GITHUB_OUTPUT}
+ fi
+ echo "commit-ref=${{ inputs.full-sync && '' || env.COMMIT_REFERENCE }}" >> ${GITHUB_OUTPUT}
- uses: actions/checkout@v4
# Checkout only workflow and scripts directory to run scripts from
@@ -121,53 +125,53 @@ jobs:
sudo chown -R "${USER}" /mnt/cloned-airflow-site-archive
ln -v -s /mnt/cloned-airflow-site-archive ./cloned-airflow-site-archive
- - name: Pre-process docs folders
+ - name: Pre-process docs packages
env:
- DOCUMENTS_FOLDERS: ${{ inputs.document-folders }}
- id: docs-folders-processed
+ DOCUMENTS_PACKAGES: ${{ inputs.document-packages }}
+ id: docs-packages-processed
run: |
echo "sparse-checkout<<EOF" >> ${GITHUB_OUTPUT}
- if [[ "${DOCUMENTS_FOLDERS}" != "all" ]]; then
- echo "Preprocessing docs folders: ${DOCUMENTS_FOLDERS}"
- folders=""
+ if [[ "${DOCUMENTS_PACKAGES}" != "all" ]]; then
+ echo "Preprocessing docs packages: ${DOCUMENTS_PACKAGES}"
+ packages=""
sparse_checkout=""
separator=""
- for folder in ${DOCUMENTS_FOLDERS}; do
- if [[ "${folder}" != apache-airflow-providers* ]]; then
- folders="${folders}${separator}apache-airflow-providers-${folder/./-}"
- echo "docs-archive/apache-airflow-providers-${folder/./-}" >> ${GITHUB_OUTPUT}
+ for package in ${DOCUMENTS_PACKAGES}; do
+ if [[ "${package}" != apache-airflow-providers* ]]; then
+ packages="${packages}${separator}apache-airflow-providers-${package/./-}"
+ echo "docs-archive/apache-airflow-providers-${package/./-}" >> ${GITHUB_OUTPUT}
else
- folders="${folders}${separator}${folder}"
- echo "docs-archive/${folder}" >> ${GITHUB_OUTPUT}
+ packages="${packages}${separator}${package}"
+ echo "docs-archive/${package}" >> ${GITHUB_OUTPUT}
fi
separator=" "
done
else
- folders="all"
+ packages="all"
echo "docs-archive" >> ${GITHUB_OUTPUT}
fi
echo "EOF" >> ${GITHUB_OUTPUT}
- echo "docs-folders-processed=${folders}"
- echo "docs-folders-processed=${folders}" >> ${GITHUB_OUTPUT}
+ set -x
+ echo "docs-packages-processed=${packages}" >> ${GITHUB_OUTPUT}
- name: >
- Checkout (${{ inputs.commit-sha || github.sha }}) to /mnt/cloned-airflow-site-archive
- with docs: ${{ steps.docs-folders-processed.outputs.docs-folders-processed }}
+ Checkout (${{ steps.parameters.outputs.commit-ref }}) to /mnt/cloned-airflow-site-archive
+ with docs: ${{ steps.docs-packages-processed.outputs.docs-packages-processed }}
uses: actions/checkout@v4
with:
path: ./cloned-airflow-site-archive
fetch-depth: 2
sparse-checkout: |
- ${{ steps.docs-folders-processed.outputs.sparse-checkout }}
- ref: ${{ inputs.commit-sha || github.sha }}
- if: steps.docs-folders-processed.outputs.docs-folders-processed != 'all'
+ ${{ steps.docs-packages-processed.outputs.sparse-checkout }}
+ ref: ${{ steps.parameters.outputs.commit-ref }}
+ if: steps.docs-packages-processed.outputs.docs-packages-processed != 'all'
- name: >
- Checkout (${{ inputs.commit-sha || github.sha }}) to /mnt/cloned-airflow-site-archive (whole repo)
+ Checkout (${{ steps.parameters.outputs.commit-ref }}) to /mnt/cloned-airflow-site-archive (whole repo)
uses: actions/checkout@v4
with:
path: ./cloned-airflow-site-archive
fetch-depth: 2
- ref: ${{ inputs.commit-sha || github.sha }}
- if: steps.docs-folders-processed.outputs.docs-folders-processed == 'all'
+ ref: ${{ steps.parameters.outputs.commit-ref }}
+ if: steps.docs-packages-processed.outputs.docs-packages-processed == 'all'
- name: "Install uv"
run: curl -LsSf https://astral.sh/uv/install.sh | sh
- name: Add staging watermarks
@@ -177,24 +181,24 @@ jobs:
chmod a+x add_watermark.py
mkdir -p images
curl -sSf -o images/staging.png https://raw.githubusercontent.com/apache/airflow-site/refs/heads/main/.github/scripts/images/staging.png
- uv run add_watermark.py --pattern 'main.min.*.css' --folder docs-archive \
+ uv run add_watermark.py --pattern 'main.min.*.css' --package docs-archive \
--image-directory images --url-prefix /docs/images
if: inputs.destination == 'staging'
- name: >
- Syncing ${{ inputs.commit-sha || github.sha }}: ${{ inputs.destination }}:
- ${{ inputs.sync-type }} ${{ steps.docs-folders-processed.outputs.docs-folders-processed }}
- wih parallel aws cli methods = ${{ inputs.processes }}
+ Syncing ${{ steps.parameters.outputs.commit-ref }}: ${{ inputs.destination }}:
+ ${{ steps.parameters.outputs.sync-type }} ${{ steps.docs-packages-processed.outputs.docs-packages-processed }}
+ wih parallel aws cli = ${{ inputs.processes }}
env:
- COMMIT_SHA: ${{ inputs.commit-sha || github.sha }}
- SYNC_TYPE: ${{ inputs.sync-type }}
- PROCESSES: ${{ inputs.processes }}
- DOCUMENTS_FOLDERS: ${{ steps.docs-folders-processed.outputs.docs-folders-processed }}
+ COMMIT_REF: ${{ steps.parameters.outputs.commit-ref }}
+ SYNC_TYPE: ${{ steps.parameters.outputs.sync-type }}
+ DOCUMENTS_PACKAGES: ${{ steps.docs-packages-processed.outputs.docs-packages-processed }}
DESTINATION_LOCATION: ${{ steps.parameters.outputs.destination-location }}
+ PROCESSES: ${{ inputs.processes }}
run: |
# show what's being run
set -x
if [[ "${SYNC_TYPE}" == "single_commit" ]]; then
- echo "Syncing ${COMMIT_SHA}"
+ echo "Syncing ${COMMIT_REF}"
else
echo "Syncing whole repo"
fi
@@ -205,6 +209,6 @@ jobs:
uv run ./scripts/github_to_s3.py \
--bucket-path ${DESTINATION_LOCATION} \
--local-path /mnt/cloned-airflow-site-archive/docs-archive \
- --document-folders "${DOCUMENTS_FOLDERS}" \
- --commit-sha ${COMMIT_SHA} --sync-type ${SYNC_TYPE} \
+ --document-packages "${DOCUMENTS_PACKAGES}" \
+ --commit-ref ${COMMIT_REF} --sync-type ${SYNC_TYPE} \
--processes ${PROCESSES}
diff --git a/.github/workflows/s3-to-github.yml b/.github/workflows/s3-to-github.yml
index d4650ff7cf..db3c8f6017 100644
--- a/.github/workflows/s3-to-github.yml
+++ b/.github/workflows/s3-to-github.yml
@@ -28,8 +28,8 @@ on: # yamllint disable-line rule:truthy
- live
- staging
default: live
- document-folders:
- description: "Document folders to sync or short package ids (separated with spaces)"
+ document-packages:
+ description: "Document packages - long or short seprated by spaces"
required: false
default: "all"
type: string
@@ -49,7 +49,7 @@ jobs:
runs-on: ubuntu-latest
env:
SOURCE: ${{ inputs.source }}
- DOCUMENT_FOLDERS: ${{ inputs.document-folders }}
+ DOCUMENT_PACKAGES: ${{ inputs.document-packages }}
COMMIT_CHANGES: ${{ inputs.commit-changes }}
PROCESSES: ${{ inputs.processes }}
steps:
@@ -59,7 +59,7 @@ jobs:
echo "Input parameters summary"
echo "========================="
echo "Source: ${SOURCE}"
- echo "Document folders: ${DOCUMENT_FOLDERS}"
+ echo "Document packages: ${DOCUMENT_PACKAGES}"
echo "Commit changes: ${COMMIT_CHANGES}"
echo "Processes: ${PROCESSES}"
if [[ "${SOURCE}" == "live" ]]; then
@@ -86,37 +86,37 @@ jobs:
aws-secret-access-key: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-2
- - name: Pre-process docs folders
+ - name: Pre-process docs packages
env:
- DOCUMENTS_FOLDERS: ${{ inputs.document-folders }}
- id: docs-folders-processed
+ DOCUMENT_PACKAGES: ${{ inputs.document-packages }}
+ id: docs-packages-processed
run: |
echo "sparse-checkout<<EOF" >> ${GITHUB_OUTPUT}
echo ".github" >> ${GITHUB_OUTPUT}
echo "scripts" >> ${GITHUB_OUTPUT}
- if [[ "${DOCUMENTS_FOLDERS}" != "all" ]]; then
- echo "Preprocessing docs folders: ${DOCUMENTS_FOLDERS}"
- folders=""
+ if [[ "${DOCUMENT_PACKAGES}" != "all" ]]; then
+ echo "Preprocessing docs packages: ${DOCUMENT_PACKAGES}"
+ packages=""
sparse_checkout=""
separator=""
- for folder in ${DOCUMENTS_FOLDERS}; do
- if [[ "${folder}" != apache-airflow-providers* && "${folder}" != "apache-airflow" \
- && "${folder}" != "docker-stack" && "${folder}" != "helm-chart" ]]; then
- folders="${folders}${separator}apache-airflow-providers-${folder/./-}"
- echo "docs-archive/apache-airflow-providers-${folder/./-}" >> ${GITHUB_OUTPUT}
+ for package in ${DOCUMENT_PACKAGES}; do
+ if [[ "${package}" != apache-airflow-providers* && "${package}" != "apache-airflow" \
+ && "${package}" != "docker-stack" && "${package}" != "helm-chart" ]]; then
+ packages="${packages}${separator}apache-airflow-providers-${package/./-}"
+ echo "docs-archive/apache-airflow-providers-${package/./-}" >> ${GITHUB_OUTPUT}
else
- folders="${folders}${separator}${folder}"
- echo "docs-archive/${folder}" >> ${GITHUB_OUTPUT}
+ packages="${packages}${separator}${package}"
+ echo "docs-archive/${package}" >> ${GITHUB_OUTPUT}
fi
separator=" "
done
else
- folders="all"
+ packages="all"
echo "docs-archive" >> ${GITHUB_OUTPUT}
fi
echo "EOF" >> ${GITHUB_OUTPUT}
- echo "docs-folders-processed=${folders}"
- echo "docs-folders-processed=${folders}" >> ${GITHUB_OUTPUT}
+ echo "docs-packages-processed=${packages}"
+ echo "docs-packages-processed=${packages}" >> ${GITHUB_OUTPUT}
- name: Create /mnt/cloned-airflow-site-archive directory
run: |
@@ -125,14 +125,14 @@ jobs:
- name: >
Checkout (${{ inputs.commit-sha || github.sha }}) to /mnt/cloned-airflow-site-archive
- with docs: ${{ steps.docs-folders-processed.outputs.docs-folders-processed }}
+ with docs: ${{ steps.docs-packages-processed.outputs.docs-packages-processed }}
uses: actions/checkout@v4
with:
path: ./cloned-airflow-site-archive
fetch-depth: 1
sparse-checkout: |
- ${{ steps.docs-folders-processed.outputs.sparse-checkout }}
- if: steps.docs-folders-processed.outputs.docs-folders-processed != 'all'
+ ${{ steps.docs-packages-processed.outputs.sparse-checkout }}
+ if: steps.docs-packages-processed.outputs.docs-packages-processed != 'all'
- name: >
Checkout (${{ inputs.commit-sha || github.sha }}) to /mnt/cloned-airflow-site-archive (whole repo)
@@ -140,15 +140,15 @@ jobs:
with:
path: ./cloned-airflow-site-archive
fetch-depth: 1
- if: steps.docs-folders-processed.outputs.docs-folders-processed == 'all'
+ if: steps.docs-packages-processed.outputs.docs-packages-processed == 'all'
- name: "Check space available"
run: df -h
- - name: Syncing ${{ inputs.source }} (${{ inputs.document-folders }})
+ - name: Syncing ${{ inputs.source }} (${{ inputs.document-packages }})
env:
PROCESSES: ${{ inputs.processes }}
- DOCUMENTS_FOLDERS: ${{ inputs.document-folders }}
+ DOCUMENT_PACKAGES: ${{ inputs.document-packages }}
SOURCE_LOCATION: ${{ steps.parameters.outputs.source-location }}
run: |
set -x
@@ -156,7 +156,7 @@ jobs:
uv run ./scripts/s3_to_github.py \
--bucket-path ${SOURCE_LOCATION} \
--local-path ./docs-archive \
- --document-folders "${DOCUMENT_FOLDERS}" \
+ --document-packages "${DOCUMENT_PACKAGES}" \
--processes "${PROCESSES}"
working-directory: /mnt/cloned-airflow-site-archive
diff --git a/README.md b/README.md
index 25526b0a26..7f78f7e6df 100644
--- a/README.md
+++ b/README.md
@@ -1,32 +1,58 @@
-# airflow-site-archive
-
-### Documentation Syncing Process
-### S3 To GitHub
-**Sync S3 to Github**: Use the `scripts/s3_to_github.py` script to download the latest documentation from S3 to your ./docs-archive folder.
-It has the following command line arguments:
-- `--bucket-path`: The S3 bucket path where the documentation is stored.
-- `--local-path`: The local path where the documentation will be downloaded.
-- `--document-folder`: The folder in the S3 bucket where the documentation is stored (This is optional if any particular
- folder need to be synced, provide the folder name ex: `apache-airflow-providers-amazon`).
-```bash
-uv run ./scripts/s3_to_github.py --bucket-path s3://staging-docs-airflow-apache-org/docs/ --local-path ./docs-archive
+# Airflow sync archive
+
+The repository stores the archive of generated documentation from Apache Airflow.
+
+The scripts and workflows here allow to keep the repository in sync with the S3 buckets - both live and
+sync - wehre the documentation is stored. Sync in both direction is possible.
+
+In the future we will automate synchronization of the repoitory after any change to the buckets, currently
+manual synchronization S3 -> Bucket for the `live` ucket documentation is done using the `S3 to GitHub workflow`
+that subsequently uses `s3-to-github.py`, and syncing the repository to the `staging` bucket is done
+using the `GitHub to S3 workflow` that uses `github-to-s3.py` script. The scripts can also be used to
+perform manual syncs of changes when we modify the documentation in the repository and want to
+sync it to either of the S3 buckets.
+
+You can see the arguments for the scripts in the `s3-to-github.py` and `github-to-s3.py` by passing `--help`
+options:
+
+* ``uv run scripts/s3_to_github.py --help``:
+
```
+usage: s3_to_github.py [-h] --bucket-path BUCKET_PATH --local-path LOCAL_PATH [--document-packages DOCUMENT_PACKAGES] [--processes PROCESSES]
+
+Sync S3 to GitHub
+options:
+ -h, --help show this help message and exit
+ --bucket-path BUCKET_PATH
+ S3 bucket name with path
+ --local-path LOCAL_PATH
+ local path to sync
+ --document-packages DOCUMENT_PACKAGES
+ Document packages to sync
+ --processes PROCESSES
+ Number of processes
+```
-### GitHub To S3
-**Sync Github to S3**: Use the `scripts/github_to_s3.py` script to upload the latest documentation from your ./docs-archive folder to S3.
-It has two modes:
-1. **Last commit**: Syncs only last commit changes to S3.
-2. **Full sync**: Syncs all files under `./docs-archive` to S3.
-It has the following command line arguments:
+* ``uv run scripts/github_to_s3.py --help``:
+
+```
+usage: github_to_s3.py [-h] --bucket-path BUCKET_PATH --local-path LOCAL_PATH [--document-packages DOCUMENT_PACKAGES] [--commit-ref COMMIT_REF] [--sync-type {full-sync,single-commit}] [--processes PROCESSES]
-- `--bucket-path`: The S3 bucket path where the documentation will be stored.
-- `--local-path`: The local path where the documentation is stored.
-- `--document-folder`: The folder in the local path where the documentation is stored (This is optional if any particular
- folder need to be synced, provide the folder name ex: `apache-airflow-providers-amazon`).
-- `--sync-type`: The type of sync to perform. Can be either `last_commit` or `full_sync`.
-- `--commit-sha`: The commit sha to sync to S3. This is only required if the sync type is `last_commit`.
+Sync GitHub to S3
-```bash
-uv run ./scripts/github_to_s3.py --bucket-path s3://staging-docs-airflow-apache-org/docs/ --local-path ./docs-archive --sync-type last-commit
+options:
+ -h, --help show this help message and exit
+ --bucket-path BUCKET_PATH
+ S3 bucket name with path
+ --local-path LOCAL_PATH
+ local path to sync
+ --document-packages DOCUMENT_PACKAGES
+ Document package ids to sync (long or short) separated with spaces ('all' means all packages)
+ --commit-ref COMMIT_REF
+ Commit ref to sync (sha/HEAD/branch)
+ --sync-type {full-sync,single-commit}
+ Sync type
+ --processes PROCESSES
+ Number of processes
```
diff --git a/scripts/github_to_s3.py b/scripts/github_to_s3.py
index 4b78226d54..6ae9fc31ca 100644
--- a/scripts/github_to_s3.py
+++ b/scripts/github_to_s3.py
@@ -31,7 +31,7 @@ from pathlib import Path
from rich.console import Console
-from transfer_utils import CommonTransferUtils, convert_short_name_to_folder_name, sort_priority_folders
+from transfer_utils import CommonTransferUtils, convert_short_name_to_full_package_name, sort_priority_packages
console = Console(width=200, color_system="standard")
@@ -40,16 +40,16 @@ class GithubToS3(CommonTransferUtils):
super().__init__(bucket, local_path)
@staticmethod
- def fetch_commit_files(commit_sha: str, diff_filter: str="ACM"):
- console.print(f"[blue] Fetching files from last commit {commit_sha} [/]")
+ def fetch_commit_files(commit_ref: str, diff_filter: str="ACM"):
+ console.print(f"[blue] Fetching files from last commit {commit_ref} [/]")
cmd = [
"git",
"diff-tree",
"--no-commit-id",
"--name-only",
"-r",
- commit_sha + "^",
- commit_sha,
+ commit_ref + "^",
+ commit_ref,
f"--diff-filter={diff_filter}"
]
result = subprocess.run(cmd, check=False, capture_output=True, text=True)
@@ -61,17 +61,17 @@ class GithubToS3(CommonTransferUtils):
sys.exit(1)
return result.stdout.splitlines() if result.stdout else []
- def sync_single_commit_files(self, commit_sha: str, processes: int):
+ def sync_single_commit_files(self, commit_ref: str, processes: int):
'''
There are two parts here.
- 1. When any file gets removed under docs folder, we will remove from target location
- 2. When any file gets added/changed/modified under docs folder, we will copy from source to target location
+ 1. When any file gets removed under docs package, we will remove from target location
+ 2. When any file gets added/changed/modified under docs package, we will copy from source to target location
'''
# Fetching `d` excludes deleted files
# Fetching `D` includes deleted files
- files_cp_required = self.fetch_commit_files(commit_sha, diff_filter="d")
- files_del_required = self.fetch_commit_files(commit_sha, diff_filter="D")
+ files_cp_required = self.fetch_commit_files(commit_ref, diff_filter="d")
+ files_del_required = self.fetch_commit_files(commit_ref, diff_filter="D")
files_cp_required_under_docs = [f for f in files_cp_required if f.startswith("docs-archive/")]
files_del_required_required_under_docs = [f for f in files_del_required if f.startswith("docs-archive/")]
@@ -91,16 +91,16 @@ class GithubToS3(CommonTransferUtils):
self.run_with_pool(self.remove, delete_files_pool_args, processes=processes)
self.run_with_pool(self.copy, copy_files_pool_args, processes=processes)
- def full_sync(self, processes: int, folders: list[str] | None = None):
- if folders:
- console.print(f"[blue] Syncing folders {folders} from {self.local_path} to {self.bucket_name} [/]")
+ def full_sync(self, processes: int, packages: list[str] | None = None):
+ if packages:
+ console.print(f"[blue] Syncing packages {packages} from {self.local_path} to {self.bucket_name} [/]")
else:
console.print(f"[blue] Syncing all files from {self.local_path} to {self.bucket_name} [/]")
- list_of_folders = os.listdir(self.local_path) if not folders else folders
+ list_of_packages = os.listdir(self.local_path) if not packages else packages
pool_args = []
- for folder in sort_priority_folders(list_of_folders):
- source = os.path.join(self.local_path, folder)
- dest = f"s3://{self.bucket_name}/{self.prefix}".rstrip("/") + "/" + folder
+ for package in sort_priority_packages(list_of_packages):
+ source = os.path.join(self.local_path, package)
+ dest = f"s3://{self.bucket_name}/{self.prefix}".rstrip("/") + "/" + package
pool_args.append((source, dest))
self.run_with_pool(self.sync, pool_args, processes=processes)
@@ -111,11 +111,12 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Sync GitHub to S3")
parser.add_argument("--bucket-path", required=True, help="S3 bucket name with path")
parser.add_argument("--local-path", required=True, help="local path to sync")
- parser.add_argument("--document-folders", help="Document folders to sync "
- "(or short provider-ids) separated with spaces "
- "('all' means all folders)", default="")
- parser.add_argument("--commit-sha", help="Commit SHA to sync", default="")
- parser.add_argument("--sync-type", help="Sync type", default="single_commit")
+ parser.add_argument("--document-packages", help="Document package ids to sync "
+ "(long or short) separated with spaces "
+ "('all' means all packages)", default="")
+ parser.add_argument("--commit-ref", help="Commit ref to sync (sha/HEAD/branch)", default="")
+ parser.add_argument("--sync-type", help="Sync type", choices=('full-sync', 'single-commit'),
+ default="single-commit")
parser.add_argument("--processes", help="Number of processes", type=int, default=8)
args = parser.parse_args()
@@ -123,36 +124,36 @@ if __name__ == "__main__":
syncer = GithubToS3(bucket=args.bucket_path, local_path=args.local_path)
syncer.check_bucket()
- document_folders = args.document_folders
+ document_packages = args.document_packages
# Make sure you are in the right directory for git commands
os.chdir(Path(args.local_path).parent.as_posix())
# Force color
os.environ["FORCE_COLOR"] = "1"
- if document_folders != "all" and args.sync_type == "single_commit":
- console.print(f"[red] Invalid folder name {document_folders} for sync type {args.sync_type} - only "
- f"all can be used with single_commit[/]")
+ if document_packages != "all" and args.sync_type == "single-commit":
+ console.print(f"[red] Invalid package name {document_packages} for sync type {args.sync_type} - only "
+ f"all can be used with single-commit[/]")
sys.exit(1)
- if document_folders and document_folders != "all" and args.sync_type == "full_sync":
- folders_to_sync = []
- for _folder in document_folders.split(" "):
- full_local_path = Path(f"{args.local_path}/{_folder}")
+ if document_packages and document_packages != "all" and args.sync_type == "full-sync":
+ packages_to_sync = []
+ for _package in document_packages.split(" "):
+ full_local_path = Path(f"{args.local_path}/{_package}")
if not full_local_path.exists():
- full_local_path = Path(f"{args.local_path}/{convert_short_name_to_folder_name(_folder)}")
+ full_local_path = Path(f"{args.local_path}/{convert_short_name_to_full_package_name(_package)}")
if full_local_path.exists():
- console.print(f"[blue] Document folder {_folder} exists in bucket {args.bucket_path}.[/]")
- folders_to_sync.append(_folder)
+ console.print(f"[blue] Document package {_package} exists in bucket {args.bucket_path}.[/]")
+ packages_to_sync.append(_package)
else:
- console.print(f"[red] Document folder {full_local_path} does not exist.[/]")
+ console.print(f"[red] Document package {full_local_path} does not exist.[/]")
sys.exit(1)
- syncer.full_sync(processes=int(args.processes), folders=folders_to_sync)
- elif args.sync_type == "full_sync":
+ syncer.full_sync(processes=int(args.processes), packages=packages_to_sync)
+ elif args.sync_type == "full-sync":
syncer.full_sync(processes=int(args.processes))
- elif args.sync_type == "single_commit" and args.commit_sha and document_folders == "all":
- console.print(f"[blue] Syncing last commit {args.commit_sha} from {args.local_path} [/]")
- syncer.sync_single_commit_files(args.commit_sha, processes=int(args.processes))
+ elif args.sync_type == "single-commit" and args.commit_ref and document_packages == "all":
+ console.print(f"[blue] Syncing last commit {args.commit_ref} from {args.local_path} [/]")
+ syncer.sync_single_commit_files(args.commit_ref, processes=int(args.processes))
else:
- console.print(f"[red] Invalid sync type {args.sync_type} with document folders {document_folders} "
- f"and commit sha {args.commit_sha}[/]")
+ console.print(f"[red] Invalid sync type {args.sync_type} with document packages {document_packages} "
+ f"and commit ref {args.commit_ref}[/]")
sys.exit(1)
diff --git a/scripts/s3_to_github.py b/scripts/s3_to_github.py
index 0eac7ffb10..f780ffd7b4 100644
--- a/scripts/s3_to_github.py
+++ b/scripts/s3_to_github.py
@@ -29,7 +29,7 @@ import sys
from rich.console import Console
-from transfer_utils import CommonTransferUtils, convert_short_name_to_folder_name, sort_priority_folders, \
+from transfer_utils import CommonTransferUtils, convert_short_name_to_full_package_name, sort_priority_packages, \
sort_priority_tuples
console = Console(width=200, color_system="standard")
@@ -40,18 +40,18 @@ class S3TOGithub(CommonTransferUtils):
def __init__(self, bucket:str , local_path: str):
super().__init__(bucket, local_path)
- def verify_document_folder(self, document_folder: str):
+ def verify_document_package(self, document_package: str):
response= self.s3_client.list_objects_v2(
Bucket=self.bucket_name,
- Prefix=self.prefix.rstrip("/") + "/" + document_folder,
+ Prefix=self.prefix.rstrip("/") + "/" + document_package,
)
return response["KeyCount"] > 0
- def sync_s3_to_github(self, processes: int, folders: list[str] | None = None,
+ def sync_s3_to_github(self, processes: int, packages: list[str] | None = None,
remote_prefix: str = "docs/"):
console.print("[blue] Syncing files from S3 to GitHub...[/]")
- prefixes = self.get_list_of_folders() if not folders else [
- f"{remote_prefix}{folder}" for folder in folders
+ prefixes = self.get_list_of_packages() if not packages else [
+ f"{remote_prefix}{package}" for package in packages
]
pool_args = []
for pref in prefixes:
@@ -70,31 +70,31 @@ if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Sync S3 to GitHub")
parser.add_argument("--bucket-path", required=True, help="S3 bucket name with path")
parser.add_argument("--local-path", required=True, help="local path to sync")
- parser.add_argument("--document-folders", help="Document folders to sync", default="all")
+ parser.add_argument("--document-packages", help="Document packages to sync", default="all")
parser.add_argument("--processes", help="Number of processes", type=int, default=8)
args = parser.parse_args()
syncer = S3TOGithub(bucket=args.bucket_path, local_path=args.local_path)
syncer.check_bucket()
- _document_folders = args.document_folders
+ _document_packages = args.document_packages
# Make sure you are in the right directory for git commands
os.chdir(Path(args.local_path).parent.as_posix())
# Force color
os.environ["FORCE_COLOR"] = "1"
- if _document_folders and _document_folders != "all":
- folders_to_sync = []
- for _folder in _document_folders.split(" "):
- full_folder_name = convert_short_name_to_folder_name(_folder)
- if syncer.verify_document_folder(full_folder_name):
- console.print(f"[blue] Document folder {full_folder_name} exists in bucket {args.bucket_path}.[/]")
- folders_to_sync.append(full_folder_name)
+ if _document_packages and _document_packages != "all":
+ packages_to_sync = []
+ for _package in _document_packages.split(" "):
+ full_package_name = convert_short_name_to_full_package_name(_package)
+ if syncer.verify_document_package(full_package_name):
+ console.print(f"[blue] Document package {full_package_name} exists in bucket {args.bucket_path}.[/]")
+ packages_to_sync.append(full_package_name)
else:
- console.print(f"[red] Document folder {full_folder_name} does not exist in bucket {args.bucket_path}.[/]")
+ console.print(f"[red] Document package {full_package_name} does not exist in bucket {args.bucket_path}.[/]")
sys.exit(1)
- folders_to_sync = sort_priority_folders(folders_to_sync)
- syncer.sync_s3_to_github(processes=int(args.processes), folders=folders_to_sync)
+ packages_to_sync = sort_priority_packages(packages_to_sync)
+ syncer.sync_s3_to_github(processes=int(args.processes), packages=packages_to_sync)
else:
syncer.sync_s3_to_github(processes=int(args.processes))
diff --git a/scripts/transfer_utils.py b/scripts/transfer_utils.py
index 4692c8cdc8..6dd480bcdf 100644
--- a/scripts/transfer_utils.py
+++ b/scripts/transfer_utils.py
@@ -16,13 +16,13 @@ KNOWN_PACKAGES = ["apache-airflow", "helm-chart", "docker-stack"]
console = Console(width=200, color_system="standard")
-def track_progress(folder: str, file_path: Path):
+def track_progress(package: str, file_path: Path):
while file_path.exists():
sleep(10)
if not file_path.exists():
break
num_lines = file_path.read_text().count("\n")
- console.print(f"{folder}:[blue] Processed {num_lines} files[/]")
+ console.print(f"{package}:[blue] Processed {num_lines} files[/]")
class CommonTransferUtils:
@@ -61,8 +61,8 @@ class CommonTransferUtils:
console.print(f"[red] Error: {e}[/]")
sys.exit(1)
- def get_list_of_folders(self) -> list[str]:
- folders = []
+ def get_list_of_packages(self) -> list[str]:
+ packages = []
try:
response = self.s3_client.list_objects_v2(
Bucket=self.bucket_name,
@@ -71,8 +71,8 @@ class CommonTransferUtils:
)
if 'CommonPrefixes' in response:
for cur_prefix in response['CommonPrefixes']:
- folders.append(cur_prefix['Prefix'])
- return sorted(folders)
+ packages.append(cur_prefix['Prefix'])
+ return sorted(packages)
except Exception as e:
console.print(f"[yellow] Error: {e}[/]")
return []
@@ -104,7 +104,7 @@ class CommonTransferUtils:
@staticmethod
def run_with_pool(func: Callable, args: Any, processes: int = 4):
# Chunksize is set to 1 - otherwise map / starmap will send tasks in chunks
- # and the prioritization we set for folders will be lost.
+ # and the prioritization we set for packages will be lost.
# Our tasks are big enough to not cause overhead of sending
# them one at a time.
with Pool(processes=processes) as pool:
@@ -136,33 +136,33 @@ class CommonTransferUtils:
)
console.print(f"{file_to_delete}[green] Delete completed[/]")
-def convert_short_name_to_folder_name(short_name: str) -> str:
+def convert_short_name_to_full_package_name(short_name: str) -> str:
if not short_name.startswith("apache-airflow-providers-") and short_name not in KNOWN_PACKAGES:
return f"apache-airflow-providers-{short_name.replace('.', '-')}"
return short_name
-# start with those folders first
-PRIORITY_FOLDERS = ["apache-airflow-providers-google", "apache-airflow-providers-amazon", "apache-airflow"]
+# start with those packages first
+PRIORITY_PACKAGES = ["apache-airflow-providers-google", "apache-airflow-providers-amazon", "apache-airflow"]
-def sort_priority_folders(folders: list[str]) -> list[str]:
+def sort_priority_packages(packages: list[str]) -> list[str]:
"""
- Sort the folders in a way that the priority folders are at the top
+ Sort the packages in a way that the priority packages are at the top
"""
- sorted_folders = []
- for folder in PRIORITY_FOLDERS:
- if folder in folders:
- sorted_folders.append(folder)
- folders.remove(folder)
- return sorted_folders + sorted(folders)
+ sorted_packages = []
+ for package in PRIORITY_PACKAGES:
+ if package in packages:
+ sorted_packages.append(package)
+ packages.remove(package)
+ return sorted_packages + sorted(packages)
def sort_priority_tuples(tuples: list[tuple[str, str]]) -> list[tuple[str, str]]:
"""
- Sort the tuples in a way that the priority folders are at the top
+ Sort the tuples in a way that the priority packages are at the top
"""
sorted_tuples = []
- for folder in PRIORITY_FOLDERS:
+ for package in PRIORITY_PACKAGES:
for tup in tuples:
- if tup[0].endswith(folder +"/"):
+ if tup[0].endswith(package +"/"):
sorted_tuples.append(tup)
tuples.remove(tup)
return sorted_tuples + sorted(tuples, key=lambda x: x[0])