You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/06/09 16:44:13 UTC
[airflow] 01/02: Improved cloud tool available in the trimmed down
CI container (#9167)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 5661dc95a7e157b612fe817057048486275d735a
Author: Jarek Potiuk <ja...@polidea.com>
AuthorDate: Tue Jun 9 09:33:16 2020 +0200
Improved cloud tool available in the trimmed down CI container (#9167)
* Improved cloud tool available in the trimmed down CI container
The tools now have shebangs which make them available for
python tools. Also /opt/airflow is now mounted from the
host Airflow sources which makes it possible for the tools to
copy files directly to/from the sources of Airflow.
It also contains one small change for Linux users - the files
created by docker gcloud are created with root user so in order to fix
that the directories mounted from the host are fixed when you exit
the tool - their ownership is changed to be owned by the host user
(cherry picked from commit de9d3401f9668c703a8f0665769fd17268f607ed)
---
.pre-commit-config.yaml | 2 +-
BREEZE.rst | 3 +
Dockerfile.ci | 21 +----
breeze | 1 +
requirements/requirements-python2.7.txt | 79 +++++++++---------
requirements/requirements-python3.5.txt | 95 +++++++++++-----------
requirements/requirements-python3.6.txt | 99 ++++++++++++-----------
requirements/requirements-python3.7.txt | 97 +++++++++++-----------
requirements/setup-2.7.md5 | 2 +-
requirements/setup-3.5.md5 | 2 +-
requirements/setup-3.6.md5 | 2 +-
requirements/setup-3.7.md5 | 2 +-
scripts/ci/ci_docs.sh | 3 +
scripts/ci/ci_fix_ownership.sh | 3 +
scripts/ci/ci_flake8.sh | 6 ++
scripts/ci/ci_mypy.sh | 3 +
scripts/ci/docker-compose/forward-credentials.yml | 2 +
scripts/ci/docker-compose/local-prod.yml | 2 +
scripts/ci/docker-compose/local.yml | 2 +
scripts/ci/in_container/_in_container_utils.sh | 16 ++--
scripts/ci/libraries/_initialization.sh | 7 ++
scripts/ci/libraries/_runs.sh | 6 ++
scripts/ci/prepare_tool_scripts.sh | 64 +++++++++++++++
23 files changed, 308 insertions(+), 211 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 4979f24..7e49b77 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -60,7 +60,7 @@ repos:
- id: insert-license
name: Add license for all JS/CSS files
files: \.(js|css)$
- exclude: ^\.github/.*$
+ exclude: ^\.github/.*$|^airflow/www/static/.*|^airflow/www_rbac/static/.*$
args:
- --comment-style
- "/**| *| */"
diff --git a/BREEZE.rst b/BREEZE.rst
index aec975c..3a1a6a0 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -243,6 +243,9 @@ it is downloaded, it will stay until you remove the downloaded images from your
For each of those CLI credentials are taken (automatically) from the credentials you have defined in
your ${HOME} directory on host.
+Those tools also have host Airflow source directory mounted in /opt/airflow path
+so you can directly transfer files to/from your airflow host sources.
+
Those are currently installed CLIs (they are available as aliases to the docker commands):
+-----------------------+----------+-------------------------------------------------+-------------------+
diff --git a/Dockerfile.ci b/Dockerfile.ci
index 8035fa2..8051431 100644
--- a/Dockerfile.ci
+++ b/Dockerfile.ci
@@ -307,26 +307,7 @@ RUN if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \
pip install ${ADDITIONAL_PYTHON_DEPS}; \
fi
-RUN \
- AWSCLI_IMAGE="amazon/aws-cli:latest" && \
- AZURECLI_IMAGE="mcr.microsoft.com/azure-cli:latest" && \
- GCLOUD_IMAGE="gcr.io/google.com/cloudsdktool/cloud-sdk:latest" && \
- echo "docker run --rm -it -v \${HOST_HOME}/.aws:/root/.aws ${AWSCLI_IMAGE} \"\$@\"" \
- > /usr/bin/aws && \
- echo "docker pull ${AWSCLI_IMAGE}" > /usr/bin/aws-update && \
- echo "docker run --rm -it -v \${HOST_HOME}/.azure:/root/.azure ${AZURECLI_IMAGE} \"\$@\"" \
- > /usr/bin/az && \
- echo "docker pull ${AZURECLI_IMAGE}" > /usr/bin/az-update && \
- echo "docker run --rm -it -v \${HOST_HOME}/.config:/root/.config ${GCLOUD_IMAGE} bq \"\$@\"" \
- > /usr/bin/bq && \
- echo "docker pull ${GCLOUD_IMAGE}" > /usr/bin/bq-update && \
- echo "docker run --rm -it -v \${HOST_HOME}/.config:/root/.config ${GCLOUD_IMAGE} gcloud \"\$@\"" \
- > /usr/bin/gcloud && \
- echo "docker pull ${GCLOUD_IMAGE}" > /usr/bin/gcloud-update && \
- echo "docker run --rm -it -v \${HOST_HOME}/.config:/root/.config ${GCLOUD_IMAGE} gsutil \"\$@\"" \
- > /usr/bin/gsutil && \
- echo "docker pull ${GCLOUD_IMAGE}" > /usr/bin/gsutil-update && \
- chmod a+x /usr/bin/aws /usr/bin/az /usr/bin/bq /usr/bin/gcloud /usr/bin/gsutil
+RUN scripts/ci/prepare_tool_scripts.sh
WORKDIR ${AIRFLOW_SOURCES}
diff --git a/breeze b/breeze
index d9cfd92..005f917 100755
--- a/breeze
+++ b/breeze
@@ -443,6 +443,7 @@ export POSTGRES_HOST_PORT="${POSTGRES_HOST_PORT}"
export POSTGRES_VERSION="${POSTGRES_VERSION}"
export MYSQL_HOST_PORT="${MYSQL_HOST_PORT}"
export MYSQL_VERSION="${MYSQL_VERSION}"
+export AIRFLOW_SOURCES="${AIRFLOW_SOURCES}"
export MYSQL_ENCODING="${MYSQL_ENCODING}"
export AIRFLOW_CI_IMAGE="${AIRFLOW_CI_IMAGE}"
export AIRFOW_PROD_IMAGE="${AIRFLOW_PROD_IMAGE}"
diff --git a/requirements/requirements-python2.7.txt b/requirements/requirements-python2.7.txt
index 323e15a..909d880 100644
--- a/requirements/requirements-python2.7.txt
+++ b/requirements/requirements-python2.7.txt
@@ -8,18 +8,18 @@ Flask-Caching==1.3.3
Flask-JWT-Extended==3.24.1
Flask-Login==0.4.1
Flask-OpenID==1.2.5
-Flask-SQLAlchemy==2.4.1
+Flask-SQLAlchemy==2.4.3
Flask-WTF==0.14.3
Flask==1.1.2
JPype1==0.7.1
-JayDeBeApi==1.1.1
+JayDeBeApi==1.2.2
Jinja2==2.10.3
-Mako==1.1.2
+Mako==1.1.3
Markdown==2.6.11
MarkupSafe==1.1.1
PyHive==0.6.2
PyJWT==1.7.1
-PyNaCl==1.3.0
+PyNaCl==1.4.0
PySmbClient==0.1.5
PyYAML==5.3.1
Pygments==2.5.2
@@ -29,10 +29,10 @@ Sphinx==1.8.5
Unidecode==1.1.1
WTForms==2.3.1
Werkzeug==0.16.1
-adal==1.2.3
+adal==1.2.4
alabaster==0.7.12
alembic==1.4.2
-amqp==2.5.2
+amqp==2.6.0
analytics-python==1.2.9
ansiwrap==0.8.4
apipkg==1.5
@@ -45,14 +45,14 @@ astroid==1.6.6
atlasclient==1.0.0
atomicwrites==1.4.0
attrs==19.3.0
-aws-sam-translator==1.23.0
-aws-xray-sdk==2.5.0
+aws-sam-translator==1.24.0
+aws-xray-sdk==2.6.0
azure-common==1.1.25
azure-cosmos==3.1.2
azure-datalake-store==0.0.48
azure-mgmt-containerinstance==1.5.0
azure-mgmt-nspkg==3.0.2
-azure-mgmt-resource==9.0.0
+azure-mgmt-resource==10.0.0
azure-nspkg==3.0.2
azure-storage-blob==2.1.0
azure-storage-common==2.1.0
@@ -69,18 +69,18 @@ beautifulsoup4==4.7.1
billiard==3.6.3.0
bleach==3.1.5
blinker==1.4
-boto3==1.13.15
+boto3==1.13.25
boto==2.49.0
-botocore==1.16.15
+botocore==1.16.25
cached-property==1.5.1
cachetools==3.1.1
cassandra-driver==3.20.2
cattrs==0.9.2
-celery==4.4.2
-certifi==2020.4.5.1
+celery==4.4.5
+certifi==2020.4.5.2
cffi==1.14.0
cfgv==2.0.1
-cfn-lint==0.32.1
+cfn-lint==0.33.0
cgroupspy==0.1.6
chardet==3.0.4
click==6.7
@@ -92,7 +92,7 @@ contextdecorator==0.10.0
contextlib2==0.6.0.post1
cookies==2.2.1
coverage==5.1
-croniter==0.3.31
+croniter==0.3.32
cryptography==2.9.2
cx-Oracle==7.3.0
datadog==0.36.0
@@ -115,7 +115,7 @@ execnet==1.7.1
fastavro==0.23.4
filelock==3.0.12
flake8-colors==0.1.6
-flake8==3.8.1
+flake8==3.8.3
flaky==3.6.1
flask-swagger==0.2.13
flower==0.9.4
@@ -126,27 +126,27 @@ future-fstrings==1.2.0
future==0.18.2
futures==3.3.0
gcsfs==0.2.3
-google-api-core==1.17.0
-google-api-python-client==1.8.3
+google-api-core==1.19.1
+google-api-python-client==1.9.1
google-auth-httplib2==0.0.3
google-auth-oauthlib==0.4.1
-google-auth==1.15.0
-google-cloud-bigquery==1.24.0
+google-auth==1.16.1
+google-cloud-bigquery==1.25.0
google-cloud-bigtable==1.2.1
google-cloud-container==0.5.0
google-cloud-core==1.3.0
google-cloud-dlp==0.15.0
google-cloud-language==1.3.0
google-cloud-secret-manager==1.0.0
-google-cloud-spanner==1.16.0
+google-cloud-spanner==1.17.0
google-cloud-speech==1.3.2
google-cloud-storage==1.28.1
google-cloud-texttospeech==1.0.1
google-cloud-translate==2.0.1
google-cloud-videointelligence==1.14.0
google-cloud-vision==1.0.0
-google-resumable-media==0.5.0
-googleapis-common-protos==1.51.0
+google-resumable-media==0.5.1
+googleapis-common-protos==1.52.0
graphviz==0.14
grpc-google-iam-v1==0.12.3
grpcio-gcp==0.2.2
@@ -156,13 +156,13 @@ hdfs==2.5.8
hmsclient==0.1.1
httplib2==0.18.1
humanize==0.5.1
-hvac==0.10.2
-identify==1.4.16
+hvac==0.10.3
+identify==1.4.19
idna==2.9
ijson==2.6.1
imagesize==1.2.0
-importlib-metadata==1.6.0
-importlib-resources==1.5.0
+importlib-metadata==1.6.1
+importlib-resources==2.0.0
inflection==0.3.1
ipaddress==1.0.23
ipdb==0.13.2
@@ -184,7 +184,7 @@ jupyter-client==5.3.4
jupyter-core==4.6.3
kombu==4.6.3
kubernetes==11.0.0
-lazy-object-proxy==1.4.3
+lazy-object-proxy==1.5.0
ldap3==2.7
linecache2==1.0.0
lockfile==0.12.2
@@ -198,14 +198,15 @@ mongomock==3.19.0
monotonic==1.5
more-itertools==5.0.0
moto==1.3.14
-msrest==0.6.14
+msrest==0.6.15
msrestazure==0.6.3
multi-key-dict==2.0.3
mysqlclient==1.3.14
+natsort==6.2.1
nbconvert==5.6.1
nbformat==4.4.0
networkx==2.2
-nodeenv==1.3.5
+nodeenv==1.4.0
nteract-scrapbook==0.3.1
ntlm-auth==1.4.0
numpy==1.16.6
@@ -230,7 +231,7 @@ pre-commit==1.21.0
presto-python-client==0.7.0
prison==0.1.0
prompt-toolkit==1.0.18
-protobuf==3.12.1
+protobuf==3.12.2
psutil==5.7.0
psycopg2-binary==2.8.5
ptyprocess==0.6.0
@@ -250,26 +251,28 @@ pymssql==2.1.4
pyparsing==2.4.7
pyrsistent==0.16.0
pysftp==0.2.9
-pytest-cov==2.8.1
+pytest-cov==2.9.0
pytest-forked==1.1.3
pytest-instafail==0.4.1.post0
pytest-rerunfailures==9.0
pytest-timeout==1.3.4
pytest-xdist==1.32.0
-pytest==4.6.10
+pytest==4.6.11
python-daemon==2.1.2
python-dateutil==2.8.1
python-editor==1.0.4
python-http-client==3.2.7
python-jenkins==1.7.0
python-jose==3.1.0
+python-nvd3==0.15.0
python-openid==2.2.5
+python-slugify==4.0.0
pytz==2020.1
pytzdata==2019.3
pywinrm==0.4.1
pyzmq==19.0.1
-qds-sdk==1.15.2
-redis==3.5.2
+qds-sdk==1.16.0
+redis==3.5.3
requests-futures==0.9.4
requests-kerberos==0.12.0
requests-mock==1.8.0
@@ -309,14 +312,14 @@ tabulate==0.8.7
tenacity==4.12.0
termcolor==1.1.0
testpath==0.4.4
-text-unidecode==1.2
+text-unidecode==1.3
textwrap3==0.9.2
thrift-sasl==0.4.2
thrift==0.13.0
tokenize-rt==3.2.0
toml==0.10.1
tornado==5.1.1
-tqdm==4.46.0
+tqdm==4.46.1
traceback2==1.4.0
traitlets==4.3.3
typing-extensions==3.7.4.2
@@ -329,7 +332,7 @@ urllib3==1.25.9
vertica-python==0.10.4
vine==1.3.0
virtualenv==20.0.21
-wcwidth==0.1.9
+wcwidth==0.2.4
webencodings==0.5.1
websocket-client==0.57.0
wrapt==1.12.1
diff --git a/requirements/requirements-python3.5.txt b/requirements/requirements-python3.5.txt
index bee756e..a167c60 100644
--- a/requirements/requirements-python3.5.txt
+++ b/requirements/requirements-python3.5.txt
@@ -8,70 +8,70 @@ Flask-Caching==1.3.3
Flask-JWT-Extended==3.24.1
Flask-Login==0.4.1
Flask-OpenID==1.2.5
-Flask-SQLAlchemy==2.4.1
+Flask-SQLAlchemy==2.4.3
Flask-WTF==0.14.3
Flask==1.1.2
JPype1==0.7.1
-JayDeBeApi==1.1.1
+JayDeBeApi==1.2.2
Jinja2==2.10.3
-Mako==1.1.2
+Mako==1.1.3
Markdown==2.6.11
MarkupSafe==1.1.1
PyHive==0.6.2
PyJWT==1.7.1
-PyNaCl==1.3.0
+PyNaCl==1.4.0
PySmbClient==0.1.5
PyYAML==5.3.1
Pygments==2.6.1
SQLAlchemy-JSONField==0.9.0
SQLAlchemy==1.3.17
-Sphinx==3.0.3
+Sphinx==3.1.0
Unidecode==1.1.1
WTForms==2.3.1
Werkzeug==0.16.1
-adal==1.2.3
+adal==1.2.4
alabaster==0.7.12
alembic==1.4.2
-amqp==2.5.2
+amqp==2.6.0
analytics-python==1.2.9
ansiwrap==0.8.4
apipkg==1.5
-apispec==3.3.0
+apispec==3.3.1
appdirs==1.4.4
argcomplete==1.11.1
asn1crypto==1.3.0
aspy.yaml==1.3.0
-astroid==2.4.1
+astroid==2.4.2
atlasclient==1.0.0
attrs==19.3.0
-aws-sam-translator==1.23.0
-aws-xray-sdk==2.5.0
+aws-sam-translator==1.24.0
+aws-xray-sdk==2.6.0
azure-common==1.1.25
azure-cosmos==3.1.2
azure-datalake-store==0.0.48
azure-mgmt-containerinstance==1.5.0
-azure-mgmt-resource==9.0.0
+azure-mgmt-resource==10.0.0
azure-nspkg==3.0.2
azure-storage-blob==2.1.0
azure-storage-common==2.1.0
azure-storage==0.36.0
-backcall==0.1.0
+backcall==0.2.0
bcrypt==3.1.7
beautifulsoup4==4.7.1
billiard==3.6.3.0
blinker==1.4
-boto3==1.13.15
+boto3==1.13.25
boto==2.49.0
-botocore==1.16.15
+botocore==1.16.25
cached-property==1.5.1
cachetools==4.1.0
cassandra-driver==3.20.2
cattrs==0.9.2
-celery==4.4.2
-certifi==2020.4.5.1
+celery==4.4.5
+certifi==2020.4.5.2
cffi==1.14.0
cfgv==2.0.1
-cfn-lint==0.32.1
+cfn-lint==0.33.0
cgroupspy==0.1.6
chardet==3.0.4
click==6.7
@@ -80,7 +80,7 @@ colorama==0.4.3
colorlog==4.0.2
configparser==3.5.3
coverage==5.1
-croniter==0.3.31
+croniter==0.3.32
cryptography==2.9.2
cx-Oracle==7.3.0
datadog==0.36.0
@@ -102,7 +102,7 @@ execnet==1.7.1
fastavro==0.23.4
filelock==3.0.12
flake8-colors==0.1.6
-flake8==3.8.1
+flake8==3.8.3
flaky==3.6.1
flask-swagger==0.2.13
flower==0.9.4
@@ -112,27 +112,27 @@ funcsigs==1.0.2
future-fstrings==1.2.0
future==0.18.2
gcsfs==0.6.2
-google-api-core==1.17.0
-google-api-python-client==1.8.3
+google-api-core==1.19.1
+google-api-python-client==1.9.1
google-auth-httplib2==0.0.3
google-auth-oauthlib==0.4.1
-google-auth==1.15.0
-google-cloud-bigquery==1.24.0
+google-auth==1.16.1
+google-cloud-bigquery==1.25.0
google-cloud-bigtable==1.2.1
google-cloud-container==0.5.0
google-cloud-core==1.3.0
google-cloud-dlp==0.15.0
google-cloud-language==1.3.0
google-cloud-secret-manager==1.0.0
-google-cloud-spanner==1.16.0
+google-cloud-spanner==1.17.0
google-cloud-speech==1.3.2
google-cloud-storage==1.28.1
google-cloud-texttospeech==1.0.1
google-cloud-translate==2.0.1
google-cloud-videointelligence==1.14.0
google-cloud-vision==1.0.0
-google-resumable-media==0.5.0
-googleapis-common-protos==1.51.0
+google-resumable-media==0.5.1
+googleapis-common-protos==1.52.0
graphviz==0.14
grpc-google-iam-v1==0.12.3
grpcio-gcp==0.2.2
@@ -142,14 +142,14 @@ hdfs==2.5.8
hmsclient==0.1.1
httplib2==0.18.1
humanize==0.5.1
-hvac==0.10.2
-identify==1.4.16
+hvac==0.10.3
+identify==1.4.19
idna==2.9
ijson==2.6.1
imagesize==1.2.0
-importlib-metadata==1.6.0
-importlib-resources==1.5.0
-inflection==0.4.0
+importlib-metadata==1.6.1
+importlib-resources==2.0.0
+inflection==0.5.0
ipdb==0.13.2
ipython-genutils==0.2.0
ipython==7.9.0
@@ -168,9 +168,9 @@ jsonschema==3.2.0
junit-xml==1.9
jupyter-client==6.1.3
jupyter-core==4.6.3
-kombu==4.6.8
+kombu==4.6.10
kubernetes==11.0.0
-lazy-object-proxy==1.4.3
+lazy-object-proxy==1.5.0
ldap3==2.7
lockfile==0.12.2
marshmallow-enum==1.5.1
@@ -181,19 +181,20 @@ mock==3.0.5
mongomock==3.19.0
more-itertools==8.3.0
moto==1.3.14
-msrest==0.6.14
+msrest==0.6.15
msrestazure==0.6.3
multi-key-dict==2.0.3
mypy-extensions==0.4.3
mypy==0.720
mysqlclient==1.3.14
+natsort==7.0.1
nbclient==0.1.0
nbformat==5.0.6
networkx==2.4
-nodeenv==1.3.5
+nodeenv==1.4.0
nteract-scrapbook==0.4.1
ntlm-auth==1.4.0
-numpy==1.18.4
+numpy==1.18.5
oauthlib==3.1.0
oscrypto==1.2.0
packaging==20.4
@@ -215,7 +216,7 @@ pre-commit==1.21.0
presto-python-client==0.7.0
prison==0.1.0
prompt-toolkit==2.0.10
-protobuf==3.12.1
+protobuf==3.12.2
psutil==5.7.0
psycopg2-binary==2.8.5
ptyprocess==0.6.0
@@ -236,26 +237,28 @@ pymssql==2.1.4
pyparsing==2.4.7
pyrsistent==0.16.0
pysftp==0.2.9
-pytest-cov==2.8.1
+pytest-cov==2.9.0
pytest-forked==1.1.3
pytest-instafail==0.4.1.post0
pytest-rerunfailures==9.0
pytest-timeout==1.3.4
pytest-xdist==1.32.0
-pytest==5.4.2
+pytest==5.4.3
python-daemon==2.1.2
python-dateutil==2.8.1
python-editor==1.0.4
python-http-client==3.2.7
python-jenkins==1.7.0
python-jose==3.1.0
+python-nvd3==0.15.0
+python-slugify==4.0.0
python3-openid==3.1.0
pytz==2020.1
pytzdata==2019.3
pywinrm==0.4.1
pyzmq==19.0.1
-qds-sdk==1.15.2
-redis==3.5.2
+qds-sdk==1.16.0
+redis==3.5.3
requests-futures==0.9.4
requests-kerberos==0.12.0
requests-mock==1.8.0
@@ -274,7 +277,7 @@ setproctitle==1.1.10
six==1.15.0
slackclient==1.3.2
snowballstemmer==2.0.0
-snowflake-connector-python==2.2.6
+snowflake-connector-python==2.2.7
snowflake-sqlalchemy==1.2.3
soupsieve==2.0.1
sphinx-argparse==0.2.5
@@ -295,14 +298,14 @@ sshtunnel==0.1.5
tabulate==0.8.7
tenacity==4.12.0
termcolor==1.1.0
-text-unidecode==1.2
+text-unidecode==1.3
textwrap3==0.9.2
thrift-sasl==0.4.2
thrift==0.13.0
tokenize-rt==3.2.0
toml==0.10.1
tornado==5.1.1
-tqdm==4.46.0
+tqdm==4.46.1
traitlets==4.3.3
typed-ast==1.4.1
typing-extensions==3.7.4.2
@@ -314,7 +317,7 @@ urllib3==1.25.9
vertica-python==0.10.4
vine==1.3.0
virtualenv==20.0.21
-wcwidth==0.1.9
+wcwidth==0.2.4
websocket-client==0.57.0
wrapt==1.12.1
xmltodict==0.12.0
diff --git a/requirements/requirements-python3.6.txt b/requirements/requirements-python3.6.txt
index e020794..b189c42 100644
--- a/requirements/requirements-python3.6.txt
+++ b/requirements/requirements-python3.6.txt
@@ -8,32 +8,32 @@ Flask-Caching==1.3.3
Flask-JWT-Extended==3.24.1
Flask-Login==0.4.1
Flask-OpenID==1.2.5
-Flask-SQLAlchemy==2.4.1
+Flask-SQLAlchemy==2.4.3
Flask-WTF==0.14.3
Flask==1.1.2
JPype1==0.7.1
-JayDeBeApi==1.1.1
+JayDeBeApi==1.2.2
Jinja2==2.10.3
-Mako==1.1.2
+Mako==1.1.3
Markdown==2.6.11
MarkupSafe==1.1.1
PyHive==0.6.2
PyJWT==1.7.1
-PyNaCl==1.3.0
+PyNaCl==1.4.0
PySmbClient==0.1.5
PyYAML==5.3.1
Pygments==2.6.1
SQLAlchemy-JSONField==0.9.0
-SQLAlchemy-Utils==0.36.5
+SQLAlchemy-Utils==0.36.6
SQLAlchemy==1.3.17
-Sphinx==3.0.3
+Sphinx==3.1.0
Unidecode==1.1.1
WTForms==2.3.1
Werkzeug==0.16.1
-adal==1.2.3
+adal==1.2.4
alabaster==0.7.12
alembic==1.4.2
-amqp==2.5.2
+amqp==2.6.0
analytics-python==1.2.9
ansiwrap==0.8.4
apipkg==1.5
@@ -41,39 +41,39 @@ apispec==1.3.3
appdirs==1.4.4
argcomplete==1.11.1
asn1crypto==1.3.0
-astroid==2.4.1
+astroid==2.4.2
async-generator==1.10
atlasclient==1.0.0
attrs==19.3.0
-aws-sam-translator==1.23.0
-aws-xray-sdk==2.5.0
+aws-sam-translator==1.24.0
+aws-xray-sdk==2.6.0
azure-common==1.1.25
azure-cosmos==3.1.2
azure-datalake-store==0.0.48
azure-mgmt-containerinstance==1.5.0
-azure-mgmt-resource==9.0.0
+azure-mgmt-resource==10.0.0
azure-nspkg==3.0.2
azure-storage-blob==2.1.0
azure-storage-common==2.1.0
azure-storage==0.36.0
-backcall==0.1.0
+backcall==0.2.0
bcrypt==3.1.7
beautifulsoup4==4.7.1
billiard==3.6.3.0
black==19.10b0
blinker==1.4
-boto3==1.13.15
+boto3==1.13.25
boto==2.49.0
-botocore==1.16.15
+botocore==1.16.25
cached-property==1.5.1
cachetools==4.1.0
cassandra-driver==3.20.2
cattrs==0.9.2
-celery==4.4.2
-certifi==2020.4.5.1
+celery==4.4.5
+certifi==2020.4.5.2
cffi==1.14.0
cfgv==3.1.0
-cfn-lint==0.32.1
+cfn-lint==0.33.0
cgroupspy==0.1.6
chardet==3.0.4
click==6.7
@@ -82,7 +82,7 @@ colorama==0.4.3
colorlog==4.0.2
configparser==3.5.3
coverage==5.1
-croniter==0.3.31
+croniter==0.3.32
cryptography==2.9.2
cx-Oracle==7.3.0
datadog==0.36.0
@@ -104,7 +104,7 @@ execnet==1.7.1
fastavro==0.23.4
filelock==3.0.12
flake8-colors==0.1.6
-flake8==3.8.1
+flake8==3.8.3
flaky==3.6.1
flask-swagger==0.2.13
flower==0.9.4
@@ -114,27 +114,27 @@ funcsigs==1.0.2
future-fstrings==1.2.0
future==0.18.2
gcsfs==0.6.2
-google-api-core==1.17.0
-google-api-python-client==1.8.3
+google-api-core==1.19.1
+google-api-python-client==1.9.1
google-auth-httplib2==0.0.3
google-auth-oauthlib==0.4.1
-google-auth==1.15.0
-google-cloud-bigquery==1.24.0
+google-auth==1.16.1
+google-cloud-bigquery==1.25.0
google-cloud-bigtable==1.2.1
google-cloud-container==0.5.0
google-cloud-core==1.3.0
google-cloud-dlp==0.15.0
google-cloud-language==1.3.0
google-cloud-secret-manager==1.0.0
-google-cloud-spanner==1.16.0
+google-cloud-spanner==1.17.0
google-cloud-speech==1.3.2
google-cloud-storage==1.28.1
google-cloud-texttospeech==1.0.1
google-cloud-translate==2.0.1
google-cloud-videointelligence==1.14.0
google-cloud-vision==1.0.0
-google-resumable-media==0.5.0
-googleapis-common-protos==1.51.0
+google-resumable-media==0.5.1
+googleapis-common-protos==1.52.0
graphviz==0.14
grpc-google-iam-v1==0.12.3
grpcio-gcp==0.2.2
@@ -144,17 +144,17 @@ hdfs==2.5.8
hmsclient==0.1.1
httplib2==0.18.1
humanize==0.5.1
-hvac==0.10.2
-identify==1.4.16
+hvac==0.10.3
+identify==1.4.19
idna==2.9
ijson==2.6.1
imagesize==1.2.0
-importlib-metadata==1.6.0
-importlib-resources==1.5.0
-inflection==0.4.0
+importlib-metadata==1.6.1
+importlib-resources==2.0.0
+inflection==0.5.0
ipdb==0.13.2
ipython-genutils==0.2.0
-ipython==7.14.0
+ipython==7.15.0
iso8601==0.1.12
isodate==0.6.0
itsdangerous==1.1.0
@@ -170,33 +170,34 @@ jsonschema==3.2.0
junit-xml==1.9
jupyter-client==6.1.3
jupyter-core==4.6.3
-kombu==4.6.8
+kombu==4.6.10
kubernetes==11.0.0
-lazy-object-proxy==1.4.3
+lazy-object-proxy==1.5.0
ldap3==2.7
lockfile==0.12.2
marshmallow-enum==1.5.1
-marshmallow-sqlalchemy==0.23.0
+marshmallow-sqlalchemy==0.23.1
marshmallow==2.21.0
mccabe==0.6.1
mock==4.0.2
mongomock==3.19.0
more-itertools==8.3.0
moto==1.3.14
-msrest==0.6.14
+msrest==0.6.15
msrestazure==0.6.3
multi-key-dict==2.0.3
mypy-extensions==0.4.3
mypy==0.720
mysqlclient==1.3.14
+natsort==7.0.1
nbclient==0.3.1
nbformat==5.0.6
nest-asyncio==1.3.3
networkx==2.4
-nodeenv==1.3.5
+nodeenv==1.4.0
nteract-scrapbook==0.4.1
ntlm-auth==1.4.0
-numpy==1.18.4
+numpy==1.18.5
oauthlib==3.1.0
oscrypto==1.2.0
packaging==20.4
@@ -213,11 +214,11 @@ pexpect==4.8.0
pickleshare==0.7.5
pinotdb==0.1.1
pluggy==0.13.1
-pre-commit==2.4.0
+pre-commit==2.5.0
presto-python-client==0.7.0
prison==0.1.3
prompt-toolkit==3.0.5
-protobuf==3.12.1
+protobuf==3.12.2
psutil==5.7.0
psycopg2-binary==2.8.5
ptyprocess==0.6.0
@@ -238,13 +239,13 @@ pymssql==2.1.4
pyparsing==2.4.7
pyrsistent==0.16.0
pysftp==0.2.9
-pytest-cov==2.8.1
+pytest-cov==2.9.0
pytest-forked==1.1.3
pytest-instafail==0.4.1.post0
pytest-rerunfailures==9.0
pytest-timeout==1.3.4
pytest-xdist==1.32.0
-pytest==5.4.2
+pytest==5.4.3
python-daemon==2.1.2
python-dateutil==2.8.1
python-editor==1.0.4
@@ -258,9 +259,9 @@ pytz==2020.1
pytzdata==2019.3
pywinrm==0.4.1
pyzmq==19.0.1
-qds-sdk==1.15.2
-redis==3.5.2
-regex==2020.5.14
+qds-sdk==1.16.0
+redis==3.5.3
+regex==2020.6.8
requests-futures==0.9.4
requests-kerberos==0.12.0
requests-mock==1.8.0
@@ -279,7 +280,7 @@ setproctitle==1.1.10
six==1.15.0
slackclient==1.3.2
snowballstemmer==2.0.0
-snowflake-connector-python==2.2.6
+snowflake-connector-python==2.2.7
snowflake-sqlalchemy==1.2.3
soupsieve==2.0.1
sphinx-argparse==0.2.5
@@ -306,7 +307,7 @@ thrift-sasl==0.4.2
thrift==0.13.0
toml==0.10.1
tornado==5.1.1
-tqdm==4.46.0
+tqdm==4.46.1
traitlets==4.3.3
typed-ast==1.4.1
typing-extensions==3.7.4.2
@@ -318,7 +319,7 @@ urllib3==1.25.9
vertica-python==0.10.4
vine==1.3.0
virtualenv==20.0.21
-wcwidth==0.1.9
+wcwidth==0.2.4
websocket-client==0.57.0
wrapt==1.12.1
xmltodict==0.12.0
diff --git a/requirements/requirements-python3.7.txt b/requirements/requirements-python3.7.txt
index d0dee9e..a890025 100644
--- a/requirements/requirements-python3.7.txt
+++ b/requirements/requirements-python3.7.txt
@@ -8,32 +8,32 @@ Flask-Caching==1.3.3
Flask-JWT-Extended==3.24.1
Flask-Login==0.4.1
Flask-OpenID==1.2.5
-Flask-SQLAlchemy==2.4.1
+Flask-SQLAlchemy==2.4.3
Flask-WTF==0.14.3
Flask==1.1.2
JPype1==0.7.1
-JayDeBeApi==1.1.1
+JayDeBeApi==1.2.2
Jinja2==2.10.3
-Mako==1.1.2
+Mako==1.1.3
Markdown==2.6.11
MarkupSafe==1.1.1
PyHive==0.6.2
PyJWT==1.7.1
-PyNaCl==1.3.0
+PyNaCl==1.4.0
PySmbClient==0.1.5
PyYAML==5.3.1
Pygments==2.6.1
SQLAlchemy-JSONField==0.9.0
-SQLAlchemy-Utils==0.36.5
+SQLAlchemy-Utils==0.36.6
SQLAlchemy==1.3.17
-Sphinx==3.0.3
+Sphinx==3.1.0
Unidecode==1.1.1
WTForms==2.3.1
Werkzeug==0.16.1
-adal==1.2.3
+adal==1.2.4
alabaster==0.7.12
alembic==1.4.2
-amqp==2.5.2
+amqp==2.6.0
analytics-python==1.2.9
ansiwrap==0.8.4
apipkg==1.5
@@ -41,39 +41,39 @@ apispec==1.3.3
appdirs==1.4.4
argcomplete==1.11.1
asn1crypto==1.3.0
-astroid==2.4.1
+astroid==2.4.2
async-generator==1.10
atlasclient==1.0.0
attrs==19.3.0
-aws-sam-translator==1.23.0
-aws-xray-sdk==2.5.0
+aws-sam-translator==1.24.0
+aws-xray-sdk==2.6.0
azure-common==1.1.25
azure-cosmos==3.1.2
azure-datalake-store==0.0.48
azure-mgmt-containerinstance==1.5.0
-azure-mgmt-resource==9.0.0
+azure-mgmt-resource==10.0.0
azure-nspkg==3.0.2
azure-storage-blob==2.1.0
azure-storage-common==2.1.0
azure-storage==0.36.0
-backcall==0.1.0
+backcall==0.2.0
bcrypt==3.1.7
beautifulsoup4==4.7.1
billiard==3.6.3.0
black==19.10b0
blinker==1.4
-boto3==1.13.15
+boto3==1.13.25
boto==2.49.0
-botocore==1.16.15
+botocore==1.16.25
cached-property==1.5.1
cachetools==4.1.0
cassandra-driver==3.20.2
cattrs==0.9.2
-celery==4.4.2
-certifi==2020.4.5.1
+celery==4.4.5
+certifi==2020.4.5.2
cffi==1.14.0
cfgv==3.1.0
-cfn-lint==0.32.1
+cfn-lint==0.33.0
cgroupspy==0.1.6
chardet==3.0.4
click==6.7
@@ -82,7 +82,7 @@ colorama==0.4.3
colorlog==4.0.2
configparser==3.5.3
coverage==5.1
-croniter==0.3.31
+croniter==0.3.32
cryptography==2.9.2
cx-Oracle==7.3.0
datadog==0.36.0
@@ -104,7 +104,7 @@ execnet==1.7.1
fastavro==0.23.4
filelock==3.0.12
flake8-colors==0.1.6
-flake8==3.8.1
+flake8==3.8.3
flaky==3.6.1
flask-swagger==0.2.13
flower==0.9.4
@@ -114,27 +114,27 @@ funcsigs==1.0.2
future-fstrings==1.2.0
future==0.18.2
gcsfs==0.6.2
-google-api-core==1.17.0
-google-api-python-client==1.8.3
+google-api-core==1.19.1
+google-api-python-client==1.9.1
google-auth-httplib2==0.0.3
google-auth-oauthlib==0.4.1
-google-auth==1.15.0
-google-cloud-bigquery==1.24.0
+google-auth==1.16.1
+google-cloud-bigquery==1.25.0
google-cloud-bigtable==1.2.1
google-cloud-container==0.5.0
google-cloud-core==1.3.0
google-cloud-dlp==0.15.0
google-cloud-language==1.3.0
google-cloud-secret-manager==1.0.0
-google-cloud-spanner==1.16.0
+google-cloud-spanner==1.17.0
google-cloud-speech==1.3.2
google-cloud-storage==1.28.1
google-cloud-texttospeech==1.0.1
google-cloud-translate==2.0.1
google-cloud-videointelligence==1.14.0
google-cloud-vision==1.0.0
-google-resumable-media==0.5.0
-googleapis-common-protos==1.51.0
+google-resumable-media==0.5.1
+googleapis-common-protos==1.52.0
graphviz==0.14
grpc-google-iam-v1==0.12.3
grpcio-gcp==0.2.2
@@ -144,16 +144,16 @@ hdfs==2.5.8
hmsclient==0.1.1
httplib2==0.18.1
humanize==0.5.1
-hvac==0.10.2
-identify==1.4.16
+hvac==0.10.3
+identify==1.4.19
idna==2.9
ijson==2.6.1
imagesize==1.2.0
-importlib-metadata==1.6.0
-inflection==0.4.0
+importlib-metadata==1.6.1
+inflection==0.5.0
ipdb==0.13.2
ipython-genutils==0.2.0
-ipython==7.14.0
+ipython==7.15.0
iso8601==0.1.12
isodate==0.6.0
itsdangerous==1.1.0
@@ -169,33 +169,34 @@ jsonschema==3.2.0
junit-xml==1.9
jupyter-client==6.1.3
jupyter-core==4.6.3
-kombu==4.6.8
+kombu==4.6.10
kubernetes==11.0.0
-lazy-object-proxy==1.4.3
+lazy-object-proxy==1.5.0
ldap3==2.7
lockfile==0.12.2
marshmallow-enum==1.5.1
-marshmallow-sqlalchemy==0.23.0
+marshmallow-sqlalchemy==0.23.1
marshmallow==2.21.0
mccabe==0.6.1
mock==4.0.2
mongomock==3.19.0
more-itertools==8.3.0
moto==1.3.14
-msrest==0.6.14
+msrest==0.6.15
msrestazure==0.6.3
multi-key-dict==2.0.3
mypy-extensions==0.4.3
mypy==0.720
mysqlclient==1.3.14
+natsort==7.0.1
nbclient==0.3.1
nbformat==5.0.6
nest-asyncio==1.3.3
networkx==2.4
-nodeenv==1.3.5
+nodeenv==1.4.0
nteract-scrapbook==0.4.1
ntlm-auth==1.4.0
-numpy==1.18.4
+numpy==1.18.5
oauthlib==3.1.0
oscrypto==1.2.0
packaging==20.4
@@ -212,11 +213,11 @@ pexpect==4.8.0
pickleshare==0.7.5
pinotdb==0.1.1
pluggy==0.13.1
-pre-commit==2.4.0
+pre-commit==2.5.0
presto-python-client==0.7.0
prison==0.1.3
prompt-toolkit==3.0.5
-protobuf==3.12.1
+protobuf==3.12.2
psutil==5.7.0
psycopg2-binary==2.8.5
ptyprocess==0.6.0
@@ -237,13 +238,13 @@ pymssql==2.1.4
pyparsing==2.4.7
pyrsistent==0.16.0
pysftp==0.2.9
-pytest-cov==2.8.1
+pytest-cov==2.9.0
pytest-forked==1.1.3
pytest-instafail==0.4.1.post0
pytest-rerunfailures==9.0
pytest-timeout==1.3.4
pytest-xdist==1.32.0
-pytest==5.4.2
+pytest==5.4.3
python-daemon==2.1.2
python-dateutil==2.8.1
python-editor==1.0.4
@@ -257,9 +258,9 @@ pytz==2020.1
pytzdata==2019.3
pywinrm==0.4.1
pyzmq==19.0.1
-qds-sdk==1.15.2
-redis==3.5.2
-regex==2020.5.14
+qds-sdk==1.16.0
+redis==3.5.3
+regex==2020.6.8
requests-futures==0.9.4
requests-kerberos==0.12.0
requests-mock==1.8.0
@@ -278,7 +279,7 @@ setproctitle==1.1.10
six==1.15.0
slackclient==1.3.2
snowballstemmer==2.0.0
-snowflake-connector-python==2.2.6
+snowflake-connector-python==2.2.7
snowflake-sqlalchemy==1.2.3
soupsieve==2.0.1
sphinx-argparse==0.2.5
@@ -305,7 +306,7 @@ thrift-sasl==0.4.2
thrift==0.13.0
toml==0.10.1
tornado==5.1.1
-tqdm==4.46.0
+tqdm==4.46.1
traitlets==4.3.3
typed-ast==1.4.1
typing-extensions==3.7.4.2
@@ -316,7 +317,7 @@ urllib3==1.25.9
vertica-python==0.10.4
vine==1.3.0
virtualenv==20.0.21
-wcwidth==0.1.9
+wcwidth==0.2.4
websocket-client==0.57.0
wrapt==1.12.1
xmltodict==0.12.0
diff --git a/requirements/setup-2.7.md5 b/requirements/setup-2.7.md5
index 93454c3..fa9e77d 100644
--- a/requirements/setup-2.7.md5
+++ b/requirements/setup-2.7.md5
@@ -1 +1 @@
-4b8bf8ce351fd07604aa11e00ff2cba4 /opt/airflow/setup.py
+a42983dcaa7e536b9c343d1d59ae5b32 /opt/airflow/setup.py
diff --git a/requirements/setup-3.5.md5 b/requirements/setup-3.5.md5
index 93454c3..fa9e77d 100644
--- a/requirements/setup-3.5.md5
+++ b/requirements/setup-3.5.md5
@@ -1 +1 @@
-4b8bf8ce351fd07604aa11e00ff2cba4 /opt/airflow/setup.py
+a42983dcaa7e536b9c343d1d59ae5b32 /opt/airflow/setup.py
diff --git a/requirements/setup-3.6.md5 b/requirements/setup-3.6.md5
index 93454c3..fa9e77d 100644
--- a/requirements/setup-3.6.md5
+++ b/requirements/setup-3.6.md5
@@ -1 +1 @@
-4b8bf8ce351fd07604aa11e00ff2cba4 /opt/airflow/setup.py
+a42983dcaa7e536b9c343d1d59ae5b32 /opt/airflow/setup.py
diff --git a/requirements/setup-3.7.md5 b/requirements/setup-3.7.md5
index 93454c3..fa9e77d 100644
--- a/requirements/setup-3.7.md5
+++ b/requirements/setup-3.7.md5
@@ -1 +1 @@
-4b8bf8ce351fd07604aa11e00ff2cba4 /opt/airflow/setup.py
+a42983dcaa7e536b9c343d1d59ae5b32 /opt/airflow/setup.py
diff --git a/scripts/ci/ci_docs.sh b/scripts/ci/ci_docs.sh
index e84de9b..417f6a6 100755
--- a/scripts/ci/ci_docs.sh
+++ b/scripts/ci/ci_docs.sh
@@ -28,6 +28,9 @@ function run_docs() {
--env VERBOSE_COMMANDS \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
+ --env HOST_OS="$(uname -s)" \
+ --env HOST_HOME="${HOME}" \
+ --env HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/docs/build" \
diff --git a/scripts/ci/ci_fix_ownership.sh b/scripts/ci/ci_fix_ownership.sh
index 775cc58..81f5ac1 100755
--- a/scripts/ci/ci_fix_ownership.sh
+++ b/scripts/ci/ci_fix_ownership.sh
@@ -33,11 +33,14 @@ export AIRFLOW_IMAGE=${AIRFLOW_CI_IMAGE}
export WEBSERVER_HOST_PORT=28080
HOST_USER_ID="$(id -ur)"
HOST_GROUP_ID="$(id -gr)"
+HOST_OS="$(uname -s)"
export HOST_USER_ID
export HOST_GROUP_ID
+export HOST_OS
docker-compose \
-f "${MY_DIR}/docker-compose/base.yml" \
-f "${MY_DIR}/docker-compose/local.yml" \
+ -f "${MY_DIR}/docker-compose/forward-credentials.yml" \
run airflow /opt/airflow/scripts/ci/in_container/run_fix_ownership.sh
diff --git a/scripts/ci/ci_flake8.sh b/scripts/ci/ci_flake8.sh
index 5a2da56..fac9be4 100755
--- a/scripts/ci/ci_flake8.sh
+++ b/scripts/ci/ci_flake8.sh
@@ -31,6 +31,9 @@ function run_flake8() {
--env VERBOSE_COMMANDS \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
+ --env HOST_OS="$(uname -s)" \
+ --env HOST_HOME="${HOME}" \
+ --env HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \
@@ -43,6 +46,9 @@ function run_flake8() {
--env VERBOSE_COMMANDS \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
+ --env HOST_OS="$(uname -s)" \
+ --env HOST_HOME="${HOME}" \
+ --env HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" "${FILES[@]}" \
diff --git a/scripts/ci/ci_mypy.sh b/scripts/ci/ci_mypy.sh
index 8898d30..962d2c0 100755
--- a/scripts/ci/ci_mypy.sh
+++ b/scripts/ci/ci_mypy.sh
@@ -33,6 +33,9 @@ function run_mypy() {
--env VERBOSE_COMMANDS \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
+ --env HOST_OS="$(uname -s)" \
+ --env HOST_HOME="${HOME}" \
+ --env HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \
"-v" "${AIRFLOW_SOURCES}/.mypy_cache:/opt/airflow/.mypy_cache" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
diff --git a/scripts/ci/docker-compose/forward-credentials.yml b/scripts/ci/docker-compose/forward-credentials.yml
index 875b1ce..0eb0dbb 100644
--- a/scripts/ci/docker-compose/forward-credentials.yml
+++ b/scripts/ci/docker-compose/forward-credentials.yml
@@ -22,6 +22,8 @@ services:
# Useful for gcloud/aws/kubernetes etc. authorisation to be passed
# To inside docker. Use with care - your credentials will be available to
# Everything you install in Docker
+ # If you add it here - also add it to "in_container_fix_ownership" method in
+ # the _in_container_utils.sh file to make it friendly for Linux users
volumes:
- ${HOME}/.aws:/root/.aws:cached
- ${HOME}/.azure:/root/.azure:cached
diff --git a/scripts/ci/docker-compose/local-prod.yml b/scripts/ci/docker-compose/local-prod.yml
index ae8317d..a82b4f8 100644
--- a/scripts/ci/docker-compose/local-prod.yml
+++ b/scripts/ci/docker-compose/local-prod.yml
@@ -40,4 +40,6 @@ services:
- HOST_USER_ID
- HOST_GROUP_ID
- HOST_HOME=${HOME}
+ - HOST_AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
+ - HOST_OS
- PYTHONDONTWRITEBYTECODE
diff --git a/scripts/ci/docker-compose/local.yml b/scripts/ci/docker-compose/local.yml
index 9603417..3c9e40b 100644
--- a/scripts/ci/docker-compose/local.yml
+++ b/scripts/ci/docker-compose/local.yml
@@ -60,6 +60,8 @@ services:
- HOST_USER_ID
- HOST_GROUP_ID
- HOST_HOME=${HOME}
+ - HOST_AIRFLOW_SOURCES=${AIRFLOW_SOURCES}
+ - HOST_OS
- PYTHONDONTWRITEBYTECODE
ports:
- "${WEBSERVER_HOST_PORT}:8080"
diff --git a/scripts/ci/in_container/_in_container_utils.sh b/scripts/ci/in_container/_in_container_utils.sh
index 9197a04..5d9fbb6 100644
--- a/scripts/ci/in_container/_in_container_utils.sh
+++ b/scripts/ci/in_container/_in_container_utils.sh
@@ -89,13 +89,19 @@ function in_container_cleanup_pycache() {
#
# Fixes ownership of files generated in container - if they are owned by root, they will be owned by
-# The host user.
+# The host user. Only needed if the host is Linux - on Mac, ownership of files is automatically
+# changed to the Host user via osxfs filesystem
#
function in_container_fix_ownership() {
- set +o pipefail
- sudo find "${AIRFLOW_SOURCES}" -print0 -user root \
- | sudo xargs --null chown -v "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference >/dev/null 2>&1
- set -o pipefail
+ if [[ ${HOST_OS:=} == "Linux" ]]; then
+ set +o pipefail
+ echo "Fixing ownership of mounted files"
+ sudo find "${AIRFLOW_SOURCES}" -print0 -user root \
+ | sudo xargs --null chown "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference >/dev/null 2>&1
+ sudo find "/root/.aws" "/root/.azure" "/root/.config" "/root/.docker" -print0 -user root \
+ | sudo xargs --null chown "${HOST_USER_ID}.${HOST_GROUP_ID}" --no-dereference || true >/dev/null 2>&1
+ set -o pipefail
+ fi
}
function in_container_go_to_airflow_sources() {
diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh
index 9f86ba4..c8cd9f7 100644
--- a/scripts/ci/libraries/_initialization.sh
+++ b/scripts/ci/libraries/_initialization.sh
@@ -114,6 +114,13 @@ function initialize_common_environment {
HOST_GROUP_ID="$(id -gr)"
export HOST_GROUP_ID
+ # Set host OS. This is used to set the ownership properly when exiting
+ # The container on Linux - all files created inside docker are created with root user
+ # but they should be restored back to the host user
+ HOST_OS="$(uname -s)"
+ export HOST_OS
+
+
# Add the right volume mount for sources, depending which mount strategy is used
if [[ ${MOUNT_SOURCE_DIR_FOR_STATIC_CHECKS} == "true" ]]; then
print_info
diff --git a/scripts/ci/libraries/_runs.sh b/scripts/ci/libraries/_runs.sh
index cd5a44a..76b674d 100644
--- a/scripts/ci/libraries/_runs.sh
+++ b/scripts/ci/libraries/_runs.sh
@@ -25,6 +25,9 @@ function run_docs() {
--env VERBOSE_COMMANDS \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
+ --env HOST_OS="$(uname -s)" \
+ --env HOST_HOME="${HOME}" \
+ --env HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \
--rm \
"${AIRFLOW_CI_IMAGE}" \
"--" "/opt/airflow/docs/build" \
@@ -40,6 +43,9 @@ function run_generate_requirements() {
--env VERBOSE_COMMANDS \
--env HOST_USER_ID="$(id -ur)" \
--env HOST_GROUP_ID="$(id -gr)" \
+ --env HOST_OS="$(uname -s)" \
+ --env HOST_HOME="${HOME}" \
+ --env HOST_AIRFLOW_SOURCES="${AIRFLOW_SOURCES}" \
--env UPGRADE_WHILE_GENERATING_REQUIREMENTS \
--env PYTHON_MAJOR_MINOR_VERSION \
--env CHECK_REQUIREMENTS_ONLY \
diff --git a/scripts/ci/prepare_tool_scripts.sh b/scripts/ci/prepare_tool_scripts.sh
new file mode 100755
index 0000000..7a98c50
--- /dev/null
+++ b/scripts/ci/prepare_tool_scripts.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+set -euo pipefail
+
+function prepare_tool_script() {
+ IMAGE="${1}"
+ VOLUME="${2}"
+ TOOL="${3}"
+ COMMAND="${4:-}"
+
+ TARGET_TOOL_PATH="/usr/bin/${TOOL}"
+ TARGET_TOOL_UPDATE_PATH="/usr/bin/${TOOL}-update"
+
+ cat >"${TARGET_TOOL_PATH}" <<EOF
+#!/usr/bin/env bash
+docker run --rm -it \
+ -v "\${HOST_AIRFLOW_SOURCES}/tmp:/tmp" \
+ -v "\${HOST_AIRFLOW_SOURCES}/files:/files" \
+ -v "\${HOST_AIRFLOW_SOURCES}:/opt/airflow" \
+ -v "\${HOST_HOME}/${VOLUME}:/root/${VOLUME}" \
+ "${IMAGE}" ${COMMAND} "\$@"
+RES=\$?
+if [[ \${HOST_OS} == "Linux" ]]; then
+ docker run --rm \
+ -v "\${HOST_AIRFLOW_SOURCES}/tmp:/tmp" \
+ -v "\${HOST_AIRFLOW_SOURCES}/files:/files" \
+ -v "\${HOST_HOME}/${VOLUME}:/root/${VOLUME}" \
+ "\${AIRFLOW_CI_IMAGE}" bash -c \
+ "find '/tmp/' '/files/' '/root/${VOLUME}' -user root -print0 | xargs --null chown '\${HOST_USER_ID}.\${HOST_GROUP_ID}' --no-dereference" >/dev/null 2>&1
+fi
+exit \${RES}
+EOF
+
+ cat >"${TARGET_TOOL_UPDATE_PATH}" <<EOF
+#!/usr/bin/env bash
+docker pull "${IMAGE}"
+EOF
+
+ chmod a+x "${TARGET_TOOL_PATH}" "${TARGET_TOOL_UPDATE_PATH}"
+}
+
+GCLOUD_IMAGE="gcr.io/google.com/cloudsdktool/cloud-sdk:latest"
+
+prepare_tool_script "amazon/aws-cli:latest" ".aws" aws
+prepare_tool_script "mcr.microsoft.com/azure-cli:latest" ".azure" az az
+prepare_tool_script "${GCLOUD_IMAGE}" ".config/gcloud" bq bq
+prepare_tool_script "${GCLOUD_IMAGE}" ".config/gcloud" gcloud gcloud
+prepare_tool_script "${GCLOUD_IMAGE}" ".config/gcloud" gsutil gsutil