You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2020/11/29 20:11:33 UTC
[airflow] branch v1-10-test updated (55cef31 -> f1b0cfd)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a change to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git.
from 55cef31 fixup! Use AIRFLOW_CONSTRAINTS_LOCATION when passed during docker build (#12604)
new f71934d Update setup.py to get non-conflicting set of dependencies (#12636)
new 2182972 fixup! Support creation of configmaps & secrets and extra env & envFrom configuration in Helm Chart (#12164)
new 15e759e fixup! Update setup.py to get non-conflicting set of dependencies (#12636)
new f1b0cfd fixup! fixup! Update setup.py to get non-conflicting set of dependencies (#12636)
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
chart/tests/test_extra_env_env_from.py | 32 +++++++++++++-------------
setup.py | 41 ++++++++++++++++++++++++----------
2 files changed, 45 insertions(+), 28 deletions(-)
[airflow] 02/04: fixup! Support creation of configmaps & secrets
and extra env & envFrom configuration in Helm Chart (#12164)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 218297242648b0b7bc52789bae334bd2b45bf498
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Nov 29 20:13:32 2020 +0100
fixup! Support creation of configmaps & secrets and extra env & envFrom configuration in Helm Chart (#12164)
---
chart/tests/test_extra_env_env_from.py | 32 ++++++++++++++++----------------
1 file changed, 16 insertions(+), 16 deletions(-)
diff --git a/chart/tests/test_extra_env_env_from.py b/chart/tests/test_extra_env_env_from.py
index 7e1b28d..170fc7a 100644
--- a/chart/tests/test_extra_env_env_from.py
+++ b/chart/tests/test_extra_env_env_from.py
@@ -29,29 +29,29 @@ RELEASE_NAME = "TEST-EXTRA-ENV-ENV-FROM"
# Test Params: k8s object key and paths with expected env / envFrom
PARAMS = [
(
- ("Job", f"{RELEASE_NAME}-create-user"),
+ ("Job", "{}-create-user".format(RELEASE_NAME)),
("spec.template.spec.containers[0]",),
),
(
- ("Job", f"{RELEASE_NAME}-run-airflow-migrations"),
+ ("Job", "{}-run-airflow-migrations".format(RELEASE_NAME)),
("spec.template.spec.containers[0]",),
),
(
- ("Deployment", f"{RELEASE_NAME}-scheduler"),
+ ("Deployment", "{}-scheduler".format(RELEASE_NAME)),
(
"spec.template.spec.initContainers[0]",
"spec.template.spec.containers[0]",
),
),
(
- ("StatefulSet", f"{RELEASE_NAME}-worker"),
+ ("StatefulSet", "{}-worker".format(RELEASE_NAME)),
(
"spec.template.spec.initContainers[0]",
"spec.template.spec.containers[0]",
),
),
(
- ("Deployment", f"{RELEASE_NAME}-webserver"),
+ ("Deployment", "{}-webserver".format(RELEASE_NAME)),
("spec.template.spec.initContainers[0]", "spec.template.spec.containers[0]"),
),
]
@@ -79,39 +79,39 @@ class ExtraEnvEnvFromTest(unittest.TestCase):
"""
)
values = yaml.safe_load(values_str)
- cls.k8s_objects = render_chart(RELEASE_NAME, values=values)
- cls.k8s_objects_by_key = prepare_k8s_lookup_dict(cls.k8s_objects)
+ cls.k8s_objects = render_chart(RELEASE_NAME, values=values) # type: ignore
+ cls.k8s_objects_by_key = prepare_k8s_lookup_dict(cls.k8s_objects) # type: ignore
@parameterized.expand(PARAMS)
def test_extra_env(self, k8s_obj_key, env_paths):
expected_env_as_str = textwrap.dedent(
- f"""
+ """
- name: PLATFORM
value: FR
- name: TEST
valueFrom:
secretKeyRef:
key: connection
- name: {RELEASE_NAME}-some-secret
- """
+ name: {}-some-secret
+ """.format(RELEASE_NAME)
).lstrip()
k8s_object = self.k8s_objects_by_key[k8s_obj_key]
for path in env_paths:
- env = jmespath.search(f"{path}.env", k8s_object)
+ env = jmespath.search("{}.env".format(path), k8s_object)
self.assertIn(expected_env_as_str, yaml.dump(env))
@parameterized.expand(PARAMS)
def test_extra_env_from(self, k8s_obj_key, env_from_paths):
expected_env_from_as_str = textwrap.dedent(
- f"""
+ """
- secretRef:
- name: {RELEASE_NAME}-airflow-connections
+ name: {}-airflow-connections
- configMapRef:
- name: {RELEASE_NAME}-airflow-variables
- """
+ name: {}-airflow-variables
+ """.format(RELEASE_NAME, RELEASE_NAME)
).lstrip()
k8s_object = self.k8s_objects_by_key[k8s_obj_key]
for path in env_from_paths:
- env_from = jmespath.search(f"{path}.envFrom", k8s_object)
+ env_from = jmespath.search("{}.envFrom".format(path), k8s_object)
self.assertIn(expected_env_from_as_str, yaml.dump(env_from))
[airflow] 04/04: fixup! fixup! Update setup.py to get
non-conflicting set of dependencies (#12636)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit f1b0cfdbd878f9982cd007d7747ad702b93d65b8
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Nov 29 21:10:03 2020 +0100
fixup! fixup! Update setup.py to get non-conflicting set of dependencies (#12636)
---
setup.py | 25 +++++++++++++++++--------
1 file changed, 17 insertions(+), 8 deletions(-)
diff --git a/setup.py b/setup.py
index a19afef..4f566f6 100644
--- a/setup.py
+++ b/setup.py
@@ -182,7 +182,7 @@ atlas = [
'atlasclient>=0.1.2',
]
aws = [
- 'boto3~=1.10,<1.16',
+ 'boto3~=1.10,<1.11', # required by snowflake
]
azure_blob_storage = [
'azure-storage>=0.34.0, <0.37.0',
@@ -208,8 +208,8 @@ cassandra = [
'cassandra-driver>=3.13.0,<3.21.0',
]
celery = [
- 'celery~=4.3',
- 'celery==4.4.0;python_version<"3.0"',
+ 'celery~=4.3;python_version>="3.0"',
+ 'celery==4.3.1;python_version<"3.0"',
'flower>=0.7.3, <1.0',
'kombu==4.6.3;python_version<"3.0"',
'tornado>=4.2.0, <6.0', # Dep of flower. Pin to a version that works on Py3.5.2
@@ -224,7 +224,8 @@ cloudant = [
crypto = [
# Cryptography 3.2 for python 2.7 is broken
# https://github.com/pyca/cryptography/issues/5359#issuecomment-727622403
- 'cryptography>=0.9.3,<3.2; python_version<"3.0"',
+ # Snowflake requires <3.0
+ 'cryptography>=0.9.3,<3.0; python_version<"3.0"',
'cryptography>=0.9.3;python_version>="3.0"',
]
dask = [
@@ -262,7 +263,8 @@ flask_oauth = [
'requests-oauthlib==1.1.0',
]
gcp = [
- 'PyOpenSSL',
+ 'PyOpenSSL<20.0.0;python_version<"3.0"',
+ 'PyOpenSSL;python_version>="3.0"',
'google-api-python-client>=1.6.0, <2.0.0',
'google-auth>=1.0.0, <2.0.0',
'google-auth-httplib2>=0.0.1',
@@ -426,6 +428,8 @@ devel = [
'click==6.7',
'contextdecorator;python_version<"3.4"',
'coverage',
+ 'docutils>=0.14, <0.16; python_version<"3.0"',
+ 'docutils;python_version>="3.0"',
'ecdsa<0.15', # Required for moto 1.3.14
'flake8>=3.6.0',
'flake8-colors',
@@ -445,11 +449,12 @@ devel = [
'paramiko',
'pipdeptree',
'pre-commit',
+ 'pyrsistent<=0.16.0;python_version<"3.0"',
+ 'pyrsistent;python_version>="3.0"',
'pysftp',
'pytest<6.0.0', # FIXME: pylint complaining for pytest.mark.* on v6.0
'pytest-cov',
'pytest-instafail',
- 'pytest-rerunfailures',
'pytest-timeouts',
'pywinrm',
'qds-sdk>=1.9.6',
@@ -597,6 +602,8 @@ INSTALL_REQUIREMENTS = [
'colorlog==4.0.2',
'configparser>=3.5.0, <3.6.0',
'croniter>=0.3.17, <0.4',
+ 'cryptography>=0.9.3,<3.0; python_version<"3.0"', # required by snowflake
+ 'cryptography>=0.9.3;python_version>="3.0"',
'dill>=0.2.2, <0.4',
'email-validator',
'enum34~=1.1.6;python_version<"3.4"',
@@ -632,14 +639,16 @@ INSTALL_REQUIREMENTS = [
'python-dateutil>=2.3, <3',
'python-nvd3~=0.15.0',
'python-slugify>=3.0.0,<5.0',
- 'requests>=2.20.0, <2.24.0', # Required to keep snowflake happy
+ 'requests>=2.20.0, <2.23.0;python_version<"3.0"', # Required to keep snowflake happy
+ 'requests>=2.20.0, <2.24.0;python_version>="3.0"', # Required to keep snowflake happy
'setproctitle>=1.1.8, <2',
'sqlalchemy~=1.3',
'sqlalchemy_jsonfield==0.8.0;python_version<"3.5"',
'sqlalchemy_jsonfield~=0.9;python_version>="3.5"',
'tabulate>=0.7.5, <0.9',
'tenacity==4.12.0',
- 'thrift>=0.9.2',
+ 'thrift>=0.9.2;python_version>="3.0"',
+ 'thrift==0.9.3;python_version<"3.0"', # required by thrifts_sasl for python 2.0
'typing;python_version<"3.5"',
'typing-extensions>=3.7.4;python_version<"3.8"',
'tzlocal>=1.4,<2.0.0',
[airflow] 03/04: fixup! Update setup.py to get non-conflicting set
of dependencies (#12636)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 15e759e409b665555a23de3b4fa0e81d6ae013d0
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Nov 29 20:36:58 2020 +0100
fixup! Update setup.py to get non-conflicting set of dependencies (#12636)
---
setup.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/setup.py b/setup.py
index c3c250e..a19afef 100644
--- a/setup.py
+++ b/setup.py
@@ -209,6 +209,7 @@ cassandra = [
]
celery = [
'celery~=4.3',
+ 'celery==4.4.0;python_version<"3.0"',
'flower>=0.7.3, <1.0',
'kombu==4.6.3;python_version<"3.0"',
'tornado>=4.2.0, <6.0', # Dep of flower. Pin to a version that works on Py3.5.2
[airflow] 01/04: Update setup.py to get non-conflicting set of
dependencies (#12636)
Posted by po...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit f71934db9adb0f0ab14e12d746d5ea07d5358aa7
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Nov 29 19:45:58 2020 +0100
Update setup.py to get non-conflicting set of dependencies (#12636)
This change upgrades setup.py and setup.cfg to provide non-conflicting
`pip check` valid set of constraints for CI image.
---
setup.py | 21 ++++++++++++++-------
1 file changed, 14 insertions(+), 7 deletions(-)
diff --git a/setup.py b/setup.py
index f5f2a53..c3c250e 100644
--- a/setup.py
+++ b/setup.py
@@ -182,11 +182,12 @@ atlas = [
'atlasclient>=0.1.2',
]
aws = [
- 'boto3~=1.10',
+ 'boto3~=1.10,<1.16',
]
azure_blob_storage = [
'azure-storage>=0.34.0, <0.37.0',
- 'azure-storage-blob<12.0',
+ 'azure-storage-blob',
+ 'azure-storage-common',
]
azure_container_instances = [
'azure-mgmt-containerinstance>=1.5.0,<2'
@@ -336,7 +337,9 @@ papermill = [
'papermill[all]>=1.0.0',
'nteract-scrapbook[all]>=0.2.1',
'pyarrow<1.0.0',
- 'fsspec<0.8.0;python_version=="3.5"'
+ 'fsspec<0.8.0;python_version=="3.5"',
+ 'black==20.8b0;python_version>="3.6"' # we need to limit black version as we have click < 7
+
]
password = [
'bcrypt>=2.0.0',
@@ -355,7 +358,7 @@ qds = [
'qds-sdk>=1.10.4',
]
rabbitmq = [
- 'amqp',
+ 'amqp<5.0.0',
]
redis = [
'redis~=3.2',
@@ -378,6 +381,7 @@ sentry = [
]
slack = [
'slackclient>=1.0.0,<2.0.0',
+ 'websocket-client<0.55.0'
]
snowflake = [
'snowflake-connector-python>=1.5.2',
@@ -421,11 +425,13 @@ devel = [
'click==6.7',
'contextdecorator;python_version<"3.4"',
'coverage',
+ 'ecdsa<0.15', # Required for moto 1.3.14
'flake8>=3.6.0',
'flake8-colors',
'flaky',
'freezegun',
'gitpython',
+ 'idna<2.9', # Required for moto 1.3.14
'importlib-metadata~=2.0; python_version<"3.8"',
'ipdb',
'jira',
@@ -436,6 +442,7 @@ devel = [
'packaging',
'parameterized',
'paramiko',
+ 'pipdeptree',
'pre-commit',
'pysftp',
'pytest<6.0.0', # FIXME: pylint complaining for pytest.mark.* on v6.0
@@ -443,7 +450,6 @@ devel = [
'pytest-instafail',
'pytest-rerunfailures',
'pytest-timeouts',
- 'pytest-xdist',
'pywinrm',
'qds-sdk>=1.9.6',
'requests_mock',
@@ -606,11 +612,12 @@ INSTALL_REQUIREMENTS = [
'graphviz>=0.12',
'gunicorn>=19.5.0, <21.0',
'importlib-metadata~=2.0; python_version<"3.8"',
+ 'importlib_resources~=1.4',
'iso8601>=0.1.12',
'jinja2>=2.10.1, <2.12.0',
'json-merge-patch==0.2',
'jsonschema~=3.0',
- 'lazy_object_proxy~=1.3',
+ 'lazy_object_proxy<1.5.0', # Required to keep pip-check happy with astroid
'markdown>=2.5.2, <3.0',
'marshmallow-sqlalchemy>=0.16.1, <0.24.0;python_version>="3.6"',
'marshmallow-sqlalchemy>=0.16.1, <0.19.0;python_version<"3.6"',
@@ -624,7 +631,7 @@ INSTALL_REQUIREMENTS = [
'python-dateutil>=2.3, <3',
'python-nvd3~=0.15.0',
'python-slugify>=3.0.0,<5.0',
- 'requests>=2.20.0, <3',
+ 'requests>=2.20.0, <2.24.0', # Required to keep snowflake happy
'setproctitle>=1.1.8, <2',
'sqlalchemy~=1.3',
'sqlalchemy_jsonfield==0.8.0;python_version<"3.5"',