You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2020/11/21 00:04:30 UTC
[airflow] 08/09: Add upgrade check rule to ensure on "latest"
versions (#12514)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 41bb8f28f1febbd80109435fdc1dcfc0dce44a76
Author: Ash Berlin-Taylor <as...@firemirror.com>
AuthorDate: Fri Nov 20 21:21:13 2020 +0000
Add upgrade check rule to ensure on "latest" versions (#12514)
This checks against PyPI to make sure this is run with the latest
non-preview release of apache-airflow-upgrade-check, and the latest
1.10.x of apache-airflow
(cherry picked from commit d7ace0267c34bd5520d321a495399710c1c49cd1)
---
airflow/upgrade/rules/__init__.py | 2 +-
airflow/upgrade/rules/aaa_airflow_version_check.py | 87 ++++++++++++++++++++++
setup.py | 2 +
.../rules/test_aaa_airflow_version_check.py | 75 +++++++++++++++++++
4 files changed, 165 insertions(+), 1 deletion(-)
diff --git a/airflow/upgrade/rules/__init__.py b/airflow/upgrade/rules/__init__.py
index 4735c7f..97d0160 100644
--- a/airflow/upgrade/rules/__init__.py
+++ b/airflow/upgrade/rules/__init__.py
@@ -21,7 +21,7 @@ def get_rules():
"""Automatically discover all rules"""
rule_classes = []
path = os.path.dirname(os.path.abspath(__file__))
- for file in os.listdir(path):
+ for file in sorted(os.listdir(path)):
if not file.endswith(".py") or file in ("__init__.py", "base_rule.py"):
continue
py_file = file[:-3]
diff --git a/airflow/upgrade/rules/aaa_airflow_version_check.py b/airflow/upgrade/rules/aaa_airflow_version_check.py
new file mode 100644
index 0000000..ad84eb4
--- /dev/null
+++ b/airflow/upgrade/rules/aaa_airflow_version_check.py
@@ -0,0 +1,87 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This module starts with `aaa_` so that it is sorted first alphabetically, but is still a valid python module
+# name (starting with digitis is not valid)
+
+from __future__ import absolute_import
+
+from packaging.version import Version
+import requests
+
+from airflow.upgrade.rules.base_rule import BaseRule
+
+try:
+ import importlib.metadata as importlib_metadata
+except ImportError:
+ import importlib_metadata
+
+
+class VersionCheckRule(BaseRule):
+
+ title = "Check for latest versions of apache-airflow and checker"
+
+ description = """\
+Check that the latest version of apache-airflow-upgrade-check is installed, and
+that you are on the latest 1.10.x release of apache-airflow."""
+
+ def pypi_releases(self, distname):
+ """
+ Get all the non-dev releases of a dist from PyPI
+ """
+
+ resp = requests.get("https://pypi.org/pypi/{}/json".format(distname))
+ resp.raise_for_status()
+
+ for rel_string in resp.json()["releases"].keys():
+ ver = Version(rel_string)
+ if ver.is_devrelease or ver.is_prerelease:
+ continue
+ yield ver
+
+ def check(self):
+
+ current_airflow_version = Version(__import__("airflow").__version__)
+ try:
+ upgrade_check_ver = Version(
+ importlib_metadata.distribution("apache-airflow-upgrade-check").version,
+ )
+ except importlib_metadata.PackageNotFoundError:
+ upgrade_check_ver = Version("0.0.0")
+
+ try:
+ latest_airflow_v1_release = sorted(
+ filter(lambda v: v.major == 1, self.pypi_releases("apache-airflow"))
+ )[-1]
+
+ if current_airflow_version < latest_airflow_v1_release:
+ yield (
+ "There is a more recent version of apache-airflow. Please upgrade to {} and re-run this"
+ " script"
+ ).format(latest_airflow_v1_release)
+
+ latest_upgrade_check_release = sorted(
+ self.pypi_releases("apache-airflow-upgrade-check")
+ )[-1]
+
+ if upgrade_check_ver < latest_upgrade_check_release:
+ yield (
+ "There is a more recent version of apache-airflow-upgrade-check. Please upgrade to {}"
+ " and re-run this script"
+ ).format(latest_upgrade_check_release)
+ except Exception as e:
+ yield "Unable to go ask PyPI.org for latest release information: " + str(e)
diff --git a/setup.py b/setup.py
index 30917d2..9617ac7 100644
--- a/setup.py
+++ b/setup.py
@@ -603,6 +603,7 @@ INSTALL_REQUIREMENTS = [
'future>=0.16.0, <0.19',
'graphviz>=0.12',
'gunicorn>=19.5.0, <21.0',
+ 'importlib-metadata~=2.0; python_version<"3.8"',
'iso8601>=0.1.12',
'jinja2>=2.10.1, <2.12.0',
'json-merge-patch==0.2',
@@ -611,6 +612,7 @@ INSTALL_REQUIREMENTS = [
'markdown>=2.5.2, <3.0',
'marshmallow-sqlalchemy>=0.16.1, <0.24.0;python_version>="3.6"',
'marshmallow-sqlalchemy>=0.16.1, <0.19.0;python_version<"3.6"',
+ 'packaging',
'pandas>=0.17.1, <2.0',
'pendulum==1.4.4',
'pep562~=1.0;python_version<"3.7"',
diff --git a/tests/upgrade/rules/test_aaa_airflow_version_check.py b/tests/upgrade/rules/test_aaa_airflow_version_check.py
new file mode 100644
index 0000000..1c4fce0
--- /dev/null
+++ b/tests/upgrade/rules/test_aaa_airflow_version_check.py
@@ -0,0 +1,75 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from airflow.upgrade.rules.aaa_airflow_version_check import VersionCheckRule
+
+
+class TestVersionCheckRule:
+ def test_new_airflow_version_available(self, requests_mock):
+ requests_mock.get(
+ "https://pypi.org/pypi/apache-airflow/json",
+ json={
+ "releases": {
+ # The values here don't matter
+ "1.10.22b1": None,
+ "1.10.20": None,
+ "1.10.21": None,
+ }
+ },
+ )
+ requests_mock.get(
+ "https://pypi.org/pypi/apache-airflow-upgrade-check/json",
+ json={
+ "releases": {
+ "0.0.0": None,
+ }
+ },
+ )
+
+ results = list(VersionCheckRule().check())
+ assert len(results) == 1, results
+
+ msg = results[0]
+ assert " 1.10.21 " in msg
+ assert "apache-airflow" in msg
+ assert "apache-airflow-upgrade-check" not in msg
+
+ def test_new_airflow_upgrade_check_version_available(self, requests_mock):
+ requests_mock.get(
+ "https://pypi.org/pypi/apache-airflow/json",
+ json={
+ "releases": {
+ # The values here don't matter
+ "1.10.0": None,
+ }
+ },
+ )
+ requests_mock.get(
+ "https://pypi.org/pypi/apache-airflow-upgrade-check/json",
+ json={
+ "releases": {
+ "99.0.0": None,
+ }
+ },
+ )
+
+ results = list(VersionCheckRule().check())
+ assert len(results) == 1, results
+
+ msg = results[0]
+ assert " 99.0.0 " in msg
+ assert "apache-airflow-upgrade-check" in msg