You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2021/02/03 21:34:40 UTC
[airflow] branch v1-10-stable updated: Created
DatabaseVersionCheckRule class (#13955)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch v1-10-stable
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v1-10-stable by this push:
new 8c239b7 Created DatabaseVersionCheckRule class (#13955)
8c239b7 is described below
commit 8c239b798cdd73cc8677cb694c873e48b595ca6b
Author: Dr. Dennis Akpenyi <de...@gmail.com>
AuthorDate: Wed Feb 3 22:34:26 2021 +0100
Created DatabaseVersionCheckRule class (#13955)
This upgrade check inspects the version of the supported database
backend (PostgreSQL, MySQL, and SQLite) so as to verify if the version
is supported in Airflow 2.0
This is ease upgrade to Airflow 2.0
closes: #13850
---
.../postgres_mysql_sqlite_version_upgrade_check.py | 58 +++++++++++
..._postgres_mysql_sqlite_version_upgrade_check.py | 115 +++++++++++++++++++++
2 files changed, 173 insertions(+)
diff --git a/airflow/upgrade/rules/postgres_mysql_sqlite_version_upgrade_check.py b/airflow/upgrade/rules/postgres_mysql_sqlite_version_upgrade_check.py
new file mode 100644
index 0000000..272f741
--- /dev/null
+++ b/airflow/upgrade/rules/postgres_mysql_sqlite_version_upgrade_check.py
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from packaging.version import Version
+
+from airflow.configuration import conf
+from airflow.upgrade.rules.base_rule import BaseRule
+from airflow.utils.db import provide_session
+
+
+class DatabaseVersionCheckRule(BaseRule):
+ title = "Check versions of PostgreSQL, MySQL, and SQLite to ease upgrade to Airflow 2.0"
+
+ description = """\
+From Airflow 2.0, the following database versions are supported:
+PostgreSQl - 9.6, 10, 11, 12, 13;
+MySQL - 5.7, 8;
+SQLite - 3.15+
+ """
+
+ @provide_session
+ def check(self, session=None):
+
+ more_info = "See link below for more details: https://github.com/apache/airflow#requirements"
+
+ conn_str = conf.get(section="core", key="sql_alchemy_conn")
+
+ if "sqlite" in conn_str:
+ min_req_sqlite_version = Version('3.15')
+ installed_sqlite_version = Version(session.execute('select sqlite_version();').scalar())
+ if installed_sqlite_version < min_req_sqlite_version:
+ return "From Airflow 2.0, SQLite version below 3.15 is no longer supported. \n" + more_info
+
+ elif "postgres" in conn_str:
+ min_req_postgres_version = Version('9.6')
+ installed_postgres_version = Version(session.execute('SHOW server_version;').scalar())
+ if installed_postgres_version < min_req_postgres_version:
+ return "From Airflow 2.0, PostgreSQL version below 9.6 is no longer supported. \n" + more_info
+
+ elif "mysql" in conn_str:
+ min_req_mysql_version = Version('5.7')
+ installed_mysql_version = Version(session.execute('SELECT VERSION();').scalar())
+ if installed_mysql_version < min_req_mysql_version:
+ return "From Airflow 2.0, MySQL version below 5.7 is no longer supported. \n" + more_info
diff --git a/tests/upgrade/rules/test_postgres_mysql_sqlite_version_upgrade_check.py b/tests/upgrade/rules/test_postgres_mysql_sqlite_version_upgrade_check.py
new file mode 100644
index 0000000..4ce7802
--- /dev/null
+++ b/tests/upgrade/rules/test_postgres_mysql_sqlite_version_upgrade_check.py
@@ -0,0 +1,115 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from unittest import TestCase
+
+from airflow.upgrade.rules.postgres_mysql_sqlite_version_upgrade_check import DatabaseVersionCheckRule
+from tests.compat import patch
+from tests.test_utils.config import conf_vars
+
+SQLITE_CONN = "sqlite:////home/user/airflow.db"
+POSTGRES_CONN = "postgresql+psycopg2://username:password@localhost:5432/airflow"
+MYSQL_CONN = "mysql+mysqldb://username:password@localhost/airflow"
+
+MOCK_MSG = "See link below for more details: https://github.com/apache/airflow#requirements"
+
+
+@patch('airflow.settings.Session')
+class TestDatabaseVersionCheckRule(TestCase):
+
+ @conf_vars({("core", "sql_alchemy_conn"): SQLITE_CONN})
+ def test_valid_sqlite_check(self, MockSession):
+ session = MockSession()
+ session.execute().scalar.return_value = '3.34.1'
+
+ rule = DatabaseVersionCheckRule()
+
+ assert isinstance(rule.title, str)
+ assert isinstance(rule.description, str)
+
+ msg = rule.check(session=session)
+ assert msg is None
+
+ @conf_vars({("core", "sql_alchemy_conn"): SQLITE_CONN})
+ def test_invalid_sqlite_check(self, MockSession):
+ session = MockSession()
+ session.execute().scalar.return_value = '2.25.2'
+
+ rule = DatabaseVersionCheckRule()
+
+ assert isinstance(rule.title, str)
+ assert isinstance(rule.description, str)
+
+ expected = "From Airflow 2.0, SQLite version below 3.15 is no longer supported. \n" + MOCK_MSG
+
+ msg = rule.check(session=session)
+ assert msg == expected
+
+ @conf_vars({("core", "sql_alchemy_conn"): POSTGRES_CONN})
+ def test_valid_postgres_check(self, MockSession):
+ session = MockSession()
+ session.execute().scalar.return_value = '12.3'
+
+ rule = DatabaseVersionCheckRule()
+
+ assert isinstance(rule.title, str)
+ assert isinstance(rule.description, str)
+
+ msg = rule.check(session=session)
+ assert msg is None
+
+ @conf_vars({("core", "sql_alchemy_conn"): POSTGRES_CONN})
+ def test_invalid_postgres_check(self, MockSession):
+ session = MockSession()
+ session.execute().scalar.return_value = '9.5'
+
+ rule = DatabaseVersionCheckRule()
+
+ assert isinstance(rule.title, str)
+ assert isinstance(rule.description, str)
+
+ expected = "From Airflow 2.0, PostgreSQL version below 9.6 is no longer supported. \n" + MOCK_MSG
+
+ msg = rule.check(session=session)
+ assert msg == expected
+
+ @conf_vars({("core", "sql_alchemy_conn"): MYSQL_CONN})
+ def test_valid_mysql_check(self, MockSession):
+ session = MockSession()
+ session.execute().scalar.return_value = '8.0.23'
+
+ rule = DatabaseVersionCheckRule()
+
+ assert isinstance(rule.title, str)
+ assert isinstance(rule.description, str)
+
+ msg = rule.check(session=session)
+ assert msg is None
+
+ @conf_vars({("core", "sql_alchemy_conn"): MYSQL_CONN})
+ def test_invalid_mysql_check(self, MockSession):
+ session = MockSession()
+ session.execute().scalar.return_value = '5.6.11'
+
+ rule = DatabaseVersionCheckRule()
+
+ assert isinstance(rule.title, str)
+ assert isinstance(rule.description, str)
+
+ expected = "From Airflow 2.0, MySQL version below 5.7 is no longer supported. \n" + MOCK_MSG
+
+ msg = rule.check(session=session)
+ assert msg == expected