You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/01/30 11:50:43 UTC
[airflow] branch main updated: Replaces the usage of postgres:// with postgresql:// (#21205)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new e07139a Replaces the usage of postgres:// with postgresql:// (#21205)
e07139a is described below
commit e07139ab73ebf8d7ebd46655a304708f13a440ac
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Sun Jan 30 12:50:03 2022 +0100
Replaces the usage of postgres:// with postgresql:// (#21205)
After releasing 3.4.4 we can finally migrate to SQLAlchemy 1.4,
however SQLAlchemy 1.4 removed the use of postgres:// as valid
specification for Postgres DB SQL (leaving only postgresql://)
Due to that we need to change:
* postgres provider to return postgresql:// with get_db_uri()
* fix a number of tests that expected postgres://
We cannot do much if someone uses postgres:// specification.
Technically it might be seen as breaking change, but this is not
an airflow breaking change and users could still use SQLAlchemy
1.3 to keep the old prefix, so we can introduce this change
in Airflow without raising the major version.
Details in the [SQLAlchemy Changelog](https://docs.sqlalchemy.org/en/14/changelog/changelog_14.html#change-3687655465c25a39b968b4f5f6e9170b).
---
UPDATING.md | 19 ++++++++++++++++++-
airflow/providers/postgres/hooks/postgres.py | 3 +--
docs/apache-airflow/howto/set-up-database.rst | 18 ++++++++++++++++++
tests/models/test_base.py | 4 ++--
tests/models/test_connection.py | 24 ++++++++++++------------
tests/providers/postgres/hooks/test_postgres.py | 6 +++---
6 files changed, 54 insertions(+), 20 deletions(-)
diff --git a/UPDATING.md b/UPDATING.md
index 29e9a0f..a4e2e17 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -80,7 +80,24 @@ https://developers.google.com/style/inclusive-documentation
-->
-### Passing ``execution_date`` to ``XCom.set()``, ``XCom.clear()``, ``XCom.get_one()``, and ``XCom.get_many()`` is deprecated
+### You have to use `postgresql://` instead of `postgres://` in `sql_alchemy_conn` for SQLAlchemy 1.4.0+
+
+When you use SQLAlchemy 1.4.0+, you need ot use `postgresql://` as the database in the `sql_alchemy_conn`.
+In the previous versions of SQLAlchemy it was possible to use `postgres://`, but using it in
+SQLAlchemy 1.4.0+ results in:
+
+```
+> raise exc.NoSuchModuleError(
+ "Can't load plugin: %s:%s" % (self.group, name)
+ )
+E sqlalchemy.exc.NoSuchModuleError: Can't load plugin: sqlalchemy.dialects:postgres
+```
+
+If you cannot change the prefix of your URL immediately, Airflow continues to work with SQLAlchemy
+1.3 and you can downgrade SQLAlchemy, but we recommend to update the prefix.
+Details in the [SQLAlchemy Changelog](https://docs.sqlalchemy.org/en/14/changelog/changelog_14.html#change-3687655465c25a39b968b4f5f6e9170b).
+
+### Passing `execution_date` to `XCom.set()`, `XCom.clear()`, `XCom.get_one()`, and `XCom.get_many()` is deprecated
Continuing the effort to bind TaskInstance to a DagRun, XCom entries are now also tied to a DagRun. Use the ``run_id`` argument to specify the DagRun instead.
diff --git a/airflow/providers/postgres/hooks/postgres.py b/airflow/providers/postgres/hooks/postgres.py
index 4b495f4..a707f9c 100644
--- a/airflow/providers/postgres/hooks/postgres.py
+++ b/airflow/providers/postgres/hooks/postgres.py
@@ -15,7 +15,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-
import os
from contextlib import closing
from copy import deepcopy
@@ -139,7 +138,7 @@ class PostgresHook(DbApiHook):
def get_uri(self) -> str:
conn = self.get_connection(getattr(self, self.conn_name_attr))
- uri = super().get_uri()
+ uri = super().get_uri().replace("postgres://", "postgresql://")
if conn.extra_dejson.get('client_encoding', False):
charset = conn.extra_dejson["client_encoding"]
return f"{uri}?client_encoding={charset}"
diff --git a/docs/apache-airflow/howto/set-up-database.rst b/docs/apache-airflow/howto/set-up-database.rst
index a155880..c1e6ccb 100644
--- a/docs/apache-airflow/howto/set-up-database.rst
+++ b/docs/apache-airflow/howto/set-up-database.rst
@@ -169,6 +169,24 @@ the database configuration to load your change. See
`The pg_hba.conf File <https://www.postgresql.org/docs/current/auth-pg-hba-conf.html>`__
in the Postgres documentation to learn more.
+.. warning::
+
+ When you use SQLAlchemy 1.4.0+, you need ot use ``postgresql://`` as the database in the ``sql_alchemy_conn``.
+ In the previous versions of SQLAlchemy it was possible to use ``postgres://``, but using it in
+ SQLAlchemy 1.4.0+ results in:
+
+ .. code-block::
+
+ > raise exc.NoSuchModuleError(
+ "Can't load plugin: %s:%s" % (self.group, name)
+ )
+ E sqlalchemy.exc.NoSuchModuleError: Can't load plugin: sqlalchemy.dialects:postgres
+
+ If you cannot change the prefix of your URL immediately, Airflow continues to work with SQLAlchemy
+ 1.3 and you can downgrade SQLAlchemy, but we recommend to update the prefix.
+
+ Details in the `SQLAlchemy Changelog <https://docs.sqlalchemy.org/en/14/changelog/changelog_14.html#change-3687655465c25a39b968b4f5f6e9170b>`_.
+
We recommend using the ``psycopg2`` driver and specifying it in your SqlAlchemy connection string.
.. code-block:: text
diff --git a/tests/models/test_base.py b/tests/models/test_base.py
index 5579986..283fa69 100644
--- a/tests/models/test_base.py
+++ b/tests/models/test_base.py
@@ -25,7 +25,7 @@ from tests.test_utils.config import conf_vars
@pytest.mark.parametrize(
("dsn", "expected", "extra"),
[
- param("postgres://host/the_database", {}, {}, id="postgres"),
+ param("postgresql://host/the_database", {}, {}, id="postgres"),
param("mysql://host/the_database", {"collation": "utf8mb3_bin"}, {}, id="mysql"),
param("mysql+pymsql://host/the_database", {"collation": "utf8mb3_bin"}, {}, id="mysql+pymsql"),
param(
@@ -35,7 +35,7 @@ from tests.test_utils.config import conf_vars
id="mysql with explicit config",
),
param(
- "postgres://host/the_database",
+ "postgresql://host/the_database",
{"collation": "ascii"},
{('core', 'sql_engine_collation_for_ids'): 'ascii'},
id="postgres with explicit config",
diff --git a/tests/models/test_connection.py b/tests/models/test_connection.py
index 117e078..a527381 100644
--- a/tests/models/test_connection.py
+++ b/tests/models/test_connection.py
@@ -501,7 +501,7 @@ class TestConnection(unittest.TestCase):
@mock.patch.dict(
'os.environ',
{
- 'AIRFLOW_CONN_TEST_URI': 'postgres://username:password@ec2.compute.com:5432/the_database',
+ 'AIRFLOW_CONN_TEST_URI': 'postgresql://username:password@ec2.compute.com:5432/the_database',
},
)
def test_using_env_var(self):
@@ -517,7 +517,7 @@ class TestConnection(unittest.TestCase):
@mock.patch.dict(
'os.environ',
{
- 'AIRFLOW_CONN_TEST_URI_NO_CREDS': 'postgres://ec2.compute.com/the_database',
+ 'AIRFLOW_CONN_TEST_URI_NO_CREDS': 'postgresql://ec2.compute.com/the_database',
},
)
def test_using_unix_socket_env_var(self):
@@ -550,7 +550,7 @@ class TestConnection(unittest.TestCase):
with mock.patch.dict(
'os.environ',
{
- 'AIRFLOW_CONN_AIRFLOW_DB': 'postgres://username:password@ec2.compute.com:5432/the_database',
+ 'AIRFLOW_CONN_AIRFLOW_DB': 'postgresql://username:password@ec2.compute.com:5432/the_database',
},
):
conn = SqliteHook.get_connection(conn_id='airflow_db')
@@ -563,23 +563,23 @@ class TestConnection(unittest.TestCase):
@mock.patch.dict(
'os.environ',
{
- 'AIRFLOW_CONN_TEST_URI': 'postgres://username:password@ec2.compute.com:5432/the_database',
- 'AIRFLOW_CONN_TEST_URI_NO_CREDS': 'postgres://ec2.compute.com/the_database',
+ 'AIRFLOW_CONN_TEST_URI': 'postgresql://username:password@ec2.compute.com:5432/the_database',
+ 'AIRFLOW_CONN_TEST_URI_NO_CREDS': 'postgresql://ec2.compute.com/the_database',
},
)
def test_dbapi_get_uri(self):
conn = BaseHook.get_connection(conn_id='test_uri')
hook = conn.get_hook()
- assert 'postgres://username:password@ec2.compute.com:5432/the_database' == hook.get_uri()
+ assert 'postgresql://username:password@ec2.compute.com:5432/the_database' == hook.get_uri()
conn2 = BaseHook.get_connection(conn_id='test_uri_no_creds')
hook2 = conn2.get_hook()
- assert 'postgres://ec2.compute.com/the_database' == hook2.get_uri()
+ assert 'postgresql://ec2.compute.com/the_database' == hook2.get_uri()
@mock.patch.dict(
'os.environ',
{
- 'AIRFLOW_CONN_TEST_URI': 'postgres://username:password@ec2.compute.com:5432/the_database',
- 'AIRFLOW_CONN_TEST_URI_NO_CREDS': 'postgres://ec2.compute.com/the_database',
+ 'AIRFLOW_CONN_TEST_URI': 'postgresql://username:password@ec2.compute.com:5432/the_database',
+ 'AIRFLOW_CONN_TEST_URI_NO_CREDS': 'postgresql://ec2.compute.com/the_database',
},
)
def test_dbapi_get_sqlalchemy_engine(self):
@@ -587,13 +587,13 @@ class TestConnection(unittest.TestCase):
hook = conn.get_hook()
engine = hook.get_sqlalchemy_engine()
assert isinstance(engine, sqlalchemy.engine.Engine)
- assert 'postgres://username:password@ec2.compute.com:5432/the_database' == str(engine.url)
+ assert 'postgresql://username:password@ec2.compute.com:5432/the_database' == str(engine.url)
@mock.patch.dict(
'os.environ',
{
- 'AIRFLOW_CONN_TEST_URI': 'postgres://username:password@ec2.compute.com:5432/the_database',
- 'AIRFLOW_CONN_TEST_URI_NO_CREDS': 'postgres://ec2.compute.com/the_database',
+ 'AIRFLOW_CONN_TEST_URI': 'postgresql://username:password@ec2.compute.com:5432/the_database',
+ 'AIRFLOW_CONN_TEST_URI_NO_CREDS': 'postgresql://ec2.compute.com/the_database',
},
)
def test_get_connections_env_var(self):
diff --git a/tests/providers/postgres/hooks/test_postgres.py b/tests/providers/postgres/hooks/test_postgres.py
index 57fc2d3..a0d1430 100644
--- a/tests/providers/postgres/hooks/test_postgres.py
+++ b/tests/providers/postgres/hooks/test_postgres.py
@@ -65,7 +65,7 @@ class TestPostgresHookConn(unittest.TestCase):
assert mock_connect.call_count == 1
self.assertEqual(
- self.db_hook.get_uri(), "postgres://login:password@host/schema?client_encoding=utf-8"
+ self.db_hook.get_uri(), "postgresql://login:password@host/schema?client_encoding=utf-8"
)
@mock.patch('airflow.providers.postgres.hooks.postgres.psycopg2.connect')
@@ -168,7 +168,7 @@ class TestPostgresHookConn(unittest.TestCase):
port=1,
)
)
- assert "postgres://login:password@host:1/schema" == self.db_hook.get_uri()
+ assert "postgresql://login:password@host:1/schema" == self.db_hook.get_uri()
def test_get_uri_from_connection_with_schema_override(self):
hook = PostgresHook(schema='schema-override')
@@ -182,7 +182,7 @@ class TestPostgresHookConn(unittest.TestCase):
port=1,
)
)
- assert "postgres://login:password@host:1/schema-override" == hook.get_uri()
+ assert "postgresql://login:password@host:1/schema-override" == hook.get_uri()
class TestPostgresHook(unittest.TestCase):