You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ep...@apache.org on 2023/09/04 17:37:19 UTC
[airflow] branch v2-7-test updated (7536737e23 -> a5fa287ec4)
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a change to branch v2-7-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
omit 7536737e23 Attempt to fix the flaky TestTriggererJob test (#34075)
omit 7ac6f56c89 fixup! Update RELEASE_NOTES.rst
new c25ba74c0c Attempt to fix the flaky TestTriggererJob test (#34075)
new 693b7ed329 Bring back Pydantic 1 compatibility (#34081)
new a5fa287ec4 fixup! Update RELEASE_NOTES.rst
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (7536737e23)
\
N -- N -- N refs/heads/v2-7-test (a5fa287ec4)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
RELEASE_NOTES.rst | 3 +--
airflow/configuration.py | 4 ++++
airflow/serialization/pydantic/dataset.py | 4 ++++
airflow/serialization/pydantic/job.py | 1 +
airflow/serialization/pydantic/taskinstance.py | 1 +
airflow/serialization/serde.py | 9 ++++++++-
airflow/serialization/serialized_objects.py | 5 ++++-
setup.cfg | 6 +++++-
8 files changed, 28 insertions(+), 5 deletions(-)
[airflow] 03/03: fixup! Update RELEASE_NOTES.rst
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-7-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit a5fa287ec4c26478f9ce9154b0fe7af27e47ee9c
Author: Ephraim Anierobi <sp...@gmail.com>
AuthorDate: Mon Sep 4 14:41:15 2023 +0100
fixup! Update RELEASE_NOTES.rst
---
RELEASE_NOTES.rst | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst
index bbc971b80d..706f712e88 100644
--- a/RELEASE_NOTES.rst
+++ b/RELEASE_NOTES.rst
@@ -97,8 +97,8 @@ Bug Fixes
Misc/Internal
"""""""""""""
+- Bring back Pydantic 1 compatibility (#34081, #33998)
- Use a trimmed version of README.md for PyPI (#33637)
-- Upgrade to ``Pydantic`` 2 (#33956)
- Reorganize ``devel_only`` extra in Airflow's setup.py (#33907)
- Bumping ``FAB`` to ``4.3.4`` in order to fix issues with filters (#33931)
- Add minimum requirement for ``sqlalchemy to 1.4.24`` (#33892)
[airflow] 02/03: Bring back Pydantic 1 compatibility (#34081)
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-7-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit 693b7ed329589c561c5befc7f703ab20df2a0712
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Mon Sep 4 19:24:30 2023 +0200
Bring back Pydantic 1 compatibility (#34081)
The #33956 and #33998 updated limits for Pydantic to Pydantic 2
only and removed Pydantic 1 compatibility. However it turns out
that some of important 3rd-party libraries have not yet upgraded
and it will make it impossible to install them on Airflow 2.7.1+
if we keep the limit.
For now we bring back Pydantic 1 compatibility, we remove the limit
and filter the warnings that made us remove the compatibility code.
(cherry picked from commit 30ddfc578fa01d616c93beba6523f5e026ce5c55)
---
airflow/configuration.py | 4 ++++
airflow/serialization/pydantic/dataset.py | 4 ++++
airflow/serialization/pydantic/job.py | 1 +
airflow/serialization/pydantic/taskinstance.py | 1 +
airflow/serialization/serde.py | 9 ++++++++-
airflow/serialization/serialized_objects.py | 5 ++++-
setup.cfg | 6 +++++-
7 files changed, 27 insertions(+), 3 deletions(-)
diff --git a/airflow/configuration.py b/airflow/configuration.py
index 41dc8d718e..92b9bd0f64 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -58,6 +58,10 @@ if not sys.warnoptions:
warnings.filterwarnings(action="default", category=DeprecationWarning, module="airflow")
warnings.filterwarnings(action="default", category=PendingDeprecationWarning, module="airflow")
+ # Temporarily suppress warnings from pydantic until we upgrade minimum version of pydantic to v2
+ # Which should happen in Airflow 2.8.0
+ warnings.filterwarnings(action="ignore", category=UserWarning, module=r"pydantic._internal._config")
+
_SQLITE3_VERSION_PATTERN = re2.compile(r"(?P<version>^\d+(?:\.\d+)*)\D?.*$")
ConfigType = Union[str, int, float, bool]
diff --git a/airflow/serialization/pydantic/dataset.py b/airflow/serialization/pydantic/dataset.py
index 07d6b3bf88..096bda6ddd 100644
--- a/airflow/serialization/pydantic/dataset.py
+++ b/airflow/serialization/pydantic/dataset.py
@@ -32,6 +32,7 @@ class DagScheduleDatasetReferencePydantic(BaseModelPydantic):
"""Make sure it deals automatically with SQLAlchemy ORM classes."""
from_attributes = True
+ orm_mode = True # Pydantic 1.x compatibility.
class TaskOutletDatasetReferencePydantic(BaseModelPydantic):
@@ -47,6 +48,7 @@ class TaskOutletDatasetReferencePydantic(BaseModelPydantic):
"""Make sure it deals automatically with SQLAlchemy ORM classes."""
from_attributes = True
+ orm_mode = True # Pydantic 1.x compatibility.
class DatasetPydantic(BaseModelPydantic):
@@ -66,6 +68,7 @@ class DatasetPydantic(BaseModelPydantic):
"""Make sure it deals automatically with SQLAlchemy ORM classes."""
from_attributes = True
+ orm_mode = True # Pydantic 1.x compatibility.
class DatasetEventPydantic(BaseModelPydantic):
@@ -84,3 +87,4 @@ class DatasetEventPydantic(BaseModelPydantic):
"""Make sure it deals automatically with SQLAlchemy ORM classes."""
from_attributes = True
+ orm_mode = True # Pydantic 1.x compatibility.
diff --git a/airflow/serialization/pydantic/job.py b/airflow/serialization/pydantic/job.py
index 5b0e961e04..27c8ad8ca7 100644
--- a/airflow/serialization/pydantic/job.py
+++ b/airflow/serialization/pydantic/job.py
@@ -50,3 +50,4 @@ class JobPydantic(BaseModelPydantic):
"""Make sure it deals automatically with SQLAlchemy ORM classes."""
from_attributes = True
+ orm_mode = True # Pydantic 1.x compatibility.
diff --git a/airflow/serialization/pydantic/taskinstance.py b/airflow/serialization/pydantic/taskinstance.py
index 0bca3505cd..71d8ba576f 100644
--- a/airflow/serialization/pydantic/taskinstance.py
+++ b/airflow/serialization/pydantic/taskinstance.py
@@ -61,6 +61,7 @@ class TaskInstancePydantic(BaseModelPydantic):
"""Make sure it deals automatically with SQLAlchemy ORM classes."""
from_attributes = True
+ orm_mode = True # Pydantic 1.x compatibility.
def xcom_pull(
self,
diff --git a/airflow/serialization/serde.py b/airflow/serialization/serde.py
index ba60af6010..794ef4941f 100644
--- a/airflow/serialization/serde.py
+++ b/airflow/serialization/serde.py
@@ -317,7 +317,14 @@ def _is_pydantic(cls: Any) -> bool:
Checking is done by attributes as it is significantly faster than
using isinstance.
"""
- return hasattr(cls, "model_config") and hasattr(cls, "model_fields") and hasattr(cls, "model_fields_set")
+ return (
+ hasattr(cls, "__validators__")
+ and hasattr(cls, "__fields__")
+ and hasattr(cls, "dict") # Pydantic v1
+ or hasattr(cls, "model_config")
+ and hasattr(cls, "model_fields")
+ and hasattr(cls, "model_fields_set") # Pydantic v2
+ )
def _register():
diff --git a/airflow/serialization/serialized_objects.py b/airflow/serialization/serialized_objects.py
index 8e74788a7a..26dd817862 100644
--- a/airflow/serialization/serialized_objects.py
+++ b/airflow/serialization/serialized_objects.py
@@ -485,7 +485,10 @@ class BaseSerialization:
elif use_pydantic_models and _ENABLE_AIP_44:
def _pydantic_model_dump(model_cls: type[BaseModel], var: Any) -> dict[str, Any]:
- return model_cls.model_validate(var).model_dump(mode="json")
+ try:
+ return model_cls.model_validate(var).model_dump(mode="json") # type: ignore[attr-defined]
+ except AttributeError: # Pydantic 1.x compatibility.
+ return model_cls.from_orm(var).dict() # type: ignore[attr-defined]
if isinstance(var, Job):
return cls._encode(_pydantic_model_dump(JobPydantic, var), type_=DAT.BASE_JOB)
diff --git a/setup.cfg b/setup.cfg
index fa963c2975..367924d82d 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -123,7 +123,11 @@ install_requires =
pendulum>=2.0
pluggy>=1.0
psutil>=4.2.0
- pydantic>=2.3.0
+ # We should bump it to at least pydantic>=2.3.0 when we prepare Airflow 2.8.0 release
+ # we keep Pydantic < 1 for compatibility with packages that depend on Pydantic 1
+ # We should also remove then `filterwarning` for pydantic from airflow/configuration.py
+ # and # Pydantic v1 check in airflow/serialization/serde.py
+ pydantic>=1.10.0
pygments>=2.0.1
pyjwt>=2.0.0
python-daemon>=3.0.0
[airflow] 01/03: Attempt to fix the flaky TestTriggererJob test (#34075)
Posted by ep...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
ephraimanierobi pushed a commit to branch v2-7-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
commit c25ba74c0c2e7fc8bbe523461c1599965bdb8652
Author: Jarek Potiuk <ja...@potiuk.com>
AuthorDate: Mon Sep 4 16:40:24 2023 +0200
Attempt to fix the flaky TestTriggererJob test (#34075)
As documented in #33323, we have frequent failures of the flaky
triggerer job tests.
The flaky failures are about some errors when we close all the
sessions in teardown of the test. It turns out that the tests
had side-effect - they have not waited for the TriggererJob thread
to complete, they merely marked them to be stopped, but they have
not waited for those to complete - this is quite plausible
explanation of the flaky test failures - since those threads have
1 second sleep, it's more than likely that the session has been
created and used by the thread while the teardown has been attempting
to close all the sessions.
This side effect could also have an effect for other tests that
were run after - because in a busy test run machine, the side
effects could propagate further than just to the teardown, so it
could also explain why sometimes (very rarely) other job tests
failed with similar errors.
The fix is to join the runner after marking it to be stopped.
Fixes: #33323 (Hopefully)
(cherry picked from commit 47f79b9198f3350951dc21808c36f889bee0cd06)
---
tests/jobs/test_triggerer_job.py | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/tests/jobs/test_triggerer_job.py b/tests/jobs/test_triggerer_job.py
index 35ebe99b31..e79202d6f6 100644
--- a/tests/jobs/test_triggerer_job.py
+++ b/tests/jobs/test_triggerer_job.py
@@ -143,6 +143,8 @@ def test_trigger_logging_sensitive_info(session, capsys):
finally:
# We always have to stop the runner
triggerer_job_runner.trigger_runner.stop = True
+ triggerer_job_runner.trigger_runner.join(30)
+
# Since we have now an in-memory process of forwarding the logs to stdout,
# give it more time for the trigger event to write the log.
time.sleep(0.5)
@@ -257,6 +259,7 @@ def test_trigger_lifecycle(session):
finally:
# We always have to stop the runner
job_runner.trigger_runner.stop = True
+ job_runner.trigger_runner.join(30)
class TestTriggerRunner:
@@ -408,7 +411,7 @@ def test_trigger_create_race_condition_18392(session, tmp_path):
pytest.fail("did not observe 2 loops in the runner thread")
finally:
job_runner.trigger_runner.stop = True
- job_runner.trigger_runner.join()
+ job_runner.trigger_runner.join(30)
thread.join()
instances = path.read_text().splitlines()
assert len(instances) == 1
@@ -500,7 +503,7 @@ def test_trigger_runner_exception_stops_triggerer(session):
finally:
job_runner.trigger_runner.stop = True
# with suppress(MockTriggerException):
- job_runner.trigger_runner.join()
+ job_runner.trigger_runner.join(30)
thread.join()
@@ -531,6 +534,7 @@ def test_trigger_firing(session):
finally:
# We always have to stop the runner
job_runner.trigger_runner.stop = True
+ job_runner.trigger_runner.join(30)
def test_trigger_failing(session):
@@ -564,6 +568,7 @@ def test_trigger_failing(session):
finally:
# We always have to stop the runner
job_runner.trigger_runner.stop = True
+ job_runner.trigger_runner.join(30)
def test_trigger_cleanup(session):