You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/08/04 06:38:38 UTC
[airflow] branch main updated: AIP-47 - Migrate Airbyte DAGs to new design (#25135)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 6861bcc03f AIP-47 - Migrate Airbyte DAGs to new design (#25135)
6861bcc03f is described below
commit 6861bcc03fa2177ad96cf493dcc19826849f92e3
Author: chethanuk-plutoflume <ch...@tessian.com>
AuthorDate: Thu Aug 4 07:38:26 2022 +0100
AIP-47 - Migrate Airbyte DAGs to new design (#25135)
---
docs/apache-airflow-providers-airbyte/index.rst | 8 +++++++-
.../operators/airbyte.rst | 4 ++--
.../system/providers/airbyte}/__init__.py | 0
.../providers/airbyte}/example_airbyte_trigger_job.py | 16 +++++++++++++---
.../system/providers/google/firebase}/__init__.py | 0
.../system/providers/google/suite}/__init__.py | 0
6 files changed, 22 insertions(+), 6 deletions(-)
diff --git a/docs/apache-airflow-providers-airbyte/index.rst b/docs/apache-airflow-providers-airbyte/index.rst
index a807de80b1..3caf529d45 100644
--- a/docs/apache-airflow-providers-airbyte/index.rst
+++ b/docs/apache-airflow-providers-airbyte/index.rst
@@ -34,11 +34,17 @@ Content
Python API <_api/airflow/providers/airbyte/index>
+.. toctree::
+ :hidden:
+ :caption: System tests
+
+ System Tests <_api/tests/system/providers/airbyte/index>
+
.. toctree::
:maxdepth: 1
:caption: Resources
- Example DAGs <https://github.com/apache/airflow/tree/providers-airbyte/3.0.0/airflow/providers/airbyte/example_dags>
+ Example DAGs <https://github.com/apache/airflow/tree/providers-airbyte/3.0.0/tests/system/providers/airbyte>
PyPI Repository <https://pypi.org/project/apache-airflow-providers-airbyte/>
Installing from sources <installing-providers-from-sources>
diff --git a/docs/apache-airflow-providers-airbyte/operators/airbyte.rst b/docs/apache-airflow-providers-airbyte/operators/airbyte.rst
index b67462734e..68fd8c44cb 100644
--- a/docs/apache-airflow-providers-airbyte/operators/airbyte.rst
+++ b/docs/apache-airflow-providers-airbyte/operators/airbyte.rst
@@ -45,14 +45,14 @@ return the ``job_id`` that should be pass to the AirbyteSensor.
An example using the synchronous way:
-.. exampleinclude:: /../../airflow/providers/airbyte/example_dags/example_airbyte_trigger_job.py
+.. exampleinclude:: /../../tests/system/providers/airbyte/example_airbyte_trigger_job.py
:language: python
:start-after: [START howto_operator_airbyte_synchronous]
:end-before: [END howto_operator_airbyte_synchronous]
An example using the async way:
-.. exampleinclude:: /../../airflow/providers/airbyte/example_dags/example_airbyte_trigger_job.py
+.. exampleinclude:: /../../tests/system/providers/airbyte/example_airbyte_trigger_job.py
:language: python
:start-after: [START howto_operator_airbyte_asynchronous]
:end-before: [END howto_operator_airbyte_asynchronous]
diff --git a/airflow/providers/airbyte/example_dags/__init__.py b/tests/system/providers/airbyte/__init__.py
similarity index 100%
copy from airflow/providers/airbyte/example_dags/__init__.py
copy to tests/system/providers/airbyte/__init__.py
diff --git a/airflow/providers/airbyte/example_dags/example_airbyte_trigger_job.py b/tests/system/providers/airbyte/example_airbyte_trigger_job.py
similarity index 83%
rename from airflow/providers/airbyte/example_dags/example_airbyte_trigger_job.py
rename to tests/system/providers/airbyte/example_airbyte_trigger_job.py
index 55563ff5e0..f7b9bdf700 100644
--- a/airflow/providers/airbyte/example_dags/example_airbyte_trigger_job.py
+++ b/tests/system/providers/airbyte/example_airbyte_trigger_job.py
@@ -18,14 +18,19 @@
"""Example DAG demonstrating the usage of the AirbyteTriggerSyncOperator."""
+import os
from datetime import datetime, timedelta
from airflow import DAG
from airflow.providers.airbyte.operators.airbyte import AirbyteTriggerSyncOperator
from airflow.providers.airbyte.sensors.airbyte import AirbyteJobSensor
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+DAG_ID = "example_airbyte_operator"
+CONN_ID = '15bc3800-82e4-48c3-a32d-620661273f28'
+
with DAG(
- dag_id='example_airbyte_operator',
+ dag_id=DAG_ID,
schedule_interval=None,
start_date=datetime(2021, 1, 1),
dagrun_timeout=timedelta(minutes=60),
@@ -36,14 +41,14 @@ with DAG(
# [START howto_operator_airbyte_synchronous]
sync_source_destination = AirbyteTriggerSyncOperator(
task_id='airbyte_sync_source_dest_example',
- connection_id='15bc3800-82e4-48c3-a32d-620661273f28',
+ connection_id=CONN_ID,
)
# [END howto_operator_airbyte_synchronous]
# [START howto_operator_airbyte_asynchronous]
async_source_destination = AirbyteTriggerSyncOperator(
task_id='airbyte_async_source_dest_example',
- connection_id='15bc3800-82e4-48c3-a32d-620661273f28',
+ connection_id=CONN_ID,
asynchronous=True,
)
@@ -55,3 +60,8 @@ with DAG(
# Task dependency created via `XComArgs`:
# async_source_destination >> airbyte_sensor
+
+from tests.system.utils import get_test_run # noqa: E402
+
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)
diff --git a/airflow/providers/airbyte/example_dags/__init__.py b/tests/system/providers/google/firebase/__init__.py
similarity index 100%
copy from airflow/providers/airbyte/example_dags/__init__.py
copy to tests/system/providers/google/firebase/__init__.py
diff --git a/airflow/providers/airbyte/example_dags/__init__.py b/tests/system/providers/google/suite/__init__.py
similarity index 100%
rename from airflow/providers/airbyte/example_dags/__init__.py
rename to tests/system/providers/google/suite/__init__.py