You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/06/03 16:12:08 UTC
[airflow] branch main updated: Migrate Alibaba example DAGs to new design #22437 (#24130)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new c887275bb8 Migrate Alibaba example DAGs to new design #22437 (#24130)
c887275bb8 is described below
commit c887275bb8ad29930f5a2cc5e5270533f92b80d1
Author: chethanuk-plutoflume <ch...@tessian.com>
AuthorDate: Fri Jun 3 17:12:01 2022 +0100
Migrate Alibaba example DAGs to new design #22437 (#24130)
* Migrate Alibaba example DAGs to new design #22437
---
.../providers/alibaba/cloud/example_dags/__init__.py | 17 -----------------
docs/apache-airflow-providers-alibaba/index.rst | 2 +-
.../apache-airflow-providers-alibaba/operators/oss.rst | 2 +-
.../system/providers/alibaba}/example_oss_bucket.py | 17 +++++++++++++++--
.../system/providers/alibaba}/example_oss_object.py | 18 ++++++++++++++++--
5 files changed, 33 insertions(+), 23 deletions(-)
diff --git a/airflow/providers/alibaba/cloud/example_dags/__init__.py b/airflow/providers/alibaba/cloud/example_dags/__init__.py
deleted file mode 100644
index 217e5db960..0000000000
--- a/airflow/providers/alibaba/cloud/example_dags/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/docs/apache-airflow-providers-alibaba/index.rst b/docs/apache-airflow-providers-alibaba/index.rst
index fca2d993f3..963cf91197 100644
--- a/docs/apache-airflow-providers-alibaba/index.rst
+++ b/docs/apache-airflow-providers-alibaba/index.rst
@@ -39,7 +39,7 @@ Content
:maxdepth: 1
:caption: Resources
- Example DAGs <https://github.com/apache/airflow/tree/main/airflow/providers/alibaba/cloud/example_dags>
+ Example DAGs <https://github.com/apache/airflow/tree/main/tests/system/providers/alibaba>
PyPI Repository <https://pypi.org/project/apache-airflow-providers-alibaba/>
Installing from sources <installing-providers-from-sources>
diff --git a/docs/apache-airflow-providers-alibaba/operators/oss.rst b/docs/apache-airflow-providers-alibaba/operators/oss.rst
index 923f0bdd19..d09949be0b 100644
--- a/docs/apache-airflow-providers-alibaba/operators/oss.rst
+++ b/docs/apache-airflow-providers-alibaba/operators/oss.rst
@@ -45,7 +45,7 @@ Defining tasks
In the following code we create a new bucket and then delete the bucket.
-.. exampleinclude:: /../../airflow/providers/alibaba/cloud/example_dags/example_oss_bucket.py
+.. exampleinclude:: /../../tests/system/providers/alibaba/example_oss_bucket.py
:language: python
:start-after: [START howto_operator_oss_bucket]
:end-before: [END howto_operator_oss_bucket]
diff --git a/airflow/providers/alibaba/cloud/example_dags/example_oss_bucket.py b/tests/system/providers/alibaba/example_oss_bucket.py
similarity index 74%
rename from airflow/providers/alibaba/cloud/example_dags/example_oss_bucket.py
rename to tests/system/providers/alibaba/example_oss_bucket.py
index 6d16ce3feb..d87af7e0a8 100644
--- a/airflow/providers/alibaba/cloud/example_dags/example_oss_bucket.py
+++ b/tests/system/providers/alibaba/example_oss_bucket.py
@@ -17,15 +17,17 @@
# Ignore missing args provided by default_args
# type: ignore[call-arg]
-
+import os
from datetime import datetime
from airflow.models.dag import DAG
from airflow.providers.alibaba.cloud.operators.oss import OSSCreateBucketOperator, OSSDeleteBucketOperator
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+DAG_ID = "oss_bucket_dag"
# [START howto_operator_oss_bucket]
with DAG(
- dag_id='oss_bucket_dag',
+ dag_id=DAG_ID,
start_date=datetime(2021, 1, 1),
default_args={'bucket_name': 'your bucket', 'region': 'your region'},
max_active_runs=1,
@@ -38,4 +40,15 @@ with DAG(
delete_bucket = OSSDeleteBucketOperator(task_id='task2')
create_bucket >> delete_bucket
+
+ from tests.system.utils.watcher import watcher
+
+ # This test needs watcher in order to properly mark success/failure
+ # when "tearDown" task with trigger rule is part of the DAG
+ list(dag.tasks) >> watcher()
# [END howto_operator_oss_bucket]
+
+from tests.system.utils import get_test_run # noqa: E402
+
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)
diff --git a/airflow/providers/alibaba/cloud/example_dags/example_oss_object.py b/tests/system/providers/alibaba/example_oss_object.py
similarity index 79%
rename from airflow/providers/alibaba/cloud/example_dags/example_oss_object.py
rename to tests/system/providers/alibaba/example_oss_object.py
index ac3a076744..d648883aa7 100644
--- a/airflow/providers/alibaba/cloud/example_dags/example_oss_object.py
+++ b/tests/system/providers/alibaba/example_oss_object.py
@@ -16,7 +16,7 @@
# under the License.
# Ignore missing args provided by default_args
# type: ignore[call-arg]
-
+import os
from datetime import datetime
from airflow.models.dag import DAG
@@ -27,8 +27,10 @@ from airflow.providers.alibaba.cloud.operators.oss import (
OSSUploadObjectOperator,
)
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+DAG_ID = "oss_object_dag"
with DAG(
- dag_id='oss_object_dag',
+ dag_id=DAG_ID,
start_date=datetime(2021, 1, 1),
default_args={'bucket_name': 'your bucket', 'region': 'your region'},
max_active_runs=1,
@@ -59,3 +61,15 @@ with DAG(
)
create_object >> download_object >> delete_object >> delete_batch_object
+
+ from tests.system.utils.watcher import watcher
+
+ # This test needs watcher in order to properly mark success/failure
+ # when "tearDown" task with trigger rule is part of the DAG
+ list(dag.tasks) >> watcher()
+
+
+from tests.system.utils import get_test_run # noqa: E402
+
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)