You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/06/01 20:13:03 UTC
[airflow] branch main updated: Migrate HTTP example DAGs to new design AIP-47 (#23991)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 9398586a7c Migrate HTTP example DAGs to new design AIP-47 (#23991)
9398586a7c is described below
commit 9398586a7cf66d9cf078c40ab0d939b3fcc58c2d
Author: Chenglong Yan <al...@gmail.com>
AuthorDate: Thu Jun 2 04:12:55 2022 +0800
Migrate HTTP example DAGs to new design AIP-47 (#23991)
closes: #22448 , #22431
---
airflow/providers/http/example_dags/__init__.py | 17 ------------
docs/apache-airflow-providers-http/operators.rst | 14 +++++-----
tests/providers/http/operators/test_http_system.py | 32 ----------------------
.../system/providers/http}/example_http.py | 12 +++++++-
4 files changed, 18 insertions(+), 57 deletions(-)
diff --git a/airflow/providers/http/example_dags/__init__.py b/airflow/providers/http/example_dags/__init__.py
deleted file mode 100644
index 217e5db960..0000000000
--- a/airflow/providers/http/example_dags/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/docs/apache-airflow-providers-http/operators.rst b/docs/apache-airflow-providers-http/operators.rst
index 0deb4a021c..8c67a963ca 100644
--- a/docs/apache-airflow-providers-http/operators.rst
+++ b/docs/apache-airflow-providers-http/operators.rst
@@ -32,7 +32,7 @@ to ``true``.
Here we are poking until httpbin gives us a response text containing ``httpbin``.
-.. exampleinclude:: /../../airflow/providers/http/example_dags/example_http.py
+.. exampleinclude:: /../../tests/system/providers/http/example_http.py
:language: python
:start-after: [START howto_operator_http_http_sensor_check]
:end-before: [END howto_operator_http_http_sensor_check]
@@ -69,14 +69,14 @@ the response text back.
In the first example we are calling a ``POST`` with json data and succeed when we get the same json data back
otherwise the task will fail.
-.. exampleinclude:: /../../airflow/providers/http/example_dags/example_http.py
+.. exampleinclude:: /../../tests/system/providers/http/example_http.py
:language: python
:start-after: [START howto_operator_http_task_post_op]
:end-before: [END howto_operator_http_task_post_op]
Here we are calling a ``GET`` request and pass params to it. The task will succeed regardless of the response text.
-.. exampleinclude:: /../../airflow/providers/http/example_dags/example_http.py
+.. exampleinclude:: /../../tests/system/providers/http/example_http.py
:language: python
:start-after: [START howto_operator_http_task_get_op]
:end-before: [END howto_operator_http_task_get_op]
@@ -91,7 +91,7 @@ it on the next task downstream use ``response_filter``. This is useful if:
Below is an example of retrieving data from a REST API and only returning a nested property instead of the full
response body.
-.. exampleinclude:: /../../airflow/providers/http/example_dags/example_http.py
+.. exampleinclude:: /../../tests/system/providers/http/example_http.py
:language: python
:start-after: [START howto_operator_http_task_get_op_response_filter]
:end-before: [END howto_operator_http_task_get_op_response_filter]
@@ -99,7 +99,7 @@ response body.
In the third example we are performing a ``PUT`` operation to put / set data according to the data that is being
provided to the request.
-.. exampleinclude:: /../../airflow/providers/http/example_dags/example_http.py
+.. exampleinclude:: /../../tests/system/providers/http/example_http.py
:language: python
:start-after: [START howto_operator_http_task_put_op]
:end-before: [END howto_operator_http_task_put_op]
@@ -107,14 +107,14 @@ provided to the request.
In this example we call a ``DELETE`` operation to the ``delete`` endpoint. This time we are passing form data to the
request.
-.. exampleinclude:: /../../airflow/providers/http/example_dags/example_http.py
+.. exampleinclude:: /../../tests/system/providers/http/example_http.py
:language: python
:start-after: [START howto_operator_http_task_del_op]
:end-before: [END howto_operator_http_task_del_op]
Here we pass form data to a ``POST`` operation which is equal to a usual form submit.
-.. exampleinclude:: /../../airflow/providers/http/example_dags/example_http.py
+.. exampleinclude:: /../../tests/system/providers/http/example_http.py
:language: python
:start-after: [START howto_operator_http_task_post_op_formenc]
:end-before: [END howto_operator_http_task_post_op_formenc]
diff --git a/tests/providers/http/operators/test_http_system.py b/tests/providers/http/operators/test_http_system.py
deleted file mode 100644
index f86a898acf..0000000000
--- a/tests/providers/http/operators/test_http_system.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import os
-
-import pytest
-
-from tests.test_utils import AIRFLOW_MAIN_FOLDER
-from tests.test_utils.system_tests_class import SystemTest
-
-HTTP_DAG_FOLDER = os.path.join(AIRFLOW_MAIN_FOLDER, "airflow", "providers", "http", "example_dags")
-
-
-@pytest.mark.backend("mysql", "postgres")
-@pytest.mark.system("http")
-class HttpExampleDagsSystemTest(SystemTest):
- def test_run_example_dag_http(self):
- self.run_dag('example_http_operator', HTTP_DAG_FOLDER)
diff --git a/airflow/providers/http/example_dags/example_http.py b/tests/system/providers/http/example_http.py
similarity index 92%
rename from airflow/providers/http/example_dags/example_http.py
rename to tests/system/providers/http/example_http.py
index 59589b84d9..876c09eb40 100644
--- a/airflow/providers/http/example_dags/example_http.py
+++ b/tests/system/providers/http/example_http.py
@@ -19,14 +19,19 @@
"""Example HTTP operator and sensor"""
import json
+import os
from datetime import datetime
from airflow import DAG
from airflow.providers.http.operators.http import SimpleHttpOperator
from airflow.providers.http.sensors.http import HttpSensor
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+DAG_ID = 'example_http_operator'
+
+
dag = DAG(
- 'example_http_operator',
+ DAG_ID,
default_args={'retries': 1},
tags=['example'],
start_date=datetime(2021, 1, 1),
@@ -107,3 +112,8 @@ task_http_sensor_check = HttpSensor(
# [END howto_operator_http_http_sensor_check]
task_http_sensor_check >> task_post_op >> task_get_op >> task_get_op_response_filter
task_get_op_response_filter >> task_put_op >> task_del_op >> task_post_op_formenc
+
+from tests.system.utils import get_test_run # noqa: E402
+
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)