[airflow] branch main updated: AIP-47 - Migrate livy DAGs to new design #22439 (#24208)

[po...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/main by this push:
     new 5503603cde AIP-47 - Migrate livy DAGs to new design #22439 (#24208)
5503603cde is described below

commit 5503603cded8613721ea56599667b5a309190b23
Author: chethanuk-plutoflume <ch...@tessian.com>
AuthorDate: Sun Jun 5 11:01:18 2022 +0100

    AIP-47 - Migrate livy DAGs to new design #22439 (#24208)
---
 airflow/providers/apache/livy/example_dags/__init__.py | 16 ----------------
 docs/apache-airflow-providers-apache-livy/index.rst    |  2 +-
 .../apache-airflow-providers-apache-livy/operators.rst |  2 +-
 .../system/providers/apache/livy}/example_livy.py      | 18 +++++++++++++++++-
 4 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/airflow/providers/apache/livy/example_dags/__init__.py b/airflow/providers/apache/livy/example_dags/__init__.py
deleted file mode 100644
index 13a83393a9..0000000000
--- a/airflow/providers/apache/livy/example_dags/__init__.py
+++ /dev/null
@@ -1,16 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/docs/apache-airflow-providers-apache-livy/index.rst b/docs/apache-airflow-providers-apache-livy/index.rst
index cd2f5058c6..8ac4ff3dac 100644
--- a/docs/apache-airflow-providers-apache-livy/index.rst
+++ b/docs/apache-airflow-providers-apache-livy/index.rst
@@ -37,7 +37,7 @@ Content
     :maxdepth: 1
     :caption: Resources
 
-    Example DAGs <https://github.com/apache/airflow/tree/main/airflow/providers/apache/livy/example_dags>
+    Example DAGs <https://github.com/apache/airflow/tree/main/tests/system/providers/apache/livy>
     PyPI Repository <https://pypi.org/project/apache-airflow-providers-apache-livy/>
     Installing from sources <installing-providers-from-sources>
 
diff --git a/docs/apache-airflow-providers-apache-livy/operators.rst b/docs/apache-airflow-providers-apache-livy/operators.rst
index 6d3aba0d67..09d97bbbdb 100644
--- a/docs/apache-airflow-providers-apache-livy/operators.rst
+++ b/docs/apache-airflow-providers-apache-livy/operators.rst
@@ -29,7 +29,7 @@ LivyOperator
 
 This operator wraps the Apache Livy batch REST API, allowing to submit a Spark application to the underlying cluster.
 
-.. exampleinclude:: /../../airflow/providers/apache/livy/example_dags/example_livy.py
+.. exampleinclude:: /../../tests/system/providers/apache/livy/example_livy.py
     :language: python
     :start-after: [START create_livy]
     :end-before: [END create_livy]
diff --git a/airflow/providers/apache/livy/example_dags/example_livy.py b/tests/system/providers/apache/livy/example_livy.py
similarity index 76%
rename from airflow/providers/apache/livy/example_dags/example_livy.py
rename to tests/system/providers/apache/livy/example_livy.py
index cf7bbbfb1b..8420e756ec 100644
--- a/airflow/providers/apache/livy/example_dags/example_livy.py
+++ b/tests/system/providers/apache/livy/example_livy.py
@@ -20,13 +20,18 @@ This is an example DAG which uses the LivyOperator.
 The tasks below trigger the computation of pi on the Spark instance
 using the Java and Python executables provided in the example library.
 """
+
+import os
 from datetime import datetime
 
 from airflow import DAG
 from airflow.providers.apache.livy.operators.livy import LivyOperator
 
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+DAG_ID = "example_livy_operator"
+
 with DAG(
-    dag_id='example_livy_operator',
+    dag_id=DAG_ID,
     default_args={'args': [10]},
     schedule_interval='@daily',
     start_date=datetime(2021, 1, 1),
@@ -48,3 +53,14 @@ with DAG(
 
     livy_java_task >> livy_python_task
     # [END create_livy]
+
+    from tests.system.utils.watcher import watcher
+
+    # This test needs watcher in order to properly mark success/failure
+    # when "tearDown" task with trigger rule is part of the DAG
+    list(dag.tasks) >> watcher()
+
+from tests.system.utils import get_test_run  # noqa: E402
+
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)