You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by po...@apache.org on 2022/06/05 09:32:16 UTC
[airflow] branch main updated: AIP-47 - Migrate apache pig DAGs to new design #22439 (#24212)
This is an automated email from the ASF dual-hosted git repository.
potiuk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/main by this push:
new 0046f125ff AIP-47 - Migrate apache pig DAGs to new design #22439 (#24212)
0046f125ff is described below
commit 0046f125ffc9d127b37624bc381faaa5669c6d3a
Author: chethanuk-plutoflume <ch...@tessian.com>
AuthorDate: Sun Jun 5 10:32:10 2022 +0100
AIP-47 - Migrate apache pig DAGs to new design #22439 (#24212)
---
.../providers/apache/pig/example_dags/__init__.py | 17 ------------
docs/apache-airflow-providers-apache-pig/index.rst | 2 +-
.../operators.rst | 2 +-
.../system/providers/apache/pig}/example_pig.py | 32 ++++++++++++++--------
4 files changed, 23 insertions(+), 30 deletions(-)
diff --git a/airflow/providers/apache/pig/example_dags/__init__.py b/airflow/providers/apache/pig/example_dags/__init__.py
deleted file mode 100644
index 217e5db960..0000000000
--- a/airflow/providers/apache/pig/example_dags/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
diff --git a/docs/apache-airflow-providers-apache-pig/index.rst b/docs/apache-airflow-providers-apache-pig/index.rst
index 4fd1ffee28..e0caa0cd6f 100644
--- a/docs/apache-airflow-providers-apache-pig/index.rst
+++ b/docs/apache-airflow-providers-apache-pig/index.rst
@@ -37,7 +37,7 @@ Content
:maxdepth: 1
:caption: Resources
- Example DAGs <https://github.com/apache/airflow/tree/main/airflow/providers/apache/pig/example_dags>
+ Example DAGs <https://github.com/apache/airflow/tree/main/tests/system/providers/apache/pig>
PyPI Repository <https://pypi.org/project/apache-airflow-providers-apache-pig/>
Installing from sources <installing-providers-from-sources>
diff --git a/docs/apache-airflow-providers-apache-pig/operators.rst b/docs/apache-airflow-providers-apache-pig/operators.rst
index 04e29e175b..70065b0773 100644
--- a/docs/apache-airflow-providers-apache-pig/operators.rst
+++ b/docs/apache-airflow-providers-apache-pig/operators.rst
@@ -26,7 +26,7 @@ Pig programs are amenable to substantial parallelization, which in turns enables
use the PigOperator to execute a pig script
-.. exampleinclude:: /../../airflow/providers/apache/pig/example_dags/example_pig.py
+.. exampleinclude:: /../../tests/system/providers/apache/pig/example_pig.py
:language: python
:start-after: [START create_pig]
:end-before: [END create_pig]
diff --git a/airflow/providers/apache/pig/example_dags/example_pig.py b/tests/system/providers/apache/pig/example_pig.py
similarity index 70%
rename from airflow/providers/apache/pig/example_dags/example_pig.py
rename to tests/system/providers/apache/pig/example_pig.py
index ed1b34ab0c..325f05b8eb 100644
--- a/airflow/providers/apache/pig/example_dags/example_pig.py
+++ b/tests/system/providers/apache/pig/example_pig.py
@@ -17,24 +17,34 @@
# under the License.
"""Example DAG demonstrating the usage of the PigOperator."""
+
+import os
from datetime import datetime
from airflow import DAG
from airflow.providers.apache.pig.operators.pig import PigOperator
-dag = DAG(
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+DAG_ID = "example_adf_run_pipeline"
+
+with DAG(
dag_id='example_pig_operator',
schedule_interval=None,
start_date=datetime(2021, 1, 1),
catchup=False,
tags=['example'],
-)
-
-# [START create_pig]
-run_this = PigOperator(
- task_id="run_example_pig_script",
- pig="ls /;",
- pig_opts="-x local",
- dag=dag,
-)
-# [END create_pig]
+) as dag:
+
+ # [START create_pig]
+ run_this = PigOperator(
+ task_id="run_example_pig_script",
+ pig="ls /;",
+ pig_opts="-x local",
+ )
+ # [END create_pig]
+
+
+from tests.system.utils import get_test_run # noqa: E402
+
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+test_run = get_test_run(dag)