You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2020/06/29 06:09:23 UTC
[airflow] branch master updated: Remove almost all references to
airflow.contrib (#9559)
This is an automated email from the ASF dual-hosted git repository.
kamilbregula pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 40add26 Remove almost all references to airflow.contrib (#9559)
40add26 is described below
commit 40add26d459c2511a6d9d305ae7300f0d6104211
Author: Kamil BreguĊa <mi...@users.noreply.github.com>
AuthorDate: Mon Jun 29 08:08:58 2020 +0200
Remove almost all references to airflow.contrib (#9559)
---
.pre-commit-config.yaml | 7 ++-----
airflow/contrib/operators/gcp_dlp_operator.py | 16 ++++++++--------
airflow/providers/amazon/aws/hooks/sagemaker.py | 6 ++++--
airflow/providers/google/cloud/hooks/vision.py | 2 +-
airflow/providers/google/cloud/transfers/s3_to_gcs.py | 2 +-
.../google/cloud/utils/mlengine_prediction_summary.py | 3 ++-
docs/conf.py | 3 ---
docs/exts/docroles.py | 3 +--
docs/operators-and-hooks-ref.rst | 2 +-
tests/providers/ftp/hooks/test_ftp.py | 2 +-
tests/task/task_runner/test_cgroup_task_runner.py | 2 +-
11 files changed, 22 insertions(+), 26 deletions(-)
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 1b4ce49..bd5ce69 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -69,8 +69,7 @@ repos:
- --fuzzy-match-generates-todo
- id: insert-license
name: Add license for all JINJA template files
- files: "^airflow/www/templates/.*\\.html$|^docs/templates/.*\\.html$|^airflow/contrib/plugins/\
-metastore_browser/templates/.*\\.html$|.*\\.jinja2"
+ files: "^airflow/www/templates/.*\\.html$|^docs/templates/.*\\.html$.*\\.jinja2"
exclude: ^\.github/.*$
args:
- --comment-style
@@ -296,7 +295,6 @@ metastore_browser/templates/.*\\.html$|.*\\.jinja2"
^airflow/operators/.*$|
^airflow/sensors/.*$|
^airflow/providers/.*$|
- ^airflow/contrib/.*$|
^backport_packages/.*$
- id: base-operator
language: pygrep
@@ -309,8 +307,7 @@ metastore_browser/templates/.*\\.html$|.*\\.jinja2"
^airflow/hooks/.*$|
^airflow/operators/.*$|
^airflow/sensors/.*$|
- ^airflow/providers/.*\.py$|
- ^airflow/contrib/.*\.py$
+ ^airflow/providers/.*\.py$
- id: provide-create-sessions
language: pygrep
name: To avoid import cycles make sure provide_session and create_session are imported from
diff --git a/airflow/contrib/operators/gcp_dlp_operator.py b/airflow/contrib/operators/gcp_dlp_operator.py
index cc718c7..cbf5194 100644
--- a/airflow/contrib/operators/gcp_dlp_operator.py
+++ b/airflow/contrib/operators/gcp_dlp_operator.py
@@ -44,13 +44,13 @@ warnings.warn(
class CloudDLPDeleteDlpJobOperator(CloudDLPDeleteDLPJobOperator):
"""
This class is deprecated.
- Please use `airflow.contrib.operators.gcp_dlp_operator.CloudDLPDeleteDLPJobOperator`.
+ Please use `airflow.providers.google.cloud.operators.dlp.CloudDLPDeleteDLPJobOperator`.
"""
def __init__(self, *args, **kwargs):
warnings.warn(
"""This class is deprecated.
- Please use `airflow.contrib.operators.gcp_dlp_operator.CloudDLPDeleteDLPJobOperator`.""",
+ Please use `airflow.providers.google.cloud.operators.dlp.CloudDLPDeleteDLPJobOperator`.""",
DeprecationWarning, stacklevel=3
)
@@ -60,13 +60,13 @@ class CloudDLPDeleteDlpJobOperator(CloudDLPDeleteDLPJobOperator):
class CloudDLPGetDlpJobOperator(CloudDLPGetDLPJobOperator):
"""
This class is deprecated.
- Please use `airflow.contrib.operators.gcp_dlp_operator.CloudDLPGetDLPJobOperator`.
+ Please use `airflow.providers.google.cloud.operators.dlp.CloudDLPGetDLPJobOperator`.
"""
def __init__(self, *args, **kwargs):
warnings.warn(
"""This class is deprecated.
- Please use `airflow.contrib.operators.gcp_dlp_operator.CloudDLPGetDLPJobOperator`.""",
+ Please use `airflow.providers.google.cloud.operators.dlp.CloudDLPGetDLPJobOperator`.""",
DeprecationWarning, stacklevel=3
)
super().__init__(*args, **kwargs)
@@ -75,13 +75,13 @@ class CloudDLPGetDlpJobOperator(CloudDLPGetDLPJobOperator):
class CloudDLPGetJobTripperOperator(CloudDLPGetDLPJobTriggerOperator):
"""
This class is deprecated.
- Please use `airflow.contrib.operators.gcp_dlp_operator.CloudDLPGetDLPJobTriggerOperator`.
+ Please use `airflow.providers.google.cloud.operators.dlp.CloudDLPGetDLPJobTriggerOperator`.
"""
def __init__(self, *args, **kwargs):
warnings.warn(
"""This class is deprecated.
- Please use `airflow.contrib.operators.gcp_dlp_operator.CloudDLPGetDLPJobTriggerOperator`.""",
+ Please use `airflow.providers.google.cloud.operators.dlp.CloudDLPGetDLPJobTriggerOperator`.""",
DeprecationWarning, stacklevel=3
)
super().__init__(*args, **kwargs)
@@ -90,13 +90,13 @@ class CloudDLPGetJobTripperOperator(CloudDLPGetDLPJobTriggerOperator):
class CloudDLPListDlpJobsOperator(CloudDLPListDLPJobsOperator):
"""
This class is deprecated.
- Please use `airflow.contrib.operators.gcp_dlp_operator.CloudDLPListDLPJobsOperator`.
+ Please use `airflow.providers.google.cloud.operators.dlp.CloudDLPListDLPJobsOperator`.
"""
def __init__(self, *args, **kwargs):
warnings.warn(
"""This class is deprecated.
- Please use `airflow.contrib.operators.gcp_dlp_operator.CloudDLPListDLPJobsOperator`.""",
+ Please use `airflow.providers.google.cloud.operators.dlp.CloudDLPListDLPJobsOperator`.""",
DeprecationWarning, stacklevel=3
)
super().__init__(*args, **kwargs)
diff --git a/airflow/providers/amazon/aws/hooks/sagemaker.py b/airflow/providers/amazon/aws/hooks/sagemaker.py
index c7692f9..d46d9d3 100644
--- a/airflow/providers/amazon/aws/hooks/sagemaker.py
+++ b/airflow/providers/amazon/aws/hooks/sagemaker.py
@@ -250,10 +250,12 @@ class SageMakerHook(AwsBaseHook):
def log_stream(self, log_group, stream_name, start_time=0, skip=0):
"""
This method is deprecated.
- Please use :py:meth:`airflow.contrib.hooks.AwsLogsHook.get_log_events` instead.
+ Please use
+ :py:meth:`airflow.providers.amazon.aws.hooks.logs.AwsLogsHook.get_log_events` instead.
"""
warnings.warn("Method `log_stream` has been deprecated. "
- "Please use `airflow.contrib.hooks.AwsLogsHook.get_log_events` instead.",
+ "Please use "
+ "`airflow.providers.amazon.aws.hooks.logs.AwsLogsHook.get_log_events` instead.",
category=DeprecationWarning,
stacklevel=2)
diff --git a/airflow/providers/google/cloud/hooks/vision.py b/airflow/providers/google/cloud/hooks/vision.py
index acb9f7a..9a5f659 100644
--- a/airflow/providers/google/cloud/hooks/vision.py
+++ b/airflow/providers/google/cloud/hooks/vision.py
@@ -385,7 +385,7 @@ class CloudVisionHook(GoogleBaseHook):
) -> str:
"""
For the documentation see:
- :py:class:`~airflow.contrib.operators.gcp_vision_operator.CloudVisionReferenceImageCreateOperator`
+ :py:class:`~airflow.providers.google.cloud.operators.vision.CloudVisionCreateReferenceImageOperator`
"""
client = self.get_conn()
self.log.info('Creating ReferenceImage')
diff --git a/airflow/providers/google/cloud/transfers/s3_to_gcs.py b/airflow/providers/google/cloud/transfers/s3_to_gcs.py
index 947f11c..39bc6b0 100644
--- a/airflow/providers/google/cloud/transfers/s3_to_gcs.py
+++ b/airflow/providers/google/cloud/transfers/s3_to_gcs.py
@@ -214,7 +214,7 @@ class S3ToGCSOperator(S3ListOperator):
return files
# Following functionality may be better suited in
- # airflow/contrib/hooks/gcs.py
+ # airflow/providers/google/cloud/hooks/gcs.py
@staticmethod
def _gcs_object_is_directory(bucket):
_, blob = _parse_gcs_url(bucket)
diff --git a/airflow/providers/google/cloud/utils/mlengine_prediction_summary.py b/airflow/providers/google/cloud/utils/mlengine_prediction_summary.py
index 3f15526..63733b2 100644
--- a/airflow/providers/google/cloud/utils/mlengine_prediction_summary.py
+++ b/airflow/providers/google/cloud/utils/mlengine_prediction_summary.py
@@ -36,6 +36,7 @@ Args:
The summary will have an additional key, 'count', to represent the
total number of instances, so the keys shouldn't include 'count'.
# Usage example:
+from airflow.providers.google.cloud.operators.dataflow import DataflowCreatePythonJobOperator
def get_metric_fn():
import math # all imports must be outside of the function to be passed.
def metric_fn(inst):
@@ -48,7 +49,7 @@ def get_metric_fn():
return (log_loss, squared_err)
return metric_fn
metric_fn_encoded = base64.b64encode(dill.dumps(get_metric_fn(), recurse=True))
-airflow.contrib.operators.DataFlowPythonOperator(
+DataflowCreatePythonJobOperator(
task_id="summary-prediction",
py_options=["-m"],
py_file="airflow.providers.google.cloud.utils.mlengine_prediction_summary",
diff --git a/docs/conf.py b/docs/conf.py
index 2fc0539..be4dfde 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -500,9 +500,6 @@ autoapi_template_dir = 'autoapi_templates'
# A list of patterns to ignore when finding files
autoapi_ignore = [
'*/airflow/kubernetes/kubernetes_request_factory/*',
- '*/airflow/contrib/sensors/*',
- '*/airflow/contrib/hooks/*',
- '*/airflow/contrib/operators/*',
'*/_internal*',
'*/node_modules/*',
'*/migrations/*',
diff --git a/docs/exts/docroles.py b/docs/exts/docroles.py
index 4f74a66..858f92e 100644
--- a/docs/exts/docroles.py
+++ b/docs/exts/docroles.py
@@ -75,8 +75,7 @@ def template_field_role(app,
Sample usage::
- :template-fields:
- `airflow.contrib.operators.gcp_natural_language_operator.CloudLanguageAnalyzeSentimentOperator`
+ :template-fields:`airflow.operators.bash.BashOperator`
For further information look at:
diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst
index 40edf36..9c791d3 100644
--- a/docs/operators-and-hooks-ref.rst
+++ b/docs/operators-and-hooks-ref.rst
@@ -1090,7 +1090,7 @@ All hooks are based on :class:`airflow.providers.yandex.hooks.yandex.YandexCloud
.. note::
You can learn how to use Yandex.Cloud integrations by analyzing the
- `example DAG <https://github.com/apache/airflow/tree/master/airflow/contrib/example_dags/example_yandexcloud_dataproc.py>`_
+ `example DAG <https://github.com/apache/airflow/blob/master/airflow/providers/yandex/example_dags/example_yandexcloud_dataproc.py>`_
Service operators and hooks
'''''''''''''''''''''''''''
diff --git a/tests/providers/ftp/hooks/test_ftp.py b/tests/providers/ftp/hooks/test_ftp.py
index 23b6438..c38eec8 100644
--- a/tests/providers/ftp/hooks/test_ftp.py
+++ b/tests/providers/ftp/hooks/test_ftp.py
@@ -20,7 +20,7 @@ import io
import unittest
from unittest import mock
-from airflow.contrib.hooks import ftp_hook as fh
+from airflow.providers.ftp.hooks import ftp as fh
class TestFTPHook(unittest.TestCase):
diff --git a/tests/task/task_runner/test_cgroup_task_runner.py b/tests/task/task_runner/test_cgroup_task_runner.py
index d61ceca..c83cf62 100644
--- a/tests/task/task_runner/test_cgroup_task_runner.py
+++ b/tests/task/task_runner/test_cgroup_task_runner.py
@@ -18,7 +18,7 @@
import unittest
from unittest import mock
-from airflow.contrib.task_runner.cgroup_task_runner import CgroupTaskRunner
+from airflow.task.task_runner.cgroup_task_runner import CgroupTaskRunner
class TestCgroupTaskRunner(unittest.TestCase):