You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2020/11/04 01:18:42 UTC
[airflow] branch master updated: Remove the ability to import
operators and sensors from plugins (#12072)
This is an automated email from the ASF dual-hosted git repository.
kaxilnaik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/master by this push:
new 5e8b537 Remove the ability to import operators and sensors from plugins (#12072)
5e8b537 is described below
commit 5e8b537b85ce6aa355966df08df5c1846adb4cd6
Author: Ash Berlin-Taylor <as...@firemirror.com>
AuthorDate: Wed Nov 4 01:17:18 2020 +0000
Remove the ability to import operators and sensors from plugins (#12072)
We have deprecated this in #12069 (for inclusion in 1.10.13) and the
docs http://airflow.apache.org/docs/stable/howto/custom-operator.html
already show how to do this without a plugin.
Closes #9498
---
UPDATING.md | 20 +++++++++++
airflow/cli/commands/plugins_command.py | 4 ---
airflow/plugins_manager.py | 25 +------------
airflow/www/views.py | 2 --
docs/howto/custom-operator.rst | 2 +-
docs/plugins.rst | 56 +++++-------------------------
tests/cli/commands/test_plugins_command.py | 1 -
tests/plugins/test_plugins_manager.py | 16 +++------
tests/test_utils/mock_plugins.py | 2 --
9 files changed, 35 insertions(+), 93 deletions(-)
diff --git a/UPDATING.md b/UPDATING.md
index f958831..d84a93a 100644
--- a/UPDATING.md
+++ b/UPDATING.md
@@ -50,6 +50,26 @@ assists users migrating to a new version.
## Airflow Master
+### Adding Operators and Sensors via plugins is no longer supported
+
+Operators and Sensors should no longer be registered or imported via Airflow's plugin mechanism -- these types of classes are just treated as plain python classes by Airflow, so there is no need to register them with Airflow.
+
+If you previously had a `plugins/my_plugin.py` and you used it like this in a DAG:
+
+```
+from airflow.operators.my_plugin import MyOperator
+```
+
+You should instead import it as:
+
+```
+from my_plugin import MyOperator
+```
+
+The name under `airflow.operators.` was the plugin name, where as in the second example it is the python module name where the operator is defined.
+
+See http://airflow.apache.org/docs/stable/howto/custom-operator.html#define-an-operator-extra-link for more info.
+
### The default value for `[core] enable_xcom_pickling` has been changed to `False`
The pickle type for XCom messages has been replaced to JSON by default to prevent RCE attacks.
diff --git a/airflow/cli/commands/plugins_command.py b/airflow/cli/commands/plugins_command.py
index 40d12fe..785b8ed 100644
--- a/airflow/cli/commands/plugins_command.py
+++ b/airflow/cli/commands/plugins_command.py
@@ -25,8 +25,6 @@ from airflow import plugins_manager
PLUGINS_MANAGER_ATTRIBUTES_TO_DUMP = [
"plugins",
"import_errors",
- "operators_modules",
- "sensors_modules",
"hooks_modules",
"macros_modules",
"executors_modules",
@@ -39,8 +37,6 @@ PLUGINS_MANAGER_ATTRIBUTES_TO_DUMP = [
]
# list to maintain the order of items.
PLUGINS_ATTRIBUTES_TO_DUMP = [
- "operators",
- "sensors",
"hooks",
"executors",
"macros",
diff --git a/airflow/plugins_manager.py b/airflow/plugins_manager.py
index 99c05b9..8958742 100644
--- a/airflow/plugins_manager.py
+++ b/airflow/plugins_manager.py
@@ -38,8 +38,6 @@ import_errors: Dict[str, str] = {}
plugins = None # type: Optional[List[AirflowPlugin]]
# Plugin components to integrate as modules
-operators_modules: Optional[List[Any]] = None
-sensors_modules: Optional[List[Any]] = None
hooks_modules: Optional[List[Any]] = None
macros_modules: Optional[List[Any]] = None
executors_modules: Optional[List[Any]] = None
@@ -106,8 +104,6 @@ class AirflowPlugin:
name: Optional[str] = None
source: Optional[AirflowPluginSource] = None
- operators: List[Any] = []
- sensors: List[Any] = []
hooks: List[Any] = []
executors: List[Any] = []
macros: List[Any] = []
@@ -382,18 +378,11 @@ def integrate_dag_plugins() -> None:
"""Integrates operator, sensor, hook, macro plugins."""
# pylint: disable=global-statement
global plugins
- global operators_modules
- global sensors_modules
global hooks_modules
global macros_modules
# pylint: enable=global-statement
- if (
- operators_modules is not None
- and sensors_modules is not None
- and hooks_modules is not None
- and macros_modules is not None
- ):
+ if hooks_modules is not None and macros_modules is not None:
return
ensure_plugins_loaded()
@@ -403,8 +392,6 @@ def integrate_dag_plugins() -> None:
log.debug("Integrate DAG plugins")
- operators_modules = []
- sensors_modules = []
hooks_modules = []
macros_modules = []
@@ -412,19 +399,9 @@ def integrate_dag_plugins() -> None:
if plugin.name is None:
raise AirflowPluginException("Invalid plugin name")
- operators_module = make_module(f'airflow.operators.{plugin.name}', plugin.operators + plugin.sensors)
- sensors_module = make_module(f'airflow.sensors.{plugin.name}', plugin.sensors)
hooks_module = make_module(f'airflow.hooks.{plugin.name}', plugin.hooks)
macros_module = make_module(f'airflow.macros.{plugin.name}', plugin.macros)
- if operators_module:
- operators_modules.append(operators_module)
- sys.modules[operators_module.__name__] = operators_module # pylint: disable=no-member
-
- if sensors_module:
- sensors_modules.append(sensors_module)
- sys.modules[sensors_module.__name__] = sensors_module # pylint: disable=no-member
-
if hooks_module:
hooks_modules.append(hooks_module)
sys.modules[hooks_module.__name__] = hooks_module # pylint: disable=no-member
diff --git a/airflow/www/views.py b/airflow/www/views.py
index fe818f2..056d46e 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -2851,8 +2851,6 @@ class PluginView(AirflowBaseView):
]
plugins_attributes_to_dump = [
- "operators",
- "sensors",
"hooks",
"executors",
"macros",
diff --git a/docs/howto/custom-operator.rst b/docs/howto/custom-operator.rst
index 568799c..4aa12fd 100644
--- a/docs/howto/custom-operator.rst
+++ b/docs/howto/custom-operator.rst
@@ -61,7 +61,7 @@ Let's implement an example ``HelloOperator`` in a new file ``hello_operator.py``
For imports to work, you should place the file in a directory that
is present in the :envvar:`PYTHONPATH` env. Airflow adds ``dags/``, ``plugins/``, and ``config/`` directories
in the Airflow home to :envvar:`PYTHONPATH` by default. e.g., In our example,
- the file is placed in the ``custom_operator`` directory.
+ the file is placed in the ``custom_operator/`` directory.
See :doc:`../modules_management` for details on how Python and Airflow manage modules.
You can now use the derived custom operator as follows:
diff --git a/docs/plugins.rst b/docs/plugins.rst
index 8e0ecc0..b160c11 100644
--- a/docs/plugins.rst
+++ b/docs/plugins.rst
@@ -25,7 +25,7 @@ features to its core by simply dropping files in your
``$AIRFLOW_HOME/plugins`` folder.
The python modules in the ``plugins`` folder get imported,
-and **hooks**, **operators**, **sensors**, **macros** and web **views**
+and **hooks**, **macros** and web **views**
get integrated to Airflow's main collections and become available for use.
To troubleshoot issue with plugins, you can use ``airflow plugins`` command.
@@ -102,10 +102,6 @@ looks like:
class AirflowPlugin:
# The name of your plugin (str)
name = None
- # A list of class(es) derived from BaseOperator
- operators = []
- # A list of class(es) derived from BaseSensorOperator
- sensors = []
# A list of class(es) derived from BaseHook
hooks = []
# A list of references to inject into the macros namespace
@@ -143,22 +139,6 @@ You can derive it by inheritance (please refer to the example below). In the exa
defined as class attributes, but you can also define them as properties if you need to perform
additional initialization. Please note ``name`` inside this class must be specified.
-After the plugin is imported into Airflow,
-you can invoke it using statement like
-
-
-.. code-block:: python
-
- from airflow.{type, like "operators", "sensors"}.{name specified inside the plugin class} import *
-
-
-When you write your own plugins, make sure you understand them well.
-There are some essential properties for each type of plugin.
-For example,
-
-* For ``Operator`` plugin, an ``execute`` method is compulsory.
-* For ``Sensor`` plugin, a ``poke`` method returning a Boolean value is compulsory.
-
Make sure you restart the webserver and scheduler after making changes to plugins so that they take effect.
@@ -180,23 +160,13 @@ definitions in Airflow.
# Importing base classes that we need to derive
from airflow.hooks.base_hook import BaseHook
- from airflow.models import BaseOperator
from airflow.models.baseoperator import BaseOperatorLink
from airflow.providers.amazon.aws.transfers.gcs_to_s3 import GCSToS3Operator
- from airflow.sensors.base_sensor_operator import BaseSensorOperator
# Will show up under airflow.hooks.test_plugin.PluginHook
class PluginHook(BaseHook):
pass
- # Will show up under airflow.operators.test_plugin.PluginOperator
- class PluginOperator(BaseOperator):
- pass
-
- # Will show up under airflow.sensors.test_plugin.PluginSensorOperator
- class PluginSensorOperator(BaseSensorOperator):
- pass
-
# Will show up under airflow.macros.test_plugin.plugin_macro
# and in templates through {{ macros.test_plugin.plugin_macro }}
def plugin_macro():
@@ -255,8 +225,6 @@ definitions in Airflow.
# Defining the plugin class
class AirflowTestPlugin(AirflowPlugin):
name = "test_plugin"
- operators = [PluginOperator]
- sensors = [PluginSensorOperator]
hooks = [PluginHook]
macros = [plugin_macro]
flask_blueprints = [bp]
@@ -306,19 +274,18 @@ will automatically load the registered plugins from the entrypoint list.
# my_package/my_plugin.py
from airflow.plugins_manager import AirflowPlugin
- from airflow.models import BaseOperator
- from airflow.hooks.base_hook import BaseHook
-
- class MyOperator(BaseOperator):
- pass
+ from flask import Blueprint
- class MyHook(BaseHook):
- pass
+ # Creating a flask blueprint to integrate the templates and static folder
+ bp = Blueprint(
+ "test_plugin", __name__,
+ template_folder='templates', # registers airflow/plugins/templates as a Jinja template folder
+ static_folder='static',
+ static_url_path='/static/test_plugin')
class MyAirflowPlugin(AirflowPlugin):
name = 'my_namespace'
- operators = [MyOperator]
- hooks = [MyHook]
+ flask_blueprints = [bp]
.. code-block:: python
@@ -334,11 +301,6 @@ will automatically load the registered plugins from the entrypoint list.
}
)
-This will create a hook, and an operator accessible at:
-
-- ``airflow.hooks.my_namespace.MyHook``
-- ``airflow.operators.my_namespace.MyOperator``
-
Automatic reloading webserver
-----------------------------
diff --git a/tests/cli/commands/test_plugins_command.py b/tests/cli/commands/test_plugins_command.py
index d33adce..987a2d8 100644
--- a/tests/cli/commands/test_plugins_command.py
+++ b/tests/cli/commands/test_plugins_command.py
@@ -59,4 +59,3 @@ class TestPluginsCommand(unittest.TestCase):
stdout = temp_stdout.getvalue()
self.assertIn('plugins = [<class ', stdout)
self.assertIn('test-plugin-cli', stdout)
- self.assertIn('test_plugins_command.PluginOperator', stdout)
diff --git a/tests/plugins/test_plugins_manager.py b/tests/plugins/test_plugins_manager.py
index 07e500d..8b9fea4 100644
--- a/tests/plugins/test_plugins_manager.py
+++ b/tests/plugins/test_plugins_manager.py
@@ -108,18 +108,11 @@ class TestPluginsManager(unittest.TestCase):
name = "test_property_plugin"
@property
- def operators(self):
- from airflow.models.baseoperator import BaseOperator
-
- class PluginPropertyOperator(BaseOperator):
+ def hooks(self):
+ class TestPropertyHook(BaseHook):
pass
- return [PluginPropertyOperator]
-
- class TestNonPropertyHook(BaseHook):
- pass
-
- hooks = [TestNonPropertyHook]
+ return [TestPropertyHook]
with mock_plugin_manager(plugins=[AirflowTestPropertyPlugin()]):
from airflow import plugins_manager
@@ -127,8 +120,7 @@ class TestPluginsManager(unittest.TestCase):
plugins_manager.integrate_dag_plugins()
self.assertIn('AirflowTestPropertyPlugin', str(plugins_manager.plugins))
- self.assertIn('PluginPropertyOperator', str(plugins_manager.operators_modules[0].__dict__))
- self.assertIn("TestNonPropertyHook", str(plugins_manager.hooks_modules[0].__dict__))
+ self.assertIn("TestPropertyHook", str(plugins_manager.hooks_modules[0].__dict__))
def test_should_warning_about_incompatible_plugins(self):
class AirflowAdminViewsPlugin(AirflowPlugin):
diff --git a/tests/test_utils/mock_plugins.py b/tests/test_utils/mock_plugins.py
index 7ee6d48..7cce20b 100644
--- a/tests/test_utils/mock_plugins.py
+++ b/tests/test_utils/mock_plugins.py
@@ -20,8 +20,6 @@ from unittest import mock
PLUGINS_MANAGER_NULLABLE_ATTRIBUTES = [
"plugins",
- "operators_modules",
- "sensors_modules",
"hooks_modules",
"macros_modules",
"executors_modules",